nmcp-precomputed 3.0.7__tar.gz → 3.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/.github/workflows/publish.yml +2 -4
  2. nmcp_precomputed-3.1.0/Dockerfile +14 -0
  3. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/PKG-INFO +7 -3
  4. nmcp_precomputed-3.1.0/README.md +150 -0
  5. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/Taskfile.dist.yaml +6 -2
  6. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/Test.Dockerfile +1 -1
  7. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/pyproject.toml +8 -3
  8. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/__init__.py +2 -1
  9. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/__main__.py +3 -3
  10. nmcp_precomputed-3.1.0/src/nmcp/api/__init__.py +1 -0
  11. nmcp_precomputed-3.1.0/src/nmcp/api/app.py +90 -0
  12. nmcp_precomputed-3.1.0/src/nmcp/api/models.py +68 -0
  13. nmcp_precomputed-3.1.0/src/nmcp/api/output_location.py +80 -0
  14. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/remote_data_client.py +24 -0
  15. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/__init__.py +3 -2
  16. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/nmcp_precomputed.py +56 -51
  17. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/segment_tag_property.py +14 -2
  18. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed_worker.py +50 -11
  19. nmcp_precomputed-3.1.0/src/nmcp/service.py +33 -0
  20. nmcp_precomputed-3.1.0/uv.lock +3624 -0
  21. nmcp_precomputed-3.0.7/Dockerfile +0 -9
  22. nmcp_precomputed-3.0.7/README.md +0 -70
  23. nmcp_precomputed-3.0.7/version.json +0 -4
  24. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/.gitattributes +0 -0
  25. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/.gitignore +0 -0
  26. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/dev.sh +0 -0
  27. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/docker-compose.yml +0 -0
  28. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/docker-entry.sh +0 -0
  29. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/__init__.py +0 -0
  30. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/gql/__init__.py +0 -0
  31. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/gql/precomputed_queries.gql +0 -0
  32. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/portal_reconstruction.py +0 -0
  33. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/precomputed_entry.py +0 -0
  34. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/export/__init__.py +0 -0
  35. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/export/export_format.py +0 -0
  36. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/export/reconstruction_space.py +0 -0
  37. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/from_portal_json.py +0 -0
  38. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/from_service.py +0 -0
  39. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/list_skeletons.py +0 -0
  40. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/neuron_structure.py +0 -0
  41. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/nmcp_skeleton.py +0 -0
  42. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/segment_info.py +0 -0
  43. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/segment_property.py +0 -0
  44. {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/remove_skeleton.py +0 -0
@@ -37,10 +37,8 @@ jobs:
37
37
  with:
38
38
  version: 3.x
39
39
  repo-token: ${{ secrets.GITHUB_TOKEN }}
40
- - name: Set up Node.js
41
- uses: actions/setup-node@v3
42
- with:
43
- node-version: "18.x"
40
+ - name: Install uv
41
+ uses: astral-sh/setup-uv@v6
44
42
  - name: Login to Github Packages
45
43
  uses: docker/login-action@v2
46
44
  with:
@@ -0,0 +1,14 @@
1
+ FROM python:3.10
2
+
3
+ # Supplied by the Taskfile from pyproject.toml; selects the wheel `uv build` wrote for this version.
4
+ ARG VERSION
5
+
6
+ WORKDIR /app
7
+
8
+ COPY --chmod=755 docker-entry.sh ./
9
+
10
+ COPY dist/nmcp_precomputed-${VERSION}-py3-none-any.whl ./
11
+
12
+ RUN pip install --no-cache-dir "./nmcp_precomputed-${VERSION}-py3-none-any.whl[numerics,api]"
13
+
14
+ CMD ["./docker-entry.sh"]
@@ -1,18 +1,22 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: nmcp-precomputed
3
- Version: 3.0.7
3
+ Version: 3.1.0
4
4
  Summary: Neuron Morphology Community Portal Precomputed Toolbox
5
5
  License: MIT
6
6
  Requires-Python: <3.13,>=3.10
7
7
  Requires-Dist: gql[requests]
8
+ Provides-Extra: api
9
+ Requires-Dist: fastapi>=0.162.2; extra == 'api'
10
+ Requires-Dist: uvicorn>=0.54.0; extra == 'api'
8
11
  Provides-Extra: numerics
9
12
  Requires-Dist: allensdk; extra == 'numerics'
10
13
  Requires-Dist: cloud-files; extra == 'numerics'
11
14
  Requires-Dist: cloud-volume; extra == 'numerics'
12
- Requires-Dist: numpy<1.24; extra == 'numerics'
15
+ Requires-Dist: numpy<2,>=1.23.5; extra == 'numerics'
13
16
  Requires-Dist: pandas; extra == 'numerics'
14
17
  Provides-Extra: publish
15
18
  Requires-Dist: build; extra == 'publish'
16
19
  Requires-Dist: twine; extra == 'publish'
17
20
  Provides-Extra: test
21
+ Requires-Dist: httpx>=0.28.1; extra == 'test'
18
22
  Requires-Dist: pytest==8.2.0; extra == 'test'
@@ -0,0 +1,150 @@
1
+ # NMCP Precomputed
2
+ The NMCP Precomputed service generates reconstruction skeletons in Neuroglancer's precomputed format. It periodically
3
+ polls the API service for reconstructions that are ready for new or updated skeletons.
4
+
5
+ Given the current behavior or Neuroglancer layers and the NMCP viewer feature requirements, the service currently
6
+ generated separate data sources for the full reconstruction, as well as axon-only and dendrite-only versions.
7
+
8
+ The service uses the chunked API for acquiring reconstruction data and should be compatible with dense reconstruction
9
+ sources.
10
+
11
+ ## Local Development
12
+ A local S3-compatible service may be used during local development to avoid AWS or other cloud service storage charges.
13
+
14
+ _Note that at this time, cloud-files only appears to support this option from Linux, WSL, or MacOS._
15
+
16
+ One functional option is to run a local [MinIO](https://www.min.io/) Docker container. The following steps describe the
17
+ process (and assume the default configuration of port 9000 for MinIO). For additional details see either the MinIO
18
+ documentation, [cloud-files](https://github.com/seung-lab/cloud-files), and
19
+ [cloud-volume](https://github.com/seung-lab/cloud-volume).
20
+
21
+ A preconfigured instance is defined in `docker-compose.yml` that can be started with the following script
22
+
23
+ ```bash
24
+ docker compose -p nmcp up -d
25
+ ```
26
+
27
+ Attach to the running container to configure the precomputed bucket.
28
+
29
+ Find the container id
30
+ ```bash
31
+ docker ps
32
+ ```
33
+
34
+ Attach to the instance
35
+ ```bash
36
+ docker exec -it <container-id> /bin/bash
37
+ ```
38
+
39
+ Define an `mc` alias for the server, create the bucket, and allow public access for the Neuroglancer viewer. The
40
+ following assumes the username/password defined in `docker-compose.yml`. This is performed in the container after
41
+ attaching above, not on the host (unless you choose to install the `mc` tools on your host machine).
42
+ ```bash
43
+ mc alias set myminio http://localhost:9000 root password
44
+ mc mb myminio/aind-neuron-morphology-community-portal-local/ngv01/
45
+ mc anonymous set public myminio/aind-neuron-morphology-community-portal-local/ngv01
46
+ ```
47
+
48
+ Exit the container instance. On the host (in the Python environment created for this project w/cloud-files installed),
49
+ create an alias for the server.
50
+
51
+ ```bash
52
+ cloudfiles alias add minio s3://http://127.0.0.1:9000/
53
+ ```
54
+
55
+ Add the username and password as a secrets file in `~/.cloudvolume/secrets/minio-secret.json` (assumes defaults used
56
+ in the compose file).
57
+ ```json
58
+ {
59
+ "AWS_ACCESS_KEY_ID": "root",
60
+ "AWS_SECRET_ACCESS_KEY": "password"
61
+ }
62
+ ```
63
+
64
+ Although the bucket is set for public access in this development example, adding the authentication when generating
65
+ the precomputed data mimics the typical behavior in actual deployments.
66
+
67
+ When running the precomputed worker, pass the alias as the precomputed output argument, _e.g._, `-o minio://aind-neuron-morphology-community-portal-local/ngv01`
68
+
69
+ Here the current standard path is used as the base location (`ngv01`), however anything can used so long as it is also
70
+ used in the `NMCP_PRECOMPUTED_OUTPUT` environment variable for `nmcp-client`.
71
+
72
+ ## On-Demand API
73
+ In addition to polling, the service exposes an HTTP API that converts a posted portal-format reconstruction into a
74
+ complete, self-contained precomputed dataset (one Neuroglancer layer holding a single skeleton) at a location chosen by
75
+ the caller. It uses the same conversion as the worker, so the skeleton output is identical.
76
+
77
+ The API runs in the same process as the polling worker. API requests run concurrently with worker generations and never
78
+ wait for one to finish; they are serialized only with other API requests.
79
+
80
+ ### Startup
81
+ `python -m nmcp` (and therefore `docker-entry.sh`) starts both. Before either runs, it:
82
+
83
+ 1. validates and normalizes the worker output location (`-o`, required);
84
+ 2. loads the Allen structure tree used for soma-location tags.
85
+
86
+ If either step fails, the process exits so the container restarts it. After the first successful load, the tree is
87
+ read from allensdk's `mouse_connectivity/` cache, so later restarts do not need network access.
88
+
89
+ The worker output location is normalized at startup: protocol-only (`s3://`, `gs://`, `file://` or a configured
90
+ cloud-files alias), with no trailing slash and no `precomputed://` prefix or `|precomputed:` suffix. The worker writes
91
+ under the normalized location, and a warning is logged when the configured value differed from it. Set
92
+ `NMCP_PRECOMPUTED_OUTPUT` in normalized form, because `nmcp-client` composes its own dataset paths (`<value>/full`, …)
93
+ from the same value and must read the same datasets the worker writes.
94
+
95
+ | Environment variable | Default | Purpose |
96
+ |---|---|---|
97
+ | `NMCP_PRECOMPUTED_API_HOST` | `0.0.0.0` | Interface the API listens on |
98
+ | `NMCP_PRECOMPUTED_API_PORT` | `5000` | Port the API listens on |
99
+
100
+ ### Endpoints
101
+ `GET /` returns `"NMCP Precomputed API"`.
102
+
103
+ `POST /generateskeleton` takes the output location, an optional segment id (default `1`, range `1` to `2^64 - 1`) and
104
+ the reconstruction in the portal export format:
105
+
106
+ ```json
107
+ {
108
+ "location": "s3://bucket/path/to/dataset",
109
+ "segmentId": 1,
110
+ "reconstruction": {
111
+ "id": "…",
112
+ "neuron": {"label": "N004", "specimen": {"label": "648434", "genotype": "Dbh-Cre_KI"}},
113
+ "nodes": [{"index": 1, "structure": 1, "x": 0.0, "y": 0.0, "z": 0.0, "radius": 1.0, "parentIndex": -1, "atlasStructure": 202}]
114
+ }
115
+ }
116
+ ```
117
+
118
+ It returns once the dataset is written:
119
+
120
+ ```json
121
+ {
122
+ "serviceVersion": "3.0.7",
123
+ "reconstructionId": "…",
124
+ "result": {"location": "s3://bucket/path/to/dataset", "segmentId": 1, "nodeCount": 9, "edgeCount": 8},
125
+ "errorKind": null, "errorDescription": null, "errorInfo": null
126
+ }
127
+ ```
128
+
129
+ Every request the service handles returns 200. On failure `result` is null and `errorKind` is one of:
130
+
131
+ - `InvalidLocation`: the location cannot be parsed, has no explicit protocol, or names a format other than
132
+ precomputed.
133
+ - `ProtectedLocation`: the location is the worker's output location or lies under it. Locations are compared after
134
+ resolving cloud-files aliases, trailing slashes, the precomputed prefix and suffix, and local symlinks.
135
+ - `GenerationError`: writing the dataset failed; `errorInfo` holds the exception type and message.
136
+
137
+ A body that fails validation (a missing field, a segment id of 0) gets FastAPI's standard 422 response.
138
+
139
+ The dataset is written over whatever already exists at the location without clearing it first. Skeleton files from
140
+ earlier calls with other segment ids remain, though the segment properties list only the latest segment.
141
+
142
+ ### Running Locally
143
+ ```bash
144
+ uv sync --extra numerics --extra api --extra test
145
+ uv run python -m nmcp -u <graphql-url> -a <key> -o <worker-output>
146
+ ```
147
+
148
+ ```bash
149
+ curl -s -X POST localhost:5000/generateskeleton -H "Content-Type: application/json" -d @body.json
150
+ ```
@@ -5,7 +5,7 @@ vars:
5
5
  IMAGE: allenneuraldynamics/nmcp-precomputed-worker
6
6
  REPOIMAGE: "{{.REPOSITORY}}/{{.IMAGE}}"
7
7
  VERSION:
8
- sh: jq -r .version version.json
8
+ sh: uv version --short
9
9
  MAJOR:
10
10
  sh: string={{.VERSION}} && arr=(${string//"."/ }) && echo ${arr[0]}
11
11
  MINOR:
@@ -18,9 +18,13 @@ vars:
18
18
  IMAGELATEST: "{{.REPOIMAGE}}:latest"
19
19
 
20
20
  tasks:
21
+ module:
22
+ cmds:
23
+ - uv build --wheel
21
24
  build:
22
25
  cmds:
23
- - docker build --platform linux/amd64 --tag {{.IMAGEWITHVERSION}} .
26
+ - task: module
27
+ - docker build --platform linux/amd64 --build-arg VERSION={{.VERSION}} --tag {{.IMAGEWITHVERSION}} .
24
28
  - docker tag {{.IMAGEWITHVERSION}} {{.IMAGEWITHMINORVERSION}}
25
29
  - docker tag {{.IMAGEWITHMINORVERSION}} {{.IMAGEWITHREVERSION}}
26
30
  - docker tag {{.IMAGEWITHREVERSION}} {{.IMAGELATEST}}
@@ -8,6 +8,6 @@ COPY pyproject.toml ./
8
8
 
9
9
  COPY src ./src
10
10
 
11
- RUN pip install --no-cache-dir .
11
+ RUN pip install --no-cache-dir ".[numerics,api]"
12
12
 
13
13
  CMD ["./docker-entry.sh"]
@@ -12,7 +12,7 @@ exclude = [
12
12
 
13
13
  [project]
14
14
  name = "nmcp-precomputed"
15
- version = "3.0.7"
15
+ version = "3.1.0"
16
16
  description = "Neuron Morphology Community Portal Precomputed Toolbox"
17
17
  license = { text = "MIT" }
18
18
  requires-python = ">=3.10,<3.13"
@@ -22,7 +22,7 @@ dependencies = [
22
22
 
23
23
  [project.optional-dependencies]
24
24
  numerics = [
25
- "numpy<1.24",
25
+ "numpy>=1.23.5,<2",
26
26
  "pandas",
27
27
  "allensdk",
28
28
  "cloud-volume",
@@ -30,13 +30,18 @@ numerics = [
30
30
  ]
31
31
 
32
32
  test = [
33
- "pytest==8.2.0"
33
+ "httpx>=0.28.1",
34
+ "pytest==8.2.0",
34
35
  ]
35
36
 
36
37
  publish = [
37
38
  "build",
38
39
  "twine"
39
40
  ]
41
+ api = [
42
+ "fastapi>=0.162.2",
43
+ "uvicorn>=0.54.0",
44
+ ]
40
45
 
41
46
  [tool.pytest.ini_options]
42
47
  addopts = ["-W ignore::DeprecationWarning"]
@@ -1,5 +1,6 @@
1
1
  from .export import ExportFormat, ReconstructionSpace
2
2
  from .data import RemoteDataClient, PrecomputedEntry
3
3
  from .precomputed import SegmentInfo, SegmentProperty, SegmentTagProperty, SomaSegmentTagProperty, SegmentPropertyValues
4
- from .precomputed import (ensure_bucket_folders, create_from_json_files, create_from_reconstruction, remove_skeleton,
4
+ from .precomputed import (ensure_bucket_folders, create_from_json_files, create_from_reconstruction,
5
+ create_standalone_from_reconstruction, default_cloudvolume_info, remove_skeleton,
5
6
  list_skeletons)
@@ -1,13 +1,13 @@
1
1
  import argparse
2
2
 
3
- from .precomputed_worker import start
3
+ from .service import run
4
4
 
5
5
  parser = argparse.ArgumentParser()
6
6
 
7
7
  parser.add_argument("-u", "--url", help="URL of the GraphQL service")
8
8
  parser.add_argument("-a", "--authkey", help="authorization header for GraphQL service")
9
- parser.add_argument("-o", "--output", help="the output cloud volume location")
9
+ parser.add_argument("-o", "--output", required=True, help="the output cloud volume location")
10
10
 
11
11
  args = parser.parse_args()
12
12
 
13
- start(args.url, args.authkey, args.output)
13
+ run(args.url, args.authkey, args.output)
@@ -0,0 +1 @@
1
+ from .app import create_app, serve
@@ -0,0 +1,90 @@
1
+ import importlib.metadata
2
+ import logging
3
+ import os
4
+ import threading
5
+ from typing import cast
6
+
7
+ import uvicorn
8
+ from fastapi import FastAPI
9
+
10
+ from .models import ErrorKind, SkeletonRequest, SkeletonResponse, SkeletonResult
11
+ from .output_location import parse_output_location
12
+ from ..data.portal_reconstruction import PortalReconstruction
13
+ from ..precomputed import create_standalone_from_reconstruction, default_cloudvolume_info
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ _SERVICE_NAME = "NMCP Precomputed API"
18
+
19
+ _SERVICE_VERSION = importlib.metadata.version("nmcp-precomputed")
20
+
21
+
22
+ def create_app(worker_output: str) -> FastAPI:
23
+ protected = parse_output_location(worker_output, require_protocol=False)
24
+
25
+ generation_lock = threading.Lock()
26
+
27
+ app = FastAPI(title=_SERVICE_NAME)
28
+
29
+ @app.get("/")
30
+ def read_root() -> str:
31
+ return _SERVICE_NAME
32
+
33
+ # A plain `def`, so FastAPI runs the blocking generation in its threadpool rather than on the event loop.
34
+ @app.post("/generateskeleton")
35
+ def generate_skeleton(request: SkeletonRequest) -> SkeletonResponse:
36
+ reconstruction_id = request.reconstruction.id
37
+
38
+ def response(**kwargs) -> SkeletonResponse:
39
+ return SkeletonResponse(service_version=_SERVICE_VERSION, reconstruction_id=reconstruction_id, **kwargs)
40
+
41
+ logger.info(f"received on-demand skeleton request for reconstruction {reconstruction_id} "
42
+ f"({len(request.reconstruction.nodes)} nodes, segment {request.segment_id}) at "
43
+ f"'{request.location}'")
44
+
45
+ try:
46
+ location = parse_output_location(request.location)
47
+ except ValueError as ex:
48
+ logger.warning(f"rejected on-demand skeleton for reconstruction {reconstruction_id}: invalid output "
49
+ f"location '{request.location}': {ex}")
50
+ return response(error_kind=ErrorKind.INVALID_LOCATION,
51
+ error_description=f"Invalid output location for reconstruction {reconstruction_id}",
52
+ error_info=str(ex))
53
+
54
+ if location.is_at_or_under(protected):
55
+ logger.warning(f"rejected on-demand skeleton for reconstruction {reconstruction_id} at "
56
+ f"{location.cloudpath}: at or under the worker output {protected.cloudpath}")
57
+ return response(error_kind=ErrorKind.PROTECTED_LOCATION,
58
+ error_description=f"The output location for reconstruction {reconstruction_id} is at "
59
+ f"or under the worker's output location",
60
+ error_info=location.cloudpath)
61
+
62
+ # Chosen here so an optional request argument can replace it without touching the generation code.
63
+ dataset_info = default_cloudvolume_info()
64
+
65
+ reconstruction = cast(PortalReconstruction, request.reconstruction.model_dump(by_alias=True))
66
+
67
+ try:
68
+ with generation_lock:
69
+ skeleton = create_standalone_from_reconstruction(reconstruction, location.cloudpath,
70
+ request.segment_id, cloud_files_info=dataset_info)
71
+ except Exception as ex:
72
+ logger.exception(f"failed to generate on-demand skeleton for reconstruction {reconstruction_id} at "
73
+ f"{location.cloudpath}")
74
+ return response(error_kind=ErrorKind.GENERATION_ERROR,
75
+ error_description=f"Error generating precomputed skeleton for reconstruction "
76
+ f"{reconstruction_id}",
77
+ error_info=f"{type(ex).__name__}: {ex}")
78
+
79
+ logger.info(f"generated on-demand skeleton {request.segment_id} for reconstruction {reconstruction_id} at "
80
+ f"{location.cloudpath}")
81
+
82
+ return response(result=SkeletonResult(location=location.cloudpath, segment_id=request.segment_id,
83
+ node_count=len(skeleton.vertices), edge_count=len(skeleton.edges)))
84
+
85
+ return app
86
+
87
+
88
+ def serve(app: FastAPI) -> None:
89
+ uvicorn.run(app, host=os.environ.get("NMCP_PRECOMPUTED_API_HOST", "0.0.0.0"),
90
+ port=int(os.environ.get("NMCP_PRECOMPUTED_API_PORT", "5000")), access_log=False)
@@ -0,0 +1,68 @@
1
+ from enum import Enum
2
+
3
+ from pydantic import AliasGenerator, BaseModel, ConfigDict, Field
4
+ from pydantic.alias_generators import to_camel
5
+
6
+
7
+ class _RequestModel(BaseModel):
8
+ model_config = ConfigDict(alias_generator=to_camel)
9
+
10
+
11
+ class _ResponseModel(BaseModel):
12
+ model_config = ConfigDict(alias_generator=AliasGenerator(serialization_alias=to_camel))
13
+
14
+
15
+ class PortalSpecimenInput(_RequestModel):
16
+ label: str
17
+ genotype: str | None = None
18
+
19
+
20
+ class PortalNeuronInput(_RequestModel):
21
+ label: str
22
+ specimen: PortalSpecimenInput
23
+
24
+
25
+ class PortalNodeInput(_RequestModel):
26
+ index: int
27
+ structure: int
28
+ x: float
29
+ y: float
30
+ z: float
31
+ radius: float
32
+ parent_index: int
33
+ atlas_structure: int | None = None
34
+
35
+
36
+ class ReconstructionInput(_RequestModel):
37
+ id: str
38
+ neuron: PortalNeuronInput
39
+ nodes: list[PortalNodeInput]
40
+
41
+
42
+ class SkeletonRequest(_RequestModel):
43
+ location: str
44
+ # 0 is Neuroglancer's background segment, and segment ids are uint64.
45
+ segment_id: int = Field(default=1, ge=1, le=2 ** 64 - 1)
46
+ reconstruction: ReconstructionInput
47
+
48
+
49
+ class ErrorKind(str, Enum):
50
+ INVALID_LOCATION = "InvalidLocation"
51
+ PROTECTED_LOCATION = "ProtectedLocation"
52
+ GENERATION_ERROR = "GenerationError"
53
+
54
+
55
+ class SkeletonResult(_ResponseModel):
56
+ location: str
57
+ segment_id: int
58
+ node_count: int
59
+ edge_count: int
60
+
61
+
62
+ class SkeletonResponse(_ResponseModel):
63
+ service_version: str
64
+ reconstruction_id: str
65
+ result: SkeletonResult | None = None
66
+ error_kind: ErrorKind | None = None
67
+ error_description: str | None = None
68
+ error_info: str | None = None
@@ -0,0 +1,80 @@
1
+ import os
2
+ from dataclasses import dataclass
3
+
4
+ from cloudfiles import paths
5
+
6
+ _PRECOMPUTED_PREFIX = "precomputed://"
7
+
8
+
9
+ @dataclass(frozen=True)
10
+ class OutputLocation:
11
+ cloudpath: str
12
+ """Protocol-only location to write to: no `precomputed://` prefix or `|precomputed:` suffix, no trailing slash."""
13
+ protocol: str
14
+ host: str | None
15
+ bucket: str | None
16
+ path: str
17
+ """Comparison path: no trailing slash, and symlinks resolved for `file` locations."""
18
+
19
+ def is_at_or_under(self, other: "OutputLocation") -> bool:
20
+ if (self.protocol, self.host, self.bucket) != (other.protocol, other.host, other.bucket):
21
+ return False
22
+
23
+ if self.protocol == "file":
24
+ # Root-aware where a string-prefix test is not: a protected "/" covers every local path.
25
+ return os.path.commonpath([self.path, other.path]) == other.path
26
+
27
+ return other.path == "" or self.path == other.path or self.path.startswith(f"{other.path}/")
28
+
29
+
30
+ def parse_output_location(location: str, require_protocol: bool = True) -> OutputLocation:
31
+ """
32
+ Parse a cloud-files location the way CloudFiles itself resolves it, so that comparisons see the destination that
33
+ will actually be written to rather than the caller's spelling. Raises `ValueError` for anything unusable.
34
+ """
35
+ stripped = _remove_precomputed_prefix(location.strip())
36
+
37
+ if len(stripped) == 0:
38
+ raise ValueError("the output location is empty")
39
+
40
+ try:
41
+ fmt, protocol, _, _, _ = paths.extract_format_protocol(stripped, allow_defaults=False)
42
+ normalized = paths.normalize(stripped)
43
+ extracted = paths.extract(normalized)
44
+ except Exception as ex:
45
+ raise ValueError(f"unsupported location '{location}': {ex}") from ex
46
+
47
+ if fmt not in (None, "precomputed"):
48
+ raise ValueError(f"unsupported format '{fmt}' in location '{location}'; only precomputed is supported")
49
+
50
+ # cloud-files treats an unknown or mis-cased scheme, or a bare path, as a path relative to the working directory.
51
+ if require_protocol and protocol is None:
52
+ raise ValueError(f"the location '{location}' has no recognized protocol (e.g. s3://, gs://, file://)")
53
+
54
+ if extracted.protocol == "file":
55
+ comparison_path = os.path.realpath(extracted.path)
56
+ else:
57
+ comparison_path = extracted.path.rstrip("/")
58
+
59
+ return OutputLocation(_to_cloudpath(normalized), extracted.protocol, extracted.host, extracted.bucket,
60
+ comparison_path)
61
+
62
+
63
+ def _remove_precomputed_prefix(location: str) -> str:
64
+ if location.startswith(_PRECOMPUTED_PREFIX):
65
+ return location[len(_PRECOMPUTED_PREFIX):]
66
+
67
+ return location
68
+
69
+
70
+ def _to_cloudpath(normalized: str) -> str:
71
+ # `normalize` turns the `…|precomputed:` tail spelling into a `precomputed://` prefix, and the generation code
72
+ # adds its own, so it is removed here as well as from the raw input.
73
+ scheme, separator, rest = _remove_precomputed_prefix(normalized).partition("://")
74
+
75
+ trimmed = rest.rstrip("/")
76
+
77
+ if len(trimmed) == 0 and rest.startswith("/"):
78
+ trimmed = "/"
79
+
80
+ return f"{scheme}{separator}{trimmed}"
@@ -1,6 +1,7 @@
1
1
  import logging
2
2
  from datetime import datetime
3
3
  from enum import IntEnum
4
+ from urllib.parse import urlparse
4
5
 
5
6
  from typing_extensions import List
6
7
 
@@ -23,8 +24,31 @@ class PrecomputedStatus(IntEnum):
23
24
  FailedToGenerate = 400
24
25
 
25
26
 
27
+ class _GraphQLRetryFilter(logging.Filter):
28
+ """
29
+ Drops urllib3's per-attempt "Retrying" warnings for GraphQL requests. The worker already logs one warning per
30
+ outage with the final reason, and every poll during an outage would otherwise add one line per retry. Requests to
31
+ other paths, such as cloud-files uploads, keep theirs.
32
+ """
33
+
34
+ def __init__(self, path: str):
35
+ super().__init__()
36
+ self._path = path
37
+
38
+ def filter(self, record: logging.LogRecord) -> bool:
39
+ # urllib3 logs `"Retrying (%r) after connection broken by '%r': %s", retries, err, url`, where url is the
40
+ # request path.
41
+ is_retry = isinstance(record.msg, str) and record.msg.startswith("Retrying")
42
+
43
+ return not (is_retry and isinstance(record.args, tuple) and len(record.args) > 0 and
44
+ record.args[-1] == self._path)
45
+
46
+
26
47
  class RemoteDataClient:
27
48
  def __init__(self, url: str, auth_key: str):
49
+ # requests sends "/" for a URL with no path.
50
+ logging.getLogger("urllib3.connectionpool").addFilter(_GraphQLRetryFilter(urlparse(url).path or "/"))
51
+
28
52
  transport = RequestsHTTPTransport(
29
53
  url=url,
30
54
  verify=True,
@@ -1,6 +1,7 @@
1
1
  from .segment_property import SegmentProperty
2
- from .segment_tag_property import SegmentTagProperty, SomaSegmentTagProperty
2
+ from .segment_tag_property import SegmentTagProperty, SomaSegmentTagProperty, load_structure_tree
3
3
  from .segment_info import SegmentInfo, SegmentPropertyValues
4
4
  from .neuron_structure import NeuronStructure
5
5
  from .nmcp_precomputed import (ensure_bucket_folders, create_from_json_files, create_from_reconstruction,
6
- remove_skeleton, list_skeletons)
6
+ create_standalone_from_reconstruction, default_cloudvolume_info, remove_skeleton,
7
+ list_skeletons)