nmcp-precomputed 3.0.4__tar.gz

nmcp_precomputed-3.0.4/.github/workflows/publish-docker.yml

@@ -0,0 +1,33 @@
+name: Publish Docker Images
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Install Task
+        uses: arduino/setup-task@v2
+        with:
+          version: 3.x
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "18.x"
+      - name: Login to Github Packages
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and Push Docker Images
+        run: |
+          chmod +x docker-entry.sh
+          task release

nmcp_precomputed-3.0.4/.github/workflows/publish-pypi.yml

@@ -0,0 +1,26 @@
+name: Publish to PyPi
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Pull latest changes
+        run: git pull origin main
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          pip install --upgrade setuptools wheel twine build
+          python -m build
+          twine check dist/*
+      - name: Publish on PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.AIND_PYPI_TOKEN }}

nmcp_precomputed-3.0.4/.gitignore

@@ -0,0 +1,167 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+.idea/
+
+mouse_connectivity/
+
+# Taskfile.dist.yaml is used by the GitHub workflow.  Allow a local task file to push images to other hubs, etc.
+Taskfile.yaml
+
+GEMINI.md
+CLAUDE.md
+AGENT.md
+.claude/

nmcp_precomputed-3.0.4/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.10
+
+COPY docker-entry.sh ./
+
+RUN pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ nmcp-precomputed==3.0.7
+
+CMD ["./docker-entry.sh"]

nmcp_precomputed-3.0.4/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: nmcp-precomputed
+Version: 3.0.4
+Summary: Neuron Morphology Community Portal Precomputed Toolbox
+License: MIT
+Requires-Python: <3.11,>=3.10
+Requires-Dist: allensdk
+Requires-Dist: cloud-files
+Requires-Dist: cloud-volume
+Requires-Dist: gql[requests]
+Requires-Dist: numpy<1.24
+Requires-Dist: pandas
+Provides-Extra: publish
+Requires-Dist: build; extra == 'publish'
+Requires-Dist: twine; extra == 'publish'
+Provides-Extra: test
+Requires-Dist: pytest==8.2.0; extra == 'test'

nmcp_precomputed-3.0.4/README.md

@@ -0,0 +1,70 @@
+# NMCP Precomputed
+The NMCP Precomputed service generates reconstruction skeletons in Neuroglancer's precomputed format.  It periodically
+polls the API service for reconstructions that are ready for new or updated skeletons.
+
+Given the current behavior or Neuroglancer layers and the NMCP viewer feature requirements, the service currently
+generated separate data sources for the full reconstruction, as well as axon-only and dendrite-only versions.
+
+The service uses the chunked API for acquiring reconstruction data and should be compatible with dense reconstruction
+sources.
+
+## Local Development
+A local S3-compatible service may be used during local development to avoid AWS or other cloud service storage charges.
+
+_Note that at this time, cloud-files only appears to support this option from Linux, WSL, or MacOS._
+
+One functional option is to run a local [MinIO](https://www.min.io/) Docker container.  The following steps describe the
+process (and assume the default configuration of port 9000 for MinIO).  For additional details see either the MinIO
+documentation, [cloud-files](https://github.com/seung-lab/cloud-files), and 
+[cloud-volume](https://github.com/seung-lab/cloud-volume).
+
+A preconfigured instance is defined in `docker-compose.yml` that can be started with the following script
+
+```bash
+docker compose -p nmcp up -d
+```
+
+Attach to the running container to configure the precomputed bucket.
+
+Find the container id
+```bash
+docker ps
+```
+
+Attach to the instance
+```bash
+docker exec -it <container-id> /bin/bash
+```
+
+Define an `mc` alias for the server, create the bucket, and allow public access for the Neuroglancer viewer.  The 
+following assumes the username/password defined in `docker-compose.yml`.  This is performed in the container after
+attaching above, not on the host (unless you choose to install the `mc` tools on your host machine).
+```bash
+mc alias set myminio http://localhost:9000 minio_root_user minio_root_password
+mc mb myminio/aind-neuron-morphology-community-portal-local/ngv01/
+mc anonymous set public myminio/aind-neuron-morphology-community-portal-local/ngv01
+```
+
+Exit the container instance.  On the host (in the Python environment created for this project w/cloud-files installed),
+create an alias for the server.
+
+```bash
+cloudfiles alias add minio s3://http://127.0.0.1:9000/
+```
+
+Add the username and password as a secrets file in `~/.cloudvolume/secrets/minio-secret.json` (assumes defaults used
+in the compose file).
+```json
+{
+	"AWS_ACCESS_KEY_ID": "minio_root_user",
+	"AWS_SECRET_ACCESS_KEY": "minio_root_password"
+}
+```
+
+Although the bucket is set for public access in this development example, adding the authentication when generating
+the precomputed data mimics the typical behavior in actual deployments.
+
+When running the precomputed worker, pass the alias as the precomputed output argument, _e.g._, `-o minio://aind-neuron-morphology-community-portal-local/ngv01`
+
+Here the current standard path is used as the base location (`ngv01`), however anything can used so long as it is also
+used in the `NMCP_PRECOMPUTED` environment variable for `nmcp-client`.

nmcp_precomputed-3.0.4/Taskfile.dist.yaml

@@ -0,0 +1,33 @@
+version: '3'
+
+vars:
+  REPOSITORY: ghcr.io
+  IMAGE: allenneuraldynamics/nmcp-precomputed-worker
+  REPOIMAGE: "{{.REPOSITORY}}/{{.IMAGE}}"
+  VERSION:
+    sh: jq -r .version version.json
+  MAJOR:
+    sh: string={{.VERSION}} && arr=(${string//"."/ }) && echo ${arr[0]}
+  MINOR:
+    sh: string={{.VERSION}} && arr=(${string//"."/ }) && echo ${arr[1]}
+  REVISION:
+    sh: string={{.VERSION}} && arr=(${string//"."/ }) && echo ${arr[2]}
+  IMAGEWITHVERSION: "{{.REPOIMAGE}}:{{.MAJOR}}"
+  IMAGEWITHMINORVERSION: "{{.IMAGEWITHVERSION}}.{{.MINOR}}"
+  IMAGEWITHREVERSION: "{{.IMAGEWITHMINORVERSION}}.{{.REVISION}}"
+  IMAGELATEST: "{{.REPOIMAGE}}:latest"
+
+tasks:
+  build:
+    cmds:
+      - docker build --platform linux/amd64 --tag {{.IMAGEWITHVERSION}} .
+      - docker tag {{.IMAGEWITHVERSION}} {{.IMAGEWITHMINORVERSION}}
+      - docker tag {{.IMAGEWITHMINORVERSION}} {{.IMAGEWITHREVERSION}}
+      - docker tag {{.IMAGEWITHREVERSION}} {{.IMAGELATEST}}
+  release:
+    cmds:
+      - task: build
+      - docker push {{.IMAGEWITHVERSION}}
+      - docker push {{.IMAGEWITHMINORVERSION}}
+      - docker push {{.IMAGEWITHREVERSION}}
+      - docker push {{.IMAGELATEST}}

nmcp_precomputed-3.0.4/dev.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Start service(s) that are not typically under local development (e.g., databases), but are required to run and test services that may be under development.
+
+if [ -z "NMCP_COMPOSE_PROJECT" ]; then
+    export NMCP_COMPOSE_PROJECT="nmcp"
+fi
+
+docker compose -p ${NMCP_COMPOSE_PROJECT} up

nmcp_precomputed-3.0.4/docker-compose.yml

@@ -0,0 +1,23 @@
+# Services that will be run regardless of environment and will not be built locally as part of development.
+services:
+  s3:
+    image: minio/minio:latest
+    hostname: nmcp-s3
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: minio_root_user
+      MINIO_ROOT_PASSWORD: minio_root_password
+    volumes:
+      - s3:/data
+    command: [ "server", "--console-address", ":9001", "/data" ]
+    networks:
+      - back_tier
+    restart: unless-stopped
+
+volumes:
+  s3:
+
+networks:
+  back_tier:

nmcp_precomputed-3.0.4/docker-entry.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+logName=$(date '+%Y-%m-%d_%H-%M-%S');
+
+mkdir -p /var/log/nmcp
+
+export PYTHONPATH=$PWD
+
+python -m nmcp -u $GRAPHQL_URL -a $SERVER_AUTHENTICATION_KEY -o $PRECOMPUTED_OUTPUT  >> /var/log/nmcp/nmcp-precomputed-${logName}.log 2>&1

nmcp_precomputed-3.0.4/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/nmcp"]
+
+[tool.hatch.build.targets.sdist]
+exclude = [
+    "/tests"
+]
+
+[project]
+name = "nmcp-precomputed"
+version = "3.0.4"
+description = "Neuron Morphology Community Portal Precomputed Toolbox"
+license = { text = "MIT" }
+requires-python = ">=3.10,<3.11"
+dependencies = [
+    "numpy<1.24",
+    "pandas",
+    "gql[requests]",
+    "allensdk",
+    "cloud-volume",
+    "cloud-files"
+]
+
+[tool.pytest.ini_options]
+addopts = ["-W ignore::DeprecationWarning"]
+
+[project.optional-dependencies]
+test = [
+    "pytest==8.2.0"
+]
+publish = [
+    "build",
+    "twine"
+]

nmcp_precomputed-3.0.4/src/nmcp/__init__.py

@@ -0,0 +1,4 @@
+from .precomputed import SegmentInfo, SegmentProperty, SegmentTagProperty, SomaSegmentTagProperty, NmcpPropertyValues
+from .precomputed import (ensure_bucket_folders, create_from_json_files, create_from_dict, create_from_data,
+                          remove_skeleton, list_skeletons, extract_neuron_properties, SkeletonComponents)
+from .data import RemoteDataClient, PrecomputedEntry

nmcp_precomputed-3.0.4/src/nmcp/__main__.py

@@ -0,0 +1,13 @@
+import argparse
+
+from .precomputed_worker import main
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("-u", "--url", help="URL of the GraphQL service")
+parser.add_argument("-a", "--authkey", help="authorization header for GraphQL service")
+parser.add_argument("-o", "--output", help="the output cloud volume location")
+
+args = parser.parse_args()
+
+main(args.url, args.authkey, args.output)

nmcp_precomputed-3.0.4/src/nmcp/data/__init__.py

@@ -0,0 +1,2 @@
+from .remote_data_client import RemoteDataClient
+from .precomputed_entry import PrecomputedEntry

nmcp_precomputed-3.0.4/src/nmcp/data/precomputed_entry.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class PrecomputedEntry:
+    id: str
+    skeletonId: int
+    version: int | None
+    reconstructionId: str
+    generatedAt: float | None