bedhost 0.2.0__tar.gz → 0.16.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bedhost-0.16.0/.gitignore +42 -0
- bedhost-0.16.0/PKG-INFO +121 -0
- bedhost-0.16.0/README.md +80 -0
- bedhost-0.16.0/bedhost/__init__.py +5 -0
- bedhost-0.16.0/bedhost/_version.py +1 -0
- {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/cli.py +12 -10
- bedhost-0.16.0/bedhost/const.py +57 -0
- bedhost-0.16.0/bedhost/core.py +245 -0
- bedhost-0.16.0/bedhost/data_models.py +102 -0
- bedhost-0.16.0/bedhost/dependencies.py +70 -0
- {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/exceptions.py +3 -3
- bedhost-0.16.0/bedhost/helpers.py +228 -0
- bedhost-0.16.0/bedhost/main.py +377 -0
- bedhost-0.16.0/bedhost/mcp/__init__.py +37 -0
- bedhost-0.16.0/bedhost/mcp/errors.py +38 -0
- bedhost-0.16.0/bedhost/mcp/paging.py +89 -0
- bedhost-0.16.0/bedhost/mcp/schemas.py +282 -0
- bedhost-0.16.0/bedhost/mcp/stdio.py +24 -0
- bedhost-0.16.0/bedhost/mcp/tools.py +890 -0
- bedhost-0.16.0/bedhost/og_image.py +115 -0
- bedhost-0.16.0/bedhost/routers/base_api.py +302 -0
- bedhost-0.16.0/bedhost/routers/bed_api.py +854 -0
- bedhost-0.16.0/bedhost/routers/bedset_api.py +301 -0
- bedhost-0.16.0/bedhost/routers/objects_api.py +337 -0
- bedhost-0.16.0/bedhost/static/Roboto-Bold.ttf +0 -0
- bedhost-0.16.0/bedhost/static/Roboto-Light.ttf +0 -0
- bedhost-0.16.0/bedhost/static/UVA_CPHG.png +0 -0
- bedhost-0.16.0/bedhost/static/bedbase_bug.ico +0 -0
- bedhost-0.16.0/bedhost/static/bedbase_bug.svg +107 -0
- bedhost-0.16.0/bedhost/static/bedbase_logo.svg +122 -0
- {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/static/index.html +1 -1
- {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/templates/page.html +1 -3
- bedhost-0.16.0/bedhost/uploads.py +174 -0
- bedhost-0.16.0/bedhost/utils.py +85 -0
- bedhost-0.16.0/pyproject.toml +107 -0
- bedhost-0.16.0/tests/__init__.py +0 -0
- bedhost-0.16.0/tests/api/__init__.py +1 -0
- bedhost-0.16.0/tests/api/conftest.py +27 -0
- bedhost-0.16.0/tests/api/test_compliance.py +458 -0
- bedhost-0.16.0/tests/client.py +58 -0
- bedhost-0.16.0/tests/conftest.py +94 -0
- bedhost-0.16.0/tests/evals/baseline.json +1237 -0
- bedhost-0.16.0/tests/evals/baseline_noml.json +163 -0
- bedhost-0.16.0/tests/evals/bin/bigBedToBed +51 -0
- bedhost-0.16.0/tests/evals/fixture_agent.py +1295 -0
- bedhost-0.16.0/tests/evals/fixture_regions.py +61 -0
- bedhost-0.16.0/tests/evals/mcp_tasks.yaml +453 -0
- bedhost-0.16.0/tests/evals/mcp_tasks_noml.yaml +48 -0
- bedhost-0.16.0/tests/evals/serve_fixture.py +95 -0
- bedhost-0.16.0/tests/evals/test_task_files.py +52 -0
- bedhost-0.16.0/tests/fakes.py +236 -0
- bedhost-0.16.0/tests/fixtures/bedbase_test_config.template.yaml +34 -0
- bedhost-0.16.0/tests/integration/__init__.py +0 -0
- bedhost-0.16.0/tests/integration/conftest.py +231 -0
- bedhost-0.16.0/tests/integration/test_integration_smoke.py +12 -0
- bedhost-0.16.0/tests/mcp/__init__.py +0 -0
- bedhost-0.16.0/tests/mcp/conftest.py +40 -0
- bedhost-0.16.0/tests/mcp/test_layering.py +117 -0
- bedhost-0.16.0/tests/mcp/test_mcp.py +507 -0
- bedhost-0.16.0/tests/scripts/container-smoke.sh +43 -0
- bedhost-0.16.0/tests/scripts/services.sh +173 -0
- bedhost-0.16.0/tests/scripts/test-integration.sh +112 -0
- bedhost-0.16.0/tests/test_core.py +87 -0
- bedhost-0.16.0/tests/test_regions_helper.py +83 -0
- bedhost-0.16.0/tests/test_uploads.py +293 -0
- bedhost-0.2.0/Dockerfile +0 -30
- bedhost-0.2.0/MANIFEST.in +0 -5
- bedhost-0.2.0/PKG-INFO +0 -72
- bedhost-0.2.0/README.md +0 -44
- bedhost-0.2.0/bedhost/__init__.py +0 -8
- bedhost-0.2.0/bedhost/_version.py +0 -1
- bedhost-0.2.0/bedhost/const.py +0 -79
- bedhost-0.2.0/bedhost/data_models.py +0 -40
- bedhost-0.2.0/bedhost/helpers.py +0 -241
- bedhost-0.2.0/bedhost/main.py +0 -316
- bedhost-0.2.0/bedhost/routers/bed_api.py +0 -153
- bedhost-0.2.0/bedhost/routers/bedset_api.py +0 -137
- bedhost-0.2.0/bedhost/routers/search_api.py +0 -24
- bedhost-0.2.0/bedhost/static/bedbase_bug.ico +0 -0
- bedhost-0.2.0/bedhost/static/bedbase_bug.svg +0 -168
- bedhost-0.2.0/bedhost/static/bedbase_logo.svg +0 -181
- bedhost-0.2.0/bedhost.egg-info/PKG-INFO +0 -72
- bedhost-0.2.0/bedhost.egg-info/SOURCES.txt +0 -34
- bedhost-0.2.0/bedhost.egg-info/dependency_links.txt +0 -1
- bedhost-0.2.0/bedhost.egg-info/entry_points.txt +0 -2
- bedhost-0.2.0/bedhost.egg-info/requires.txt +0 -10
- bedhost-0.2.0/bedhost.egg-info/top_level.txt +0 -1
- bedhost-0.2.0/requirements/requirements-all.txt +0 -10
- bedhost-0.2.0/requirements/requirements-dev.txt +0 -1
- bedhost-0.2.0/setup.cfg +0 -4
- bedhost-0.2.0/setup.py +0 -66
- {bedhost-0.2.0 → bedhost-0.16.0}/LICENSE.txt +0 -0
- {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/__main__.py +0 -0
- {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/static/postgres_icon.png +0 -0
- {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/static/query-builder-elasticsearch.js +0 -0
- {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/static/query-builder.standalone.min.js +0 -0
- {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/static/style.css +0 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
output
|
|
2
|
+
bedstat_output
|
|
3
|
+
lolacore.csv
|
|
4
|
+
config.yaml
|
|
5
|
+
__pycache__
|
|
6
|
+
*/__pycache__
|
|
7
|
+
*/#*
|
|
8
|
+
.idea
|
|
9
|
+
*.DS_Store*
|
|
10
|
+
|
|
11
|
+
*.egg-info
|
|
12
|
+
build
|
|
13
|
+
|
|
14
|
+
# Ignore Jupyter notebook checkpoints
|
|
15
|
+
.ipynb_checkpoints
|
|
16
|
+
*/.ipynb_checkpoints/*
|
|
17
|
+
|
|
18
|
+
docs_jupyter/bedbase_tutorial
|
|
19
|
+
bedhost/static/bedhost-ui/
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# Models
|
|
23
|
+
*.h5
|
|
24
|
+
|
|
25
|
+
# For building
|
|
26
|
+
geniml
|
|
27
|
+
|
|
28
|
+
local_cache
|
|
29
|
+
|
|
30
|
+
!environment/
|
|
31
|
+
|
|
32
|
+
# Environments
|
|
33
|
+
.env
|
|
34
|
+
.venv
|
|
35
|
+
env/
|
|
36
|
+
venv/
|
|
37
|
+
ENV/
|
|
38
|
+
env.bak/
|
|
39
|
+
venv.bak/
|
|
40
|
+
.DS_store
|
|
41
|
+
|
|
42
|
+
ui/package-lock.json
|
bedhost-0.16.0/PKG-INFO
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: bedhost
|
|
3
|
+
Version: 0.16.0
|
|
4
|
+
Summary: A database server application that provides both a RESTful API and GUI for access to BED files and related basic statistics
|
|
5
|
+
Project-URL: Homepage, https://github.com/databio/bedhost
|
|
6
|
+
Author: Oleksandr Khoroshevskyi, Michal Stolarczyk, Ognen Duzlevski, Bingjie Xue, Nathan Sheffield
|
|
7
|
+
License-Expression: BSD-2-Clause
|
|
8
|
+
License-File: LICENSE.txt
|
|
9
|
+
Keywords: GUI,bed,bioinformatics,ngs,project,sequencing,server,workflow
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Requires-Dist: apscheduler>=3.11.0
|
|
18
|
+
Requires-Dist: bbconf>=0.17.0
|
|
19
|
+
Requires-Dist: bedboss>=0.12.0
|
|
20
|
+
Requires-Dist: cachetools>=4.2.4
|
|
21
|
+
Requires-Dist: fastapi>=0.103.0
|
|
22
|
+
Requires-Dist: geniml[ml]>=0.9.0
|
|
23
|
+
Requires-Dist: logmuse>=0.3.1
|
|
24
|
+
Requires-Dist: markdown
|
|
25
|
+
Requires-Dist: pephubclient>=0.4.1
|
|
26
|
+
Requires-Dist: pillow>=10.0.0
|
|
27
|
+
Requires-Dist: psycopg[binary,pool]
|
|
28
|
+
Requires-Dist: python-multipart>=0.0.9
|
|
29
|
+
Requires-Dist: requests
|
|
30
|
+
Requires-Dist: uvicorn
|
|
31
|
+
Requires-Dist: yacman>=1.0.0
|
|
32
|
+
Provides-Extra: local
|
|
33
|
+
Requires-Dist: bbconf[local]>=0.17.0; extra == 'local'
|
|
34
|
+
Provides-Extra: mcp
|
|
35
|
+
Requires-Dist: mcp[cli]>=2.2.0; extra == 'mcp'
|
|
36
|
+
Provides-Extra: test
|
|
37
|
+
Requires-Dist: httpx; extra == 'test'
|
|
38
|
+
Requires-Dist: mcp[cli]>=2.2.0; extra == 'test'
|
|
39
|
+
Requires-Dist: pytest; extra == 'test'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
<h1 align="center">bedhost</h1>
|
|
43
|
+
|
|
44
|
+
[](https://github.com/psf/black)
|
|
45
|
+
[](https://github.com/databio/bedhost)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
`bedhost` is a Python FastAPI module for the API that powers BEDbase.
|
|
49
|
+
It needs a path to the *bedbase configuration file*, which can be provided either via `-c`/`--config` argument or read from `$BEDBASE_CONFIG` environment variable.
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Links of interest
|
|
54
|
+
|
|
55
|
+
- **Deployed public instance**: <a href="https://bedbase.org/" target="_blank">https://bedbase.org/</a>
|
|
56
|
+
- **Documentation**: <a href="https://docs.bedbase.org/" target="_blank">https://docs.bedbase.org/bedhost</a>
|
|
57
|
+
- **API**: <a href="https://api.bedbase.org/" target="_blank">https://api.bedbase.org/</a>
|
|
58
|
+
- **DEV API**: <a href="https://dev.bedbase.org/" target="_blank">https://api-dev.bedbase.org/</a>
|
|
59
|
+
- **UI**: <a href="https://bedbase.org/" target="_blank">https://bedbase.org/</a>
|
|
60
|
+
- **DEV UI**: <a href="https://dev.bedbase.org/" target="_blank">https://dev.bedbase.org/</a>
|
|
61
|
+
- **Source Code**: <a href="https://github.com/databio/bedhost/" target="_blank">https://github.com/databio/bedhost/</a>
|
|
62
|
+
|
|
63
|
+
## Developer quick-start
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
source environment/production.env
|
|
67
|
+
BEDBASE_CONFIG=deployment/config/api.bedbase.org.yaml uvicorn bedhost.main:app --reload
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Embeddings
|
|
71
|
+
|
|
72
|
+
Search, `/v1/bed/embed`, and `/v1/bed/umap` need embedding models. By default
|
|
73
|
+
bbconf loads them all in-process, which needs `uv pip install -e ".[local]" --torch-backend cpu`.
|
|
74
|
+
The production configs instead set bbconf's `embedding_server:` block (see the
|
|
75
|
+
bbconf README), so the two text models (dense and sparse) come from embedding
|
|
76
|
+
servers and are not loaded by bedhost. The region model (`path.region2vec`) still
|
|
77
|
+
runs in-process, so the Docker image keeps CPU-only torch. The servers come from
|
|
78
|
+
these environment variables, which production reads from AWS SSM:
|
|
79
|
+
|
|
80
|
+
| Variable | Config key |
|
|
81
|
+
|---|---|
|
|
82
|
+
| `EMBEDDING_DENSE_URL` | `embedding_server.text_url` |
|
|
83
|
+
| `EMBEDDING_SPARSE_URL` | `embedding_server.sparse_url` |
|
|
84
|
+
| `EMBEDDING_API_KEY` | `embedding_server.api_key` |
|
|
85
|
+
|
|
86
|
+
To move the region model to a server too, add `embedding_server.region_url`.
|
|
87
|
+
For local runs, `source environment/production.env` sets all three from the lab's `pass` store.
|
|
88
|
+
|
|
89
|
+
If a server is down, bedhost still starts and the routes that need it return 503.
|
|
90
|
+
|
|
91
|
+
UI development:
|
|
92
|
+
```terminal
|
|
93
|
+
export VITE_API_HOST=http://localhost:8000
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Testing
|
|
97
|
+
|
|
98
|
+
Black-box compliance tests (against any deployment):
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
pytest tests/api/ --api_root=https://api.bedbase.org
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Integration tests (ephemeral Postgres + Qdrant + in-process TestClient, ~30s):
|
|
105
|
+
|
|
106
|
+
```
|
|
107
|
+
./tests/scripts/test-integration.sh
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Manual service control:
|
|
111
|
+
|
|
112
|
+
```
|
|
113
|
+
./tests/scripts/services.sh start
|
|
114
|
+
RUN_INTEGRATION_TESTS=true pytest tests/integration/ tests/api/
|
|
115
|
+
./tests/scripts/services.sh stop
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Integration tests require network access on first boot (bbconf downloads a
|
|
119
|
+
licenses CSV from GitHub and may pull HuggingFace models unless
|
|
120
|
+
`BEDHOST_INIT_ML=false` is set; `test-integration.sh` sets it).
|
|
121
|
+
`BEDHOST_LOG_LEVEL` (default `INFO`) sets the log level for bedhost, bbconf, and geniml.
|
bedhost-0.16.0/README.md
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
<h1 align="center">bedhost</h1>
|
|
2
|
+
|
|
3
|
+
[](https://github.com/psf/black)
|
|
4
|
+
[](https://github.com/databio/bedhost)
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
`bedhost` is a Python FastAPI module for the API that powers BEDbase.
|
|
8
|
+
It needs a path to the *bedbase configuration file*, which can be provided either via `-c`/`--config` argument or read from `$BEDBASE_CONFIG` environment variable.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## Links of interest
|
|
13
|
+
|
|
14
|
+
- **Deployed public instance**: <a href="https://bedbase.org/" target="_blank">https://bedbase.org/</a>
|
|
15
|
+
- **Documentation**: <a href="https://docs.bedbase.org/" target="_blank">https://docs.bedbase.org/bedhost</a>
|
|
16
|
+
- **API**: <a href="https://api.bedbase.org/" target="_blank">https://api.bedbase.org/</a>
|
|
17
|
+
- **DEV API**: <a href="https://dev.bedbase.org/" target="_blank">https://api-dev.bedbase.org/</a>
|
|
18
|
+
- **UI**: <a href="https://bedbase.org/" target="_blank">https://bedbase.org/</a>
|
|
19
|
+
- **DEV UI**: <a href="https://dev.bedbase.org/" target="_blank">https://dev.bedbase.org/</a>
|
|
20
|
+
- **Source Code**: <a href="https://github.com/databio/bedhost/" target="_blank">https://github.com/databio/bedhost/</a>
|
|
21
|
+
|
|
22
|
+
## Developer quick-start
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
source environment/production.env
|
|
26
|
+
BEDBASE_CONFIG=deployment/config/api.bedbase.org.yaml uvicorn bedhost.main:app --reload
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Embeddings
|
|
30
|
+
|
|
31
|
+
Search, `/v1/bed/embed`, and `/v1/bed/umap` need embedding models. By default
|
|
32
|
+
bbconf loads them all in-process, which needs `uv pip install -e ".[local]" --torch-backend cpu`.
|
|
33
|
+
The production configs instead set bbconf's `embedding_server:` block (see the
|
|
34
|
+
bbconf README), so the two text models (dense and sparse) come from embedding
|
|
35
|
+
servers and are not loaded by bedhost. The region model (`path.region2vec`) still
|
|
36
|
+
runs in-process, so the Docker image keeps CPU-only torch. The servers come from
|
|
37
|
+
these environment variables, which production reads from AWS SSM:
|
|
38
|
+
|
|
39
|
+
| Variable | Config key |
|
|
40
|
+
|---|---|
|
|
41
|
+
| `EMBEDDING_DENSE_URL` | `embedding_server.text_url` |
|
|
42
|
+
| `EMBEDDING_SPARSE_URL` | `embedding_server.sparse_url` |
|
|
43
|
+
| `EMBEDDING_API_KEY` | `embedding_server.api_key` |
|
|
44
|
+
|
|
45
|
+
To move the region model to a server too, add `embedding_server.region_url`.
|
|
46
|
+
For local runs, `source environment/production.env` sets all three from the lab's `pass` store.
|
|
47
|
+
|
|
48
|
+
If a server is down, bedhost still starts and the routes that need it return 503.
|
|
49
|
+
|
|
50
|
+
UI development:
|
|
51
|
+
```terminal
|
|
52
|
+
export VITE_API_HOST=http://localhost:8000
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Testing
|
|
56
|
+
|
|
57
|
+
Black-box compliance tests (against any deployment):
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
pytest tests/api/ --api_root=https://api.bedbase.org
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Integration tests (ephemeral Postgres + Qdrant + in-process TestClient, ~30s):
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
./tests/scripts/test-integration.sh
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Manual service control:
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
./tests/scripts/services.sh start
|
|
73
|
+
RUN_INTEGRATION_TESTS=true pytest tests/integration/ tests/api/
|
|
74
|
+
./tests/scripts/services.sh stop
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Integration tests require network access on first boot (bbconf downloads a
|
|
78
|
+
licenses CSV from GitHub and may pull HuggingFace models unless
|
|
79
|
+
`BEDHOST_INIT_ML=false` is set; `test-integration.sh` sets it).
|
|
80
|
+
`BEDHOST_LOG_LEVEL` (default `INFO`) sets the log level for bedhost, bbconf, and geniml.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.16.0"
|
|
@@ -1,18 +1,21 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
|
|
3
|
+
from ubiquerg import VersionInHelpParser
|
|
4
|
+
|
|
1
5
|
from . import PKG_NAME
|
|
2
6
|
from ._version import __version__
|
|
3
|
-
from
|
|
4
|
-
from yacman import select_config
|
|
5
|
-
from bbconf.const import CFG_ENV_VARS
|
|
7
|
+
from .const import CFG_ENV_VARS
|
|
6
8
|
|
|
7
9
|
|
|
8
|
-
def build_parser():
|
|
10
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
9
11
|
"""
|
|
10
|
-
|
|
12
|
+
Build the argument parser.
|
|
11
13
|
|
|
12
|
-
:
|
|
14
|
+
Returns:
|
|
15
|
+
The configured argument parser.
|
|
13
16
|
"""
|
|
14
17
|
|
|
15
|
-
env_var_val = select_config(config_env_vars=CFG_ENV_VARS)
|
|
18
|
+
# env_var_val = select_config(config_env_vars=CFG_ENV_VARS)
|
|
16
19
|
|
|
17
20
|
banner = "%(prog)s - REST API for the bedstat pipeline produced statistics"
|
|
18
21
|
additional_description = (
|
|
@@ -32,7 +35,7 @@ def build_parser():
|
|
|
32
35
|
|
|
33
36
|
subparsers = parser.add_subparsers(dest="command")
|
|
34
37
|
|
|
35
|
-
def add_subparser(cmd, description):
|
|
38
|
+
def add_subparser(cmd: str, description: str) -> argparse.ArgumentParser:
|
|
36
39
|
return subparsers.add_parser(cmd, description=description, help=description)
|
|
37
40
|
|
|
38
41
|
sps = {}
|
|
@@ -45,7 +48,6 @@ def build_parser():
|
|
|
45
48
|
required=False,
|
|
46
49
|
dest="config",
|
|
47
50
|
help="A path to the bedhost config file (YAML). If not provided, "
|
|
48
|
-
f"the first available environment variable among: {', '.join(CFG_ENV_VARS)} will be used if set."
|
|
49
|
-
f" Currently: {env_var_val}",
|
|
51
|
+
f"the first available environment variable among: {', '.join(CFG_ENV_VARS)} will be used if set.",
|
|
50
52
|
)
|
|
51
53
|
return parser
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
PKG_NAME: str = "bedhost"
|
|
4
|
+
|
|
5
|
+
# Environment variables checked (in order) for the path to the bedhost config file.
|
|
6
|
+
CFG_ENV_VARS: list[str] = ["BEDBASE_CONFIG"]
|
|
7
|
+
|
|
8
|
+
TEMPLATES_DIRNAME: str = "templates"
|
|
9
|
+
STATIC_DIRNAME: str = "../docs"
|
|
10
|
+
STATIC_PATH: str = os.path.join(
|
|
11
|
+
os.path.dirname(os.path.abspath(__file__)), STATIC_DIRNAME
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
EXAMPLE_BED: str = "bbad85f21962bb8d972444f7f9a3a932"
|
|
15
|
+
EXAMPLE_BEDSET: str = "gse218680"
|
|
16
|
+
|
|
17
|
+
# how often to save usage data (in hours)
|
|
18
|
+
USAGE_SAVE_HOURS: int = 1
|
|
19
|
+
# For how many days record usage data (every month this will be reset)
|
|
20
|
+
USAGE_RECORD_DAYS: int = 30
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
MAX_FILE_SIZE: int = 1024 * 1024 * 20
|
|
24
|
+
# Decompressed size cap for gzip uploads, so a small gzip cannot expand without bound.
|
|
25
|
+
MAX_UNCOMPRESSED_SIZE: int = 1024 * 1024 * 200
|
|
26
|
+
MAX_REGION_NUMBER: int = 5000000
|
|
27
|
+
MAX_BATCH_IDS: int = (
|
|
28
|
+
1000 # /bed/batch and /bed/collection/stats (bbconf's get_batch limit)
|
|
29
|
+
)
|
|
30
|
+
MIN_REGION_WIDTH: int = 10
|
|
31
|
+
# Chunk size used when copying an uploaded file to disk.
|
|
32
|
+
UPLOAD_CHUNK_SIZE: int = 1024 * 1024
|
|
33
|
+
# Slack for multipart boundaries and headers in the Content-Length pre-check.
|
|
34
|
+
MULTIPART_OVERHEAD: int = 64 * 1024
|
|
35
|
+
# Routes that accept BED file uploads (guarded by upload_size_guard).
|
|
36
|
+
UPLOAD_ROUTES: frozenset[str] = frozenset(
|
|
37
|
+
{"/v1/bed/embed", "/v1/bed/umap", "/v1/bed/search/bed"}
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
# Upper bounds for request inputs (security review finding O1).
|
|
41
|
+
MAX_LIST_LIMIT: int = 10000 # paged list endpoints
|
|
42
|
+
MAX_SEARCH_LIMIT: int = 100 # search + neighbour endpoints (UI max is 100)
|
|
43
|
+
# hg38 with alts/decoys/unplaced is under 3,500 contigs (hs38DH ~3,366);
|
|
44
|
+
# mm10/mm39 are under 100. The UI only sends chromosomes that actually
|
|
45
|
+
# appear in the uploaded BED file, so 10,000 is a generous ceiling that
|
|
46
|
+
# still bounds the work.
|
|
47
|
+
MAX_CHROM_ENTRIES: int = 10000 # /analyze-genome dict size
|
|
48
|
+
MAX_CHROM_NAME_LENGTH: int = 256 # /analyze-genome dict key length
|
|
49
|
+
BIGBED_TIMEOUT_SECONDS: int = 60 # /regions bigBedToBed wall-clock cap
|
|
50
|
+
MAX_REGIONS_OUTPUT_BYTES: int = 1024 * 1024 * 20 # /regions response size cap
|
|
51
|
+
|
|
52
|
+
# Public CDN base for bulk metadata export artifacts. These live directly on the
|
|
53
|
+
# storage CDN (Backblaze B2 fronted by Cloudflare), NOT behind the API's
|
|
54
|
+
# /v1/files/ redirect proxy — so export URLs must use this base rather than the
|
|
55
|
+
# config's http access-method prefix (which points at api.bedbase.org/v1/files/
|
|
56
|
+
# and, notably, 405s on HEAD, breaking DuckDB range probing).
|
|
57
|
+
EXPORTS_URL_BASE: str = "https://data2.bedbase.org/"
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Logic shared by the REST routers and the MCP server.
|
|
3
|
+
|
|
4
|
+
Must not import fastapi, starlette, or any other bedhost module except const.
|
|
5
|
+
The MCP package imports this module, and the MCP package has to work without
|
|
6
|
+
the web stack installed (tests/mcp/test_layering.py checks both rules).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import subprocess
|
|
11
|
+
import tempfile
|
|
12
|
+
import threading
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from enum import Enum
|
|
15
|
+
|
|
16
|
+
from bbconf.exceptions import BEDFileNotFoundError
|
|
17
|
+
from bbconf.models.bed_models import BedListSearchResult, QdrantSearchResult
|
|
18
|
+
from pydantic import BaseModel
|
|
19
|
+
|
|
20
|
+
from .const import EXPORTS_URL_BASE, PKG_NAME
|
|
21
|
+
|
|
22
|
+
_LOGGER = logging.getLogger(PKG_NAME)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class BaseListResponse(BaseModel):
|
|
26
|
+
count: int
|
|
27
|
+
limit: int
|
|
28
|
+
offset: int
|
|
29
|
+
results: list
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# --------------------------------------------------------------------------- #
|
|
33
|
+
# Region queries (bigBedToBed) #
|
|
34
|
+
# --------------------------------------------------------------------------- #
|
|
35
|
+
class RegionOutputTooLarge(Exception):
|
|
36
|
+
"""bigBedToBed produced more output than the allowed ceiling."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def run_bigbed_to_bed(cmd: list[str], timeout: int, max_bytes: int) -> str:
|
|
40
|
+
"""Run bigBedToBed, wait for it, return only the first 3 BED columns.
|
|
41
|
+
|
|
42
|
+
Output is read line by line and trimmed as it arrives, so memory is bounded
|
|
43
|
+
by ``max_bytes`` rather than by the size of the bigBed file.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
cmd: Full argv for the ``bigBedToBed`` invocation (no shell).
|
|
47
|
+
timeout: Wall-clock timeout in seconds.
|
|
48
|
+
max_bytes: Ceiling on the trimmed output size.
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
The first three tab-separated columns of each output line,
|
|
52
|
+
newline-terminated. Empty string if there is no output.
|
|
53
|
+
|
|
54
|
+
Raises:
|
|
55
|
+
FileNotFoundError: If the binary is missing.
|
|
56
|
+
RegionOutputTooLarge: If the output exceeds ``max_bytes``. The child is
|
|
57
|
+
killed before this is raised.
|
|
58
|
+
subprocess.TimeoutExpired: If the process exceeds ``timeout``. The
|
|
59
|
+
child is killed before this is raised.
|
|
60
|
+
subprocess.CalledProcessError: If the process exits non-zero.
|
|
61
|
+
"""
|
|
62
|
+
timed_out = threading.Event()
|
|
63
|
+
rows: list[str] = []
|
|
64
|
+
total = 0
|
|
65
|
+
# stderr goes to a file so a chatty child can never block on a full pipe.
|
|
66
|
+
with (
|
|
67
|
+
tempfile.TemporaryFile() as err,
|
|
68
|
+
subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=err, text=True) as proc,
|
|
69
|
+
):
|
|
70
|
+
|
|
71
|
+
def _kill_on_timeout():
|
|
72
|
+
timed_out.set()
|
|
73
|
+
proc.kill()
|
|
74
|
+
|
|
75
|
+
timer = threading.Timer(timeout, _kill_on_timeout)
|
|
76
|
+
timer.start()
|
|
77
|
+
try:
|
|
78
|
+
for line in proc.stdout:
|
|
79
|
+
row = "\t".join(line.rstrip("\n").split("\t", 3)[:3]) + "\n"
|
|
80
|
+
total += len(row)
|
|
81
|
+
if total > max_bytes:
|
|
82
|
+
proc.kill()
|
|
83
|
+
raise RegionOutputTooLarge(
|
|
84
|
+
f"bigBedToBed output exceeded {max_bytes} bytes"
|
|
85
|
+
)
|
|
86
|
+
rows.append(row)
|
|
87
|
+
proc.wait()
|
|
88
|
+
finally:
|
|
89
|
+
timer.cancel()
|
|
90
|
+
|
|
91
|
+
if timed_out.is_set():
|
|
92
|
+
raise subprocess.TimeoutExpired(cmd, timeout)
|
|
93
|
+
if proc.returncode != 0:
|
|
94
|
+
err.seek(0)
|
|
95
|
+
raise subprocess.CalledProcessError(
|
|
96
|
+
proc.returncode, cmd, stderr=err.read().decode(errors="replace")
|
|
97
|
+
)
|
|
98
|
+
return "".join(rows)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def build_exports_url(file_path: str) -> str:
|
|
102
|
+
"""
|
|
103
|
+
Build a public HTTPS CDN URL for an export / analysis-file S3 key.
|
|
104
|
+
|
|
105
|
+
Uses explicit string concatenation rather than ``os.path.join``: the latter
|
|
106
|
+
is a filesystem operation and, given an absolute ``file_path`` (leading
|
|
107
|
+
``/``), silently discards the base URL entirely — dropping the scheme and
|
|
108
|
+
host. Stripping a trailing slash off the base and a leading slash off the
|
|
109
|
+
key guarantees exactly one separator and always preserves the host.
|
|
110
|
+
|
|
111
|
+
Args:
|
|
112
|
+
file_path: The S3 key (relative or absolute) of the artifact.
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
The absolute ``https://data2.bedbase.org/...`` URL for the artifact.
|
|
116
|
+
"""
|
|
117
|
+
return f"{EXPORTS_URL_BASE.rstrip('/')}/{file_path.lstrip('/')}"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
# --------------------------------------------------------------------------- #
|
|
121
|
+
# BED text search #
|
|
122
|
+
# --------------------------------------------------------------------------- #
|
|
123
|
+
class SearchKind(str, Enum):
|
|
124
|
+
"""Which path search_bed took to answer a query."""
|
|
125
|
+
|
|
126
|
+
EXACT_ID = "exact_id"
|
|
127
|
+
ACCESSION = "accession"
|
|
128
|
+
SEMANTIC = "semantic"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class MLUnavailable(Exception):
|
|
132
|
+
"""Semantic search was needed but the dense encoder is not loaded."""
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@dataclass
|
|
136
|
+
class BedSearchOutcome:
|
|
137
|
+
kind: SearchKind
|
|
138
|
+
result: BedListSearchResult
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
_ACCESSION_PREFIXES = ("gsm", "gse", "encff", "encsr", "geo:", "encode:")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _parse_accession(query: str) -> tuple[str, str] | None:
|
|
145
|
+
"""Return ``(source, accession)`` for a GEO/ENCODE accession, else None."""
|
|
146
|
+
low = query.replace(" ", "").lower()
|
|
147
|
+
if not low.startswith(_ACCESSION_PREFIXES):
|
|
148
|
+
return None
|
|
149
|
+
acc = low.replace("geo:", "").replace("encode:", "")
|
|
150
|
+
if acc.startswith(("gsm", "gse")):
|
|
151
|
+
return "geo", acc
|
|
152
|
+
if acc.startswith(("encff", "encsr")):
|
|
153
|
+
return "encode", acc.upper()
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _exact_id_result(agent, bed_id: str, limit: int, offset: int):
|
|
158
|
+
"""The file itself first, then its nearest neighbours; None if no such file.
|
|
159
|
+
|
|
160
|
+
The file takes slot 0, so a page at ``offset`` holds neighbours starting at
|
|
161
|
+
``offset - 1``. If the neighbour lookup fails (for example the vector index
|
|
162
|
+
is not loaded), the result is just the file.
|
|
163
|
+
"""
|
|
164
|
+
try:
|
|
165
|
+
meta = agent.bed.get(bed_id)
|
|
166
|
+
except BEDFileNotFoundError:
|
|
167
|
+
return None
|
|
168
|
+
hit = QdrantSearchResult(id=bed_id, payload={}, score=1.0, metadata=meta)
|
|
169
|
+
|
|
170
|
+
n_limit = limit - 1 if offset == 0 else limit
|
|
171
|
+
n_offset = 0 if offset == 0 else offset - 1
|
|
172
|
+
try:
|
|
173
|
+
neighbours = (
|
|
174
|
+
agent.bed.get_neighbours(bed_id, limit=n_limit, offset=n_offset)
|
|
175
|
+
if n_limit > 0
|
|
176
|
+
else None
|
|
177
|
+
)
|
|
178
|
+
except Exception as e: # the file exists; neighbours are a bonus
|
|
179
|
+
_LOGGER.warning(f"Neighbour lookup failed for {bed_id}: {e}")
|
|
180
|
+
neighbours = None
|
|
181
|
+
|
|
182
|
+
if neighbours is None or not neighbours.results:
|
|
183
|
+
rows = [hit] if offset == 0 else []
|
|
184
|
+
return BedListSearchResult(count=1, limit=limit, offset=offset, results=rows)
|
|
185
|
+
rows = neighbours.results
|
|
186
|
+
if offset == 0:
|
|
187
|
+
rows = [hit, *rows]
|
|
188
|
+
return BedListSearchResult(
|
|
189
|
+
count=neighbours.count, limit=limit, offset=offset, results=rows
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def search_bed(
|
|
194
|
+
agent,
|
|
195
|
+
query: str,
|
|
196
|
+
genome: str | None = None,
|
|
197
|
+
assay: str | None = None,
|
|
198
|
+
limit: int = 10,
|
|
199
|
+
offset: int = 0,
|
|
200
|
+
) -> BedSearchOutcome:
|
|
201
|
+
"""Exact BED id -> GEO/ENCODE accession -> hybrid semantic search.
|
|
202
|
+
|
|
203
|
+
The first two paths need no ML models. Only the semantic path needs the
|
|
204
|
+
dense encoder; without it this raises MLUnavailable.
|
|
205
|
+
|
|
206
|
+
Args:
|
|
207
|
+
agent: The bbconf BedBaseAgent.
|
|
208
|
+
query: Free text, a GEO/ENCODE accession, or a 32-character BED id.
|
|
209
|
+
genome: Genome filter for semantic search (e.g. "hg38").
|
|
210
|
+
assay: Assay filter for semantic search (e.g. "ATAC-seq").
|
|
211
|
+
limit: Rows to return.
|
|
212
|
+
offset: Rows to skip.
|
|
213
|
+
|
|
214
|
+
Raises:
|
|
215
|
+
MLUnavailable: If the query needs semantic search and ML is off.
|
|
216
|
+
"""
|
|
217
|
+
spaceless = query.replace(" ", "")
|
|
218
|
+
if len(spaceless) == 32 and spaceless == query:
|
|
219
|
+
result = _exact_id_result(agent, query, limit, offset)
|
|
220
|
+
if result is not None:
|
|
221
|
+
return BedSearchOutcome(SearchKind.EXACT_ID, result)
|
|
222
|
+
|
|
223
|
+
if parsed := _parse_accession(query):
|
|
224
|
+
source, accession = parsed
|
|
225
|
+
_LOGGER.info(f"Searching for {source} accession {accession}")
|
|
226
|
+
found = agent.bed.search_external_file(source=source, accession=accession)
|
|
227
|
+
if found.count:
|
|
228
|
+
rows = (found.results or [])[offset : offset + limit]
|
|
229
|
+
return BedSearchOutcome(
|
|
230
|
+
SearchKind.ACCESSION,
|
|
231
|
+
BedListSearchResult(
|
|
232
|
+
count=found.count, limit=limit, offset=offset, results=rows
|
|
233
|
+
),
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
if getattr(agent.config, "dense_encoder", None) is None:
|
|
237
|
+
raise MLUnavailable("Text search unavailable (ML models disabled)")
|
|
238
|
+
result = agent.bed.hybrid_search(
|
|
239
|
+
query,
|
|
240
|
+
genome_alias=genome or "",
|
|
241
|
+
assay=assay or "",
|
|
242
|
+
limit=limit,
|
|
243
|
+
offset=offset,
|
|
244
|
+
)
|
|
245
|
+
return BedSearchOutcome(SearchKind.SEMANTIC, result)
|