bedhost 0.2.0__tar.gz → 0.16.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. bedhost-0.16.0/.gitignore +42 -0
  2. bedhost-0.16.0/PKG-INFO +121 -0
  3. bedhost-0.16.0/README.md +80 -0
  4. bedhost-0.16.0/bedhost/__init__.py +5 -0
  5. bedhost-0.16.0/bedhost/_version.py +1 -0
  6. {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/cli.py +12 -10
  7. bedhost-0.16.0/bedhost/const.py +57 -0
  8. bedhost-0.16.0/bedhost/core.py +245 -0
  9. bedhost-0.16.0/bedhost/data_models.py +102 -0
  10. bedhost-0.16.0/bedhost/dependencies.py +70 -0
  11. {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/exceptions.py +3 -3
  12. bedhost-0.16.0/bedhost/helpers.py +228 -0
  13. bedhost-0.16.0/bedhost/main.py +377 -0
  14. bedhost-0.16.0/bedhost/mcp/__init__.py +37 -0
  15. bedhost-0.16.0/bedhost/mcp/errors.py +38 -0
  16. bedhost-0.16.0/bedhost/mcp/paging.py +89 -0
  17. bedhost-0.16.0/bedhost/mcp/schemas.py +282 -0
  18. bedhost-0.16.0/bedhost/mcp/stdio.py +24 -0
  19. bedhost-0.16.0/bedhost/mcp/tools.py +890 -0
  20. bedhost-0.16.0/bedhost/og_image.py +115 -0
  21. bedhost-0.16.0/bedhost/routers/base_api.py +302 -0
  22. bedhost-0.16.0/bedhost/routers/bed_api.py +854 -0
  23. bedhost-0.16.0/bedhost/routers/bedset_api.py +301 -0
  24. bedhost-0.16.0/bedhost/routers/objects_api.py +337 -0
  25. bedhost-0.16.0/bedhost/static/Roboto-Bold.ttf +0 -0
  26. bedhost-0.16.0/bedhost/static/Roboto-Light.ttf +0 -0
  27. bedhost-0.16.0/bedhost/static/UVA_CPHG.png +0 -0
  28. bedhost-0.16.0/bedhost/static/bedbase_bug.ico +0 -0
  29. bedhost-0.16.0/bedhost/static/bedbase_bug.svg +107 -0
  30. bedhost-0.16.0/bedhost/static/bedbase_logo.svg +122 -0
  31. {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/static/index.html +1 -1
  32. {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/templates/page.html +1 -3
  33. bedhost-0.16.0/bedhost/uploads.py +174 -0
  34. bedhost-0.16.0/bedhost/utils.py +85 -0
  35. bedhost-0.16.0/pyproject.toml +107 -0
  36. bedhost-0.16.0/tests/__init__.py +0 -0
  37. bedhost-0.16.0/tests/api/__init__.py +1 -0
  38. bedhost-0.16.0/tests/api/conftest.py +27 -0
  39. bedhost-0.16.0/tests/api/test_compliance.py +458 -0
  40. bedhost-0.16.0/tests/client.py +58 -0
  41. bedhost-0.16.0/tests/conftest.py +94 -0
  42. bedhost-0.16.0/tests/evals/baseline.json +1237 -0
  43. bedhost-0.16.0/tests/evals/baseline_noml.json +163 -0
  44. bedhost-0.16.0/tests/evals/bin/bigBedToBed +51 -0
  45. bedhost-0.16.0/tests/evals/fixture_agent.py +1295 -0
  46. bedhost-0.16.0/tests/evals/fixture_regions.py +61 -0
  47. bedhost-0.16.0/tests/evals/mcp_tasks.yaml +453 -0
  48. bedhost-0.16.0/tests/evals/mcp_tasks_noml.yaml +48 -0
  49. bedhost-0.16.0/tests/evals/serve_fixture.py +95 -0
  50. bedhost-0.16.0/tests/evals/test_task_files.py +52 -0
  51. bedhost-0.16.0/tests/fakes.py +236 -0
  52. bedhost-0.16.0/tests/fixtures/bedbase_test_config.template.yaml +34 -0
  53. bedhost-0.16.0/tests/integration/__init__.py +0 -0
  54. bedhost-0.16.0/tests/integration/conftest.py +231 -0
  55. bedhost-0.16.0/tests/integration/test_integration_smoke.py +12 -0
  56. bedhost-0.16.0/tests/mcp/__init__.py +0 -0
  57. bedhost-0.16.0/tests/mcp/conftest.py +40 -0
  58. bedhost-0.16.0/tests/mcp/test_layering.py +117 -0
  59. bedhost-0.16.0/tests/mcp/test_mcp.py +507 -0
  60. bedhost-0.16.0/tests/scripts/container-smoke.sh +43 -0
  61. bedhost-0.16.0/tests/scripts/services.sh +173 -0
  62. bedhost-0.16.0/tests/scripts/test-integration.sh +112 -0
  63. bedhost-0.16.0/tests/test_core.py +87 -0
  64. bedhost-0.16.0/tests/test_regions_helper.py +83 -0
  65. bedhost-0.16.0/tests/test_uploads.py +293 -0
  66. bedhost-0.2.0/Dockerfile +0 -30
  67. bedhost-0.2.0/MANIFEST.in +0 -5
  68. bedhost-0.2.0/PKG-INFO +0 -72
  69. bedhost-0.2.0/README.md +0 -44
  70. bedhost-0.2.0/bedhost/__init__.py +0 -8
  71. bedhost-0.2.0/bedhost/_version.py +0 -1
  72. bedhost-0.2.0/bedhost/const.py +0 -79
  73. bedhost-0.2.0/bedhost/data_models.py +0 -40
  74. bedhost-0.2.0/bedhost/helpers.py +0 -241
  75. bedhost-0.2.0/bedhost/main.py +0 -316
  76. bedhost-0.2.0/bedhost/routers/bed_api.py +0 -153
  77. bedhost-0.2.0/bedhost/routers/bedset_api.py +0 -137
  78. bedhost-0.2.0/bedhost/routers/search_api.py +0 -24
  79. bedhost-0.2.0/bedhost/static/bedbase_bug.ico +0 -0
  80. bedhost-0.2.0/bedhost/static/bedbase_bug.svg +0 -168
  81. bedhost-0.2.0/bedhost/static/bedbase_logo.svg +0 -181
  82. bedhost-0.2.0/bedhost.egg-info/PKG-INFO +0 -72
  83. bedhost-0.2.0/bedhost.egg-info/SOURCES.txt +0 -34
  84. bedhost-0.2.0/bedhost.egg-info/dependency_links.txt +0 -1
  85. bedhost-0.2.0/bedhost.egg-info/entry_points.txt +0 -2
  86. bedhost-0.2.0/bedhost.egg-info/requires.txt +0 -10
  87. bedhost-0.2.0/bedhost.egg-info/top_level.txt +0 -1
  88. bedhost-0.2.0/requirements/requirements-all.txt +0 -10
  89. bedhost-0.2.0/requirements/requirements-dev.txt +0 -1
  90. bedhost-0.2.0/setup.cfg +0 -4
  91. bedhost-0.2.0/setup.py +0 -66
  92. {bedhost-0.2.0 → bedhost-0.16.0}/LICENSE.txt +0 -0
  93. {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/__main__.py +0 -0
  94. {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/static/postgres_icon.png +0 -0
  95. {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/static/query-builder-elasticsearch.js +0 -0
  96. {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/static/query-builder.standalone.min.js +0 -0
  97. {bedhost-0.2.0 → bedhost-0.16.0}/bedhost/static/style.css +0 -0
@@ -0,0 +1,42 @@
1
+ output
2
+ bedstat_output
3
+ lolacore.csv
4
+ config.yaml
5
+ __pycache__
6
+ */__pycache__
7
+ */#*
8
+ .idea
9
+ *.DS_Store*
10
+
11
+ *.egg-info
12
+ build
13
+
14
+ # Ignore Jupyter notebook checkpoints
15
+ .ipynb_checkpoints
16
+ */.ipynb_checkpoints/*
17
+
18
+ docs_jupyter/bedbase_tutorial
19
+ bedhost/static/bedhost-ui/
20
+
21
+
22
+ # Models
23
+ *.h5
24
+
25
+ # For building
26
+ geniml
27
+
28
+ local_cache
29
+
30
+ !environment/
31
+
32
+ # Environments
33
+ .env
34
+ .venv
35
+ env/
36
+ venv/
37
+ ENV/
38
+ env.bak/
39
+ venv.bak/
40
+ .DS_store
41
+
42
+ ui/package-lock.json
@@ -0,0 +1,121 @@
1
+ Metadata-Version: 2.5
2
+ Name: bedhost
3
+ Version: 0.16.0
4
+ Summary: A database server application that provides both a RESTful API and GUI for access to BED files and related basic statistics
5
+ Project-URL: Homepage, https://github.com/databio/bedhost
6
+ Author: Oleksandr Khoroshevskyi, Michal Stolarczyk, Ognen Duzlevski, Bingjie Xue, Nathan Sheffield
7
+ License-Expression: BSD-2-Clause
8
+ License-File: LICENSE.txt
9
+ Keywords: GUI,bed,bioinformatics,ngs,project,sequencing,server,workflow
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Programming Language :: Python :: 3.10
12
+ Classifier: Programming Language :: Python :: 3.11
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Programming Language :: Python :: 3.13
15
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
16
+ Requires-Python: >=3.10
17
+ Requires-Dist: apscheduler>=3.11.0
18
+ Requires-Dist: bbconf>=0.17.0
19
+ Requires-Dist: bedboss>=0.12.0
20
+ Requires-Dist: cachetools>=4.2.4
21
+ Requires-Dist: fastapi>=0.103.0
22
+ Requires-Dist: geniml[ml]>=0.9.0
23
+ Requires-Dist: logmuse>=0.3.1
24
+ Requires-Dist: markdown
25
+ Requires-Dist: pephubclient>=0.4.1
26
+ Requires-Dist: pillow>=10.0.0
27
+ Requires-Dist: psycopg[binary,pool]
28
+ Requires-Dist: python-multipart>=0.0.9
29
+ Requires-Dist: requests
30
+ Requires-Dist: uvicorn
31
+ Requires-Dist: yacman>=1.0.0
32
+ Provides-Extra: local
33
+ Requires-Dist: bbconf[local]>=0.17.0; extra == 'local'
34
+ Provides-Extra: mcp
35
+ Requires-Dist: mcp[cli]>=2.2.0; extra == 'mcp'
36
+ Provides-Extra: test
37
+ Requires-Dist: httpx; extra == 'test'
38
+ Requires-Dist: mcp[cli]>=2.2.0; extra == 'test'
39
+ Requires-Dist: pytest; extra == 'test'
40
+ Description-Content-Type: text/markdown
41
+
42
+ <h1 align="center">bedhost</h1>
43
+
44
+ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
45
+ [![Github badge](https://img.shields.io/badge/source-github-354a75?logo=github)](https://github.com/databio/bedhost)
46
+
47
+
48
+ `bedhost` is a Python FastAPI module for the API that powers BEDbase.
49
+ It needs a path to the *bedbase configuration file*, which can be provided either via `-c`/`--config` argument or read from `$BEDBASE_CONFIG` environment variable.
50
+
51
+ ---
52
+
53
+ ## Links of interest
54
+
55
+ - **Deployed public instance**: <a href="https://bedbase.org/" target="_blank">https://bedbase.org/</a>
56
+ - **Documentation**: <a href="https://docs.bedbase.org/" target="_blank">https://docs.bedbase.org/bedhost</a>
57
+ - **API**: <a href="https://api.bedbase.org/" target="_blank">https://api.bedbase.org/</a>
58
+ - **DEV API**: <a href="https://dev.bedbase.org/" target="_blank">https://api-dev.bedbase.org/</a>
59
+ - **UI**: <a href="https://bedbase.org/" target="_blank">https://bedbase.org/</a>
60
+ - **DEV UI**: <a href="https://dev.bedbase.org/" target="_blank">https://dev.bedbase.org/</a>
61
+ - **Source Code**: <a href="https://github.com/databio/bedhost/" target="_blank">https://github.com/databio/bedhost/</a>
62
+
63
+ ## Developer quick-start
64
+
65
+ ```
66
+ source environment/production.env
67
+ BEDBASE_CONFIG=deployment/config/api.bedbase.org.yaml uvicorn bedhost.main:app --reload
68
+ ```
69
+
70
+ ## Embeddings
71
+
72
+ Search, `/v1/bed/embed`, and `/v1/bed/umap` need embedding models. By default
73
+ bbconf loads them all in-process, which needs `uv pip install -e ".[local]" --torch-backend cpu`.
74
+ The production configs instead set bbconf's `embedding_server:` block (see the
75
+ bbconf README), so the two text models (dense and sparse) come from embedding
76
+ servers and are not loaded by bedhost. The region model (`path.region2vec`) still
77
+ runs in-process, so the Docker image keeps CPU-only torch. The servers come from
78
+ these environment variables, which production reads from AWS SSM:
79
+
80
+ | Variable | Config key |
81
+ |---|---|
82
+ | `EMBEDDING_DENSE_URL` | `embedding_server.text_url` |
83
+ | `EMBEDDING_SPARSE_URL` | `embedding_server.sparse_url` |
84
+ | `EMBEDDING_API_KEY` | `embedding_server.api_key` |
85
+
86
+ To move the region model to a server too, add `embedding_server.region_url`.
87
+ For local runs, `source environment/production.env` sets all three from the lab's `pass` store.
88
+
89
+ If a server is down, bedhost still starts and the routes that need it return 503.
90
+
91
+ UI development:
92
+ ```terminal
93
+ export VITE_API_HOST=http://localhost:8000
94
+ ```
95
+
96
+ ## Testing
97
+
98
+ Black-box compliance tests (against any deployment):
99
+
100
+ ```
101
+ pytest tests/api/ --api_root=https://api.bedbase.org
102
+ ```
103
+
104
+ Integration tests (ephemeral Postgres + Qdrant + in-process TestClient, ~30s):
105
+
106
+ ```
107
+ ./tests/scripts/test-integration.sh
108
+ ```
109
+
110
+ Manual service control:
111
+
112
+ ```
113
+ ./tests/scripts/services.sh start
114
+ RUN_INTEGRATION_TESTS=true pytest tests/integration/ tests/api/
115
+ ./tests/scripts/services.sh stop
116
+ ```
117
+
118
+ Integration tests require network access on first boot (bbconf downloads a
119
+ licenses CSV from GitHub and may pull HuggingFace models unless
120
+ `BEDHOST_INIT_ML=false` is set; `test-integration.sh` sets it).
121
+ `BEDHOST_LOG_LEVEL` (default `INFO`) sets the log level for bedhost, bbconf, and geniml.
@@ -0,0 +1,80 @@
1
+ <h1 align="center">bedhost</h1>
2
+
3
+ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
4
+ [![Github badge](https://img.shields.io/badge/source-github-354a75?logo=github)](https://github.com/databio/bedhost)
5
+
6
+
7
+ `bedhost` is a Python FastAPI module for the API that powers BEDbase.
8
+ It needs a path to the *bedbase configuration file*, which can be provided either via `-c`/`--config` argument or read from `$BEDBASE_CONFIG` environment variable.
9
+
10
+ ---
11
+
12
+ ## Links of interest
13
+
14
+ - **Deployed public instance**: <a href="https://bedbase.org/" target="_blank">https://bedbase.org/</a>
15
+ - **Documentation**: <a href="https://docs.bedbase.org/" target="_blank">https://docs.bedbase.org/bedhost</a>
16
+ - **API**: <a href="https://api.bedbase.org/" target="_blank">https://api.bedbase.org/</a>
17
+ - **DEV API**: <a href="https://dev.bedbase.org/" target="_blank">https://api-dev.bedbase.org/</a>
18
+ - **UI**: <a href="https://bedbase.org/" target="_blank">https://bedbase.org/</a>
19
+ - **DEV UI**: <a href="https://dev.bedbase.org/" target="_blank">https://dev.bedbase.org/</a>
20
+ - **Source Code**: <a href="https://github.com/databio/bedhost/" target="_blank">https://github.com/databio/bedhost/</a>
21
+
22
+ ## Developer quick-start
23
+
24
+ ```
25
+ source environment/production.env
26
+ BEDBASE_CONFIG=deployment/config/api.bedbase.org.yaml uvicorn bedhost.main:app --reload
27
+ ```
28
+
29
+ ## Embeddings
30
+
31
+ Search, `/v1/bed/embed`, and `/v1/bed/umap` need embedding models. By default
32
+ bbconf loads them all in-process, which needs `uv pip install -e ".[local]" --torch-backend cpu`.
33
+ The production configs instead set bbconf's `embedding_server:` block (see the
34
+ bbconf README), so the two text models (dense and sparse) come from embedding
35
+ servers and are not loaded by bedhost. The region model (`path.region2vec`) still
36
+ runs in-process, so the Docker image keeps CPU-only torch. The servers come from
37
+ these environment variables, which production reads from AWS SSM:
38
+
39
+ | Variable | Config key |
40
+ |---|---|
41
+ | `EMBEDDING_DENSE_URL` | `embedding_server.text_url` |
42
+ | `EMBEDDING_SPARSE_URL` | `embedding_server.sparse_url` |
43
+ | `EMBEDDING_API_KEY` | `embedding_server.api_key` |
44
+
45
+ To move the region model to a server too, add `embedding_server.region_url`.
46
+ For local runs, `source environment/production.env` sets all three from the lab's `pass` store.
47
+
48
+ If a server is down, bedhost still starts and the routes that need it return 503.
49
+
50
+ UI development:
51
+ ```terminal
52
+ export VITE_API_HOST=http://localhost:8000
53
+ ```
54
+
55
+ ## Testing
56
+
57
+ Black-box compliance tests (against any deployment):
58
+
59
+ ```
60
+ pytest tests/api/ --api_root=https://api.bedbase.org
61
+ ```
62
+
63
+ Integration tests (ephemeral Postgres + Qdrant + in-process TestClient, ~30s):
64
+
65
+ ```
66
+ ./tests/scripts/test-integration.sh
67
+ ```
68
+
69
+ Manual service control:
70
+
71
+ ```
72
+ ./tests/scripts/services.sh start
73
+ RUN_INTEGRATION_TESTS=true pytest tests/integration/ tests/api/
74
+ ./tests/scripts/services.sh stop
75
+ ```
76
+
77
+ Integration tests require network access on first boot (bbconf downloads a
78
+ licenses CSV from GitHub and may pull HuggingFace models unless
79
+ `BEDHOST_INIT_ML=false` is set; `test-integration.sh` sets it).
80
+ `BEDHOST_LOG_LEVEL` (default `INFO`) sets the log level for bedhost, bbconf, and geniml.
@@ -0,0 +1,5 @@
1
+ import logging
2
+
3
+ from .const import PKG_NAME
4
+
5
+ _LOGGER = logging.getLogger(PKG_NAME)
@@ -0,0 +1 @@
1
+ __version__ = "0.16.0"
@@ -1,18 +1,21 @@
1
+ import argparse
2
+
3
+ from ubiquerg import VersionInHelpParser
4
+
1
5
  from . import PKG_NAME
2
6
  from ._version import __version__
3
- from ubiquerg import VersionInHelpParser
4
- from yacman import select_config
5
- from bbconf.const import CFG_ENV_VARS
7
+ from .const import CFG_ENV_VARS
6
8
 
7
9
 
8
- def build_parser():
10
+ def build_parser() -> argparse.ArgumentParser:
9
11
  """
10
- Building argument parser
12
+ Build the argument parser.
11
13
 
12
- :return argparse.ArgumentParser
14
+ Returns:
15
+ The configured argument parser.
13
16
  """
14
17
 
15
- env_var_val = select_config(config_env_vars=CFG_ENV_VARS)
18
+ # env_var_val = select_config(config_env_vars=CFG_ENV_VARS)
16
19
 
17
20
  banner = "%(prog)s - REST API for the bedstat pipeline produced statistics"
18
21
  additional_description = (
@@ -32,7 +35,7 @@ def build_parser():
32
35
 
33
36
  subparsers = parser.add_subparsers(dest="command")
34
37
 
35
- def add_subparser(cmd, description):
38
+ def add_subparser(cmd: str, description: str) -> argparse.ArgumentParser:
36
39
  return subparsers.add_parser(cmd, description=description, help=description)
37
40
 
38
41
  sps = {}
@@ -45,7 +48,6 @@ def build_parser():
45
48
  required=False,
46
49
  dest="config",
47
50
  help="A path to the bedhost config file (YAML). If not provided, "
48
- f"the first available environment variable among: {', '.join(CFG_ENV_VARS)} will be used if set."
49
- f" Currently: {env_var_val}",
51
+ f"the first available environment variable among: {', '.join(CFG_ENV_VARS)} will be used if set.",
50
52
  )
51
53
  return parser
@@ -0,0 +1,57 @@
1
+ import os
2
+
3
+ PKG_NAME: str = "bedhost"
4
+
5
+ # Environment variables checked (in order) for the path to the bedhost config file.
6
+ CFG_ENV_VARS: list[str] = ["BEDBASE_CONFIG"]
7
+
8
+ TEMPLATES_DIRNAME: str = "templates"
9
+ STATIC_DIRNAME: str = "../docs"
10
+ STATIC_PATH: str = os.path.join(
11
+ os.path.dirname(os.path.abspath(__file__)), STATIC_DIRNAME
12
+ )
13
+
14
+ EXAMPLE_BED: str = "bbad85f21962bb8d972444f7f9a3a932"
15
+ EXAMPLE_BEDSET: str = "gse218680"
16
+
17
+ # how often to save usage data (in hours)
18
+ USAGE_SAVE_HOURS: int = 1
19
+ # For how many days record usage data (every month this will be reset)
20
+ USAGE_RECORD_DAYS: int = 30
21
+
22
+
23
+ MAX_FILE_SIZE: int = 1024 * 1024 * 20
24
+ # Decompressed size cap for gzip uploads, so a small gzip cannot expand without bound.
25
+ MAX_UNCOMPRESSED_SIZE: int = 1024 * 1024 * 200
26
+ MAX_REGION_NUMBER: int = 5000000
27
+ MAX_BATCH_IDS: int = (
28
+ 1000 # /bed/batch and /bed/collection/stats (bbconf's get_batch limit)
29
+ )
30
+ MIN_REGION_WIDTH: int = 10
31
+ # Chunk size used when copying an uploaded file to disk.
32
+ UPLOAD_CHUNK_SIZE: int = 1024 * 1024
33
+ # Slack for multipart boundaries and headers in the Content-Length pre-check.
34
+ MULTIPART_OVERHEAD: int = 64 * 1024
35
+ # Routes that accept BED file uploads (guarded by upload_size_guard).
36
+ UPLOAD_ROUTES: frozenset[str] = frozenset(
37
+ {"/v1/bed/embed", "/v1/bed/umap", "/v1/bed/search/bed"}
38
+ )
39
+
40
+ # Upper bounds for request inputs (security review finding O1).
41
+ MAX_LIST_LIMIT: int = 10000 # paged list endpoints
42
+ MAX_SEARCH_LIMIT: int = 100 # search + neighbour endpoints (UI max is 100)
43
+ # hg38 with alts/decoys/unplaced is under 3,500 contigs (hs38DH ~3,366);
44
+ # mm10/mm39 are under 100. The UI only sends chromosomes that actually
45
+ # appear in the uploaded BED file, so 10,000 is a generous ceiling that
46
+ # still bounds the work.
47
+ MAX_CHROM_ENTRIES: int = 10000 # /analyze-genome dict size
48
+ MAX_CHROM_NAME_LENGTH: int = 256 # /analyze-genome dict key length
49
+ BIGBED_TIMEOUT_SECONDS: int = 60 # /regions bigBedToBed wall-clock cap
50
+ MAX_REGIONS_OUTPUT_BYTES: int = 1024 * 1024 * 20 # /regions response size cap
51
+
52
+ # Public CDN base for bulk metadata export artifacts. These live directly on the
53
+ # storage CDN (Backblaze B2 fronted by Cloudflare), NOT behind the API's
54
+ # /v1/files/ redirect proxy — so export URLs must use this base rather than the
55
+ # config's http access-method prefix (which points at api.bedbase.org/v1/files/
56
+ # and, notably, 405s on HEAD, breaking DuckDB range probing).
57
+ EXPORTS_URL_BASE: str = "https://data2.bedbase.org/"
@@ -0,0 +1,245 @@
1
+ """
2
+ Logic shared by the REST routers and the MCP server.
3
+
4
+ Must not import fastapi, starlette, or any other bedhost module except const.
5
+ The MCP package imports this module, and the MCP package has to work without
6
+ the web stack installed (tests/mcp/test_layering.py checks both rules).
7
+ """
8
+
9
+ import logging
10
+ import subprocess
11
+ import tempfile
12
+ import threading
13
+ from dataclasses import dataclass
14
+ from enum import Enum
15
+
16
+ from bbconf.exceptions import BEDFileNotFoundError
17
+ from bbconf.models.bed_models import BedListSearchResult, QdrantSearchResult
18
+ from pydantic import BaseModel
19
+
20
+ from .const import EXPORTS_URL_BASE, PKG_NAME
21
+
22
+ _LOGGER = logging.getLogger(PKG_NAME)
23
+
24
+
25
+ class BaseListResponse(BaseModel):
26
+ count: int
27
+ limit: int
28
+ offset: int
29
+ results: list
30
+
31
+
32
+ # --------------------------------------------------------------------------- #
33
+ # Region queries (bigBedToBed) #
34
+ # --------------------------------------------------------------------------- #
35
+ class RegionOutputTooLarge(Exception):
36
+ """bigBedToBed produced more output than the allowed ceiling."""
37
+
38
+
39
+ def run_bigbed_to_bed(cmd: list[str], timeout: int, max_bytes: int) -> str:
40
+ """Run bigBedToBed, wait for it, return only the first 3 BED columns.
41
+
42
+ Output is read line by line and trimmed as it arrives, so memory is bounded
43
+ by ``max_bytes`` rather than by the size of the bigBed file.
44
+
45
+ Args:
46
+ cmd: Full argv for the ``bigBedToBed`` invocation (no shell).
47
+ timeout: Wall-clock timeout in seconds.
48
+ max_bytes: Ceiling on the trimmed output size.
49
+
50
+ Returns:
51
+ The first three tab-separated columns of each output line,
52
+ newline-terminated. Empty string if there is no output.
53
+
54
+ Raises:
55
+ FileNotFoundError: If the binary is missing.
56
+ RegionOutputTooLarge: If the output exceeds ``max_bytes``. The child is
57
+ killed before this is raised.
58
+ subprocess.TimeoutExpired: If the process exceeds ``timeout``. The
59
+ child is killed before this is raised.
60
+ subprocess.CalledProcessError: If the process exits non-zero.
61
+ """
62
+ timed_out = threading.Event()
63
+ rows: list[str] = []
64
+ total = 0
65
+ # stderr goes to a file so a chatty child can never block on a full pipe.
66
+ with (
67
+ tempfile.TemporaryFile() as err,
68
+ subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=err, text=True) as proc,
69
+ ):
70
+
71
+ def _kill_on_timeout():
72
+ timed_out.set()
73
+ proc.kill()
74
+
75
+ timer = threading.Timer(timeout, _kill_on_timeout)
76
+ timer.start()
77
+ try:
78
+ for line in proc.stdout:
79
+ row = "\t".join(line.rstrip("\n").split("\t", 3)[:3]) + "\n"
80
+ total += len(row)
81
+ if total > max_bytes:
82
+ proc.kill()
83
+ raise RegionOutputTooLarge(
84
+ f"bigBedToBed output exceeded {max_bytes} bytes"
85
+ )
86
+ rows.append(row)
87
+ proc.wait()
88
+ finally:
89
+ timer.cancel()
90
+
91
+ if timed_out.is_set():
92
+ raise subprocess.TimeoutExpired(cmd, timeout)
93
+ if proc.returncode != 0:
94
+ err.seek(0)
95
+ raise subprocess.CalledProcessError(
96
+ proc.returncode, cmd, stderr=err.read().decode(errors="replace")
97
+ )
98
+ return "".join(rows)
99
+
100
+
101
+ def build_exports_url(file_path: str) -> str:
102
+ """
103
+ Build a public HTTPS CDN URL for an export / analysis-file S3 key.
104
+
105
+ Uses explicit string concatenation rather than ``os.path.join``: the latter
106
+ is a filesystem operation and, given an absolute ``file_path`` (leading
107
+ ``/``), silently discards the base URL entirely — dropping the scheme and
108
+ host. Stripping a trailing slash off the base and a leading slash off the
109
+ key guarantees exactly one separator and always preserves the host.
110
+
111
+ Args:
112
+ file_path: The S3 key (relative or absolute) of the artifact.
113
+
114
+ Returns:
115
+ The absolute ``https://data2.bedbase.org/...`` URL for the artifact.
116
+ """
117
+ return f"{EXPORTS_URL_BASE.rstrip('/')}/{file_path.lstrip('/')}"
118
+
119
+
120
+ # --------------------------------------------------------------------------- #
121
+ # BED text search #
122
+ # --------------------------------------------------------------------------- #
123
+ class SearchKind(str, Enum):
124
+ """Which path search_bed took to answer a query."""
125
+
126
+ EXACT_ID = "exact_id"
127
+ ACCESSION = "accession"
128
+ SEMANTIC = "semantic"
129
+
130
+
131
+ class MLUnavailable(Exception):
132
+ """Semantic search was needed but the dense encoder is not loaded."""
133
+
134
+
135
+ @dataclass
136
+ class BedSearchOutcome:
137
+ kind: SearchKind
138
+ result: BedListSearchResult
139
+
140
+
141
+ _ACCESSION_PREFIXES = ("gsm", "gse", "encff", "encsr", "geo:", "encode:")
142
+
143
+
144
+ def _parse_accession(query: str) -> tuple[str, str] | None:
145
+ """Return ``(source, accession)`` for a GEO/ENCODE accession, else None."""
146
+ low = query.replace(" ", "").lower()
147
+ if not low.startswith(_ACCESSION_PREFIXES):
148
+ return None
149
+ acc = low.replace("geo:", "").replace("encode:", "")
150
+ if acc.startswith(("gsm", "gse")):
151
+ return "geo", acc
152
+ if acc.startswith(("encff", "encsr")):
153
+ return "encode", acc.upper()
154
+ return None
155
+
156
+
157
+ def _exact_id_result(agent, bed_id: str, limit: int, offset: int):
158
+ """The file itself first, then its nearest neighbours; None if no such file.
159
+
160
+ The file takes slot 0, so a page at ``offset`` holds neighbours starting at
161
+ ``offset - 1``. If the neighbour lookup fails (for example the vector index
162
+ is not loaded), the result is just the file.
163
+ """
164
+ try:
165
+ meta = agent.bed.get(bed_id)
166
+ except BEDFileNotFoundError:
167
+ return None
168
+ hit = QdrantSearchResult(id=bed_id, payload={}, score=1.0, metadata=meta)
169
+
170
+ n_limit = limit - 1 if offset == 0 else limit
171
+ n_offset = 0 if offset == 0 else offset - 1
172
+ try:
173
+ neighbours = (
174
+ agent.bed.get_neighbours(bed_id, limit=n_limit, offset=n_offset)
175
+ if n_limit > 0
176
+ else None
177
+ )
178
+ except Exception as e: # the file exists; neighbours are a bonus
179
+ _LOGGER.warning(f"Neighbour lookup failed for {bed_id}: {e}")
180
+ neighbours = None
181
+
182
+ if neighbours is None or not neighbours.results:
183
+ rows = [hit] if offset == 0 else []
184
+ return BedListSearchResult(count=1, limit=limit, offset=offset, results=rows)
185
+ rows = neighbours.results
186
+ if offset == 0:
187
+ rows = [hit, *rows]
188
+ return BedListSearchResult(
189
+ count=neighbours.count, limit=limit, offset=offset, results=rows
190
+ )
191
+
192
+
193
+ def search_bed(
194
+ agent,
195
+ query: str,
196
+ genome: str | None = None,
197
+ assay: str | None = None,
198
+ limit: int = 10,
199
+ offset: int = 0,
200
+ ) -> BedSearchOutcome:
201
+ """Exact BED id -> GEO/ENCODE accession -> hybrid semantic search.
202
+
203
+ The first two paths need no ML models. Only the semantic path needs the
204
+ dense encoder; without it this raises MLUnavailable.
205
+
206
+ Args:
207
+ agent: The bbconf BedBaseAgent.
208
+ query: Free text, a GEO/ENCODE accession, or a 32-character BED id.
209
+ genome: Genome filter for semantic search (e.g. "hg38").
210
+ assay: Assay filter for semantic search (e.g. "ATAC-seq").
211
+ limit: Rows to return.
212
+ offset: Rows to skip.
213
+
214
+ Raises:
215
+ MLUnavailable: If the query needs semantic search and ML is off.
216
+ """
217
+ spaceless = query.replace(" ", "")
218
+ if len(spaceless) == 32 and spaceless == query:
219
+ result = _exact_id_result(agent, query, limit, offset)
220
+ if result is not None:
221
+ return BedSearchOutcome(SearchKind.EXACT_ID, result)
222
+
223
+ if parsed := _parse_accession(query):
224
+ source, accession = parsed
225
+ _LOGGER.info(f"Searching for {source} accession {accession}")
226
+ found = agent.bed.search_external_file(source=source, accession=accession)
227
+ if found.count:
228
+ rows = (found.results or [])[offset : offset + limit]
229
+ return BedSearchOutcome(
230
+ SearchKind.ACCESSION,
231
+ BedListSearchResult(
232
+ count=found.count, limit=limit, offset=offset, results=rows
233
+ ),
234
+ )
235
+
236
+ if getattr(agent.config, "dense_encoder", None) is None:
237
+ raise MLUnavailable("Text search unavailable (ML models disabled)")
238
+ result = agent.bed.hybrid_search(
239
+ query,
240
+ genome_alias=genome or "",
241
+ assay=assay or "",
242
+ limit=limit,
243
+ offset=offset,
244
+ )
245
+ return BedSearchOutcome(SearchKind.SEMANTIC, result)