compose-runner 0.6.3__py2.py3-none-any.whl → 0.6.4rc1__py2.py3-none-any.whl

compose_runner/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.6.3'
-__version_tuple__ = version_tuple = (0, 6, 3)
+__version__ = version = '0.6.4rc1'
+__version_tuple__ = version_tuple = (0, 6, 4, 'rc1')
 
 __commit_id__ = commit_id = None

compose_runner/aws_lambda/run_handler.py

@@ -4,6 +4,8 @@ import json
 import logging
 import os
 import uuid
+import urllib.error
+import urllib.request
 from typing import Any, Dict, Optional
 
 import boto3
@@ -22,6 +24,8 @@ RESULTS_PREFIX_ENV = "RESULTS_PREFIX"
 NSC_KEY_ENV = "NSC_KEY"
 NV_KEY_ENV = "NV_KEY"
 
+DEFAULT_TASK_SIZE = "standard"
+
 
 def _log(job_id: str, message: str, **details: Any) -> None:
     payload = {"job_id": job_id, "message": message, **details}
@@ -29,6 +33,67 @@ def _log(job_id: str, message: str, **details: Any) -> None:
     logger.info(json.dumps(payload))
 
 
+def _compose_api_base_url(environment: str) -> str:
+    env = (environment or "production").lower()
+    if env == "staging":
+        return "https://synth.neurostore.xyz/api"
+    if env == "local":
+        return "http://localhost:81/api"
+    return "https://compose.neurosynth.org/api"
+
+
+def _fetch_meta_analysis(meta_analysis_id: str, environment: str) -> Optional[Dict[str, Any]]:
+    base_url = _compose_api_base_url(environment).rstrip("/")
+    url = f"{base_url}/meta-analyses/{meta_analysis_id}?nested=true"
+    request = urllib.request.Request(url, headers={"User-Agent": "compose-runner/submit"})
+    try:
+        with urllib.request.urlopen(request, timeout=10) as response:
+            return json.load(response)
+    except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError) as exc:
+        logger.warning("Failed to fetch meta-analysis %s: %s", meta_analysis_id, exc)
+        return None
+
+
+def _requires_large_task(specification: Dict[str, Any]) -> bool:
+    if not isinstance(specification, dict):
+        return False
+    corrector = specification.get("corrector")
+    if not isinstance(corrector, dict):
+        return False
+    if corrector.get("type") != "FWECorrector":
+        return False
+    args = corrector.get("args")
+    if not isinstance(args, dict):
+        return False
+    method = args.get("method")
+    if method is None:
+        kwargs = args.get("**kwargs")
+        if isinstance(kwargs, dict):
+            method = kwargs.get("method")
+    if isinstance(method, str) and method.lower() == "montecarlo":
+        return True
+    return False
+
+
+def _select_task_size(meta_analysis_id: str, environment: str, artifact_prefix: str) -> str:
+    doc = _fetch_meta_analysis(meta_analysis_id, environment)
+    if not doc:
+        return DEFAULT_TASK_SIZE
+    specification = doc.get("specification")
+    try:
+        if _requires_large_task(specification):
+            _log(
+                artifact_prefix,
+                "workflow.task_size_selected",
+                task_size="large",
+                reason="montecarlo_fwe",
+            )
+            return "large"
+    except Exception as exc:  # noqa: broad-except
+        logger.warning("Failed to evaluate specification for %s: %s", meta_analysis_id, exc)
+    return DEFAULT_TASK_SIZE
+
+
 def _job_input(
     payload: Dict[str, Any],
     artifact_prefix: str,
@@ -36,6 +101,7 @@ def _job_input(
     prefix: Optional[str],
     nsc_key: Optional[str],
     nv_key: Optional[str],
+    task_size: str,
 ) -> Dict[str, Any]:
     no_upload_flag = bool(payload.get("no_upload", False))
     doc: Dict[str, Any] = {
@@ -44,6 +110,7 @@ def _job_input(
         "environment": payload.get("environment", "production"),
         "no_upload": "true" if no_upload_flag else "false",
         "results": {"bucket": bucket or "", "prefix": prefix or ""},
+        "task_size": task_size,
     }
     n_cores = payload.get("n_cores")
     doc["n_cores"] = str(n_cores) if n_cores is not None else ""
@@ -76,7 +143,10 @@ def handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
     nsc_key = payload.get("nsc_key") or os.environ.get(NSC_KEY_ENV)
     nv_key = payload.get("nv_key") or os.environ.get(NV_KEY_ENV)
 
-    job_input = _job_input(payload, artifact_prefix, bucket, prefix, nsc_key, nv_key)
+    environment = payload.get("environment", "production")
+    task_size = _select_task_size(payload["meta_analysis_id"], environment, artifact_prefix)
+
+    job_input = _job_input(payload, artifact_prefix, bucket, prefix, nsc_key, nv_key, task_size)
     params = {
         "stateMachineArn": os.environ[STATE_MACHINE_ARN_ENV],
         "name": artifact_prefix,

compose_runner/ecs_task.py

@@ -93,6 +93,7 @@ def main() -> None:
     nv_key = os.environ.get(NV_KEY_ENV) or None
     no_upload = _bool_from_env(os.environ.get(NO_UPLOAD_ENV))
     n_cores = _resolve_n_cores(os.environ.get(N_CORES_ENV))
+    compose_runner_version = os.environ.get("COMPOSE_RUNNER_VERSION", "unknown")
 
     bucket = os.environ.get(RESULTS_BUCKET_ENV)
     prefix = os.environ.get(RESULTS_PREFIX_ENV)
@@ -106,6 +107,7 @@ def main() -> None:
         meta_analysis_id=meta_analysis_id,
         environment=environment,
         no_upload=no_upload,
+        compose_runner_version=compose_runner_version,
     )
     try:
         url, _ = run_compose(
@@ -125,6 +127,7 @@ def main() -> None:
             "result_url": url,
             "artifacts_bucket": bucket,
             "artifacts_prefix": prefix,
+            "compose_runner_version": compose_runner_version,
         }
 
         if bucket:

compose_runner/tests/cassettes/test_lambda_handlers/test_select_task_size_uses_large_for_montecarlo.yaml

@@ -0,0 +1,60 @@
+interactions:
+- request:
+    method: GET
+    uri: https://synth.neurostore.xyz/api/meta-analyses/ZPSvyvhZAopz?nested=true
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      User-Agent:
+      - python-requests/2.32.4
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx/1.21.6
+      Date:
+      - Tue, 21 Oct 2025 14:08:45 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Accept-Encoding
+      Content-Encoding:
+      - gzip
+      Strict-Transport-Security:
+      - max-age=31536000
+    body:
+      string: '{"id": "ZPSvyvhZAopz", "created_at": "2025-10-21T04:57:40.236536+00:00",
+        "updated_at": null, "user": "github|12564882", "username": "James Kent", "name":
+        "Untitled MKDADensity Meta Analysis: included", "description": "MKDADensity
+        meta analysis with FWECorrector", "provenance": null, "specification": {"id":
+        "zQdMa4uAaYYU", "created_at": "2025-10-21T04:57:39.888528+00:00", "updated_at":
+        null, "user": "github|12564882", "username": "James Kent", "type": "CBMA",
+        "estimator": {"type": "MKDADensity", "args": {"null_method": "approximate",
+        "n_iters": 5000, "**kwargs": {}, "kernel__r": 10, "kernel__value": 1}}, "database_studyset":
+        null, "filter": "included", "corrector": {"type": "FWECorrector", "args":
+        {"voxel_thresh": 0.001, "n_iters": 5000, "vfwe_only": false, "method": "montecarlo"}},
+        "conditions": [true], "weights": [1.0]}, "neurostore_analysis": {"id": "8S5xRedCGRkz",
+        "created_at": "2025-10-21T04:57:40.255480+00:00", "updated_at": null, "neurostore_id":
+        null, "exception": null, "traceback": null, "status": "PENDING"}, "studyset":
+        {"id": "9jPvdkuRufUP", "created_at": "2025-10-21T04:57:40.008456+00:00", "updated_at":
+        null, "user": "github|12564882", "username": "James Kent", "snapshot": null,
+        "neurostore_id": "3EmvH2LELwR2", "version": null, "url": "https://neurostore.org/api/studysets/3EmvH2LELwR2"},
+        "annotation": {"id": "YVLt6DRFKdd5", "created_at": "2025-10-21T04:57:40.121637+00:00",
+        "updated_at": null, "user": "github|12564882", "username": "James Kent", "snapshot":
+        null, "neurostore_id": "TebrRstj8ofh", "studyset": "3EmvH2LELwR2", "url":
+        "https://neurostore.org/api/annotations/TebrRstj8ofh"}, "project": "D2cTfoxNfpLy",
+        "cached_studyset": "9jPvdkuRufUP", "cached_annotation": "YVLt6DRFKdd5", "run_key":
+        "PDeDnh_8MXc88xoVJySz3w", "results": [], "neurostore_url": null}'
+    http_version: HTTP/1.1
+version: 1

compose_runner/tests/cassettes/test_lambda_handlers/test_select_task_size_uses_standard_for_fdr.yaml

@@ -0,0 +1,55 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Connection:
+      - close
+      Host:
+      - synth.neurostore.xyz
+      User-Agent:
+      - compose-runner/submit
+    method: GET
+    uri: https://synth.neurostore.xyz/api/meta-analyses/VtFZJFniCKvG?nested=true
+  response:
+    body:
+      string: '{"id": "VtFZJFniCKvG", "created_at": "2025-10-21T14:10:35.309383+00:00",
+        "updated_at": null, "user": "github|12564882", "username": "James Kent", "name":
+        "Untitled MKDADensity Meta Analysis: included (1)", "description": "MKDADensity
+        meta analysis with FDRCorrector", "provenance": null, "specification": {"id":
+        "DtVzKEKGaXLu", "created_at": "2025-10-21T14:10:34.564365+00:00", "updated_at":
+        null, "user": "github|12564882", "username": "James Kent", "type": "CBMA",
+        "estimator": {"type": "MKDADensity", "args": {"null_method": "approximate",
+        "n_iters": 5000, "**kwargs": {}, "kernel__r": 10, "kernel__value": 1}}, "database_studyset":
+        null, "filter": "included", "corrector": {"type": "FDRCorrector", "args":
+        {"method": "indep", "alpha": 0.05}}, "conditions": [true], "weights": [1.0]},
+        "neurostore_analysis": {"id": "564c8kRnJVT4", "created_at": "2025-10-21T14:10:35.325173+00:00",
+        "updated_at": null, "neurostore_id": null, "exception": null, "traceback":
+        null, "status": "PENDING"}, "studyset": {"id": "FA3BDBdGRZ5d", "created_at":
+        "2025-10-21T14:10:34.821625+00:00", "updated_at": null, "user": "github|12564882",
+        "username": "James Kent", "snapshot": null, "neurostore_id": "3EmvH2LELwR2",
+        "version": null, "url": "https://neurostore.org/api/studysets/3EmvH2LELwR2"},
+        "annotation": {"id": "XELVYV7ftp7e", "created_at": "2025-10-21T14:10:35.183354+00:00",
+        "updated_at": null, "user": "github|12564882", "username": "James Kent", "snapshot":
+        null, "neurostore_id": "TebrRstj8ofh", "studyset": "3EmvH2LELwR2", "url":
+        "https://neurostore.org/api/annotations/TebrRstj8ofh"}, "project": "D2cTfoxNfpLy",
+        "cached_studyset": "FA3BDBdGRZ5d", "cached_annotation": "XELVYV7ftp7e", "run_key":
+        "V_jTcP2zfNlWD4KhwKKcJw", "results": [], "neurostore_url": null}'
+    headers:
+      Connection:
+      - close
+      Content-Length:
+      - '1750'
+      Content-Type:
+      - application/json
+      Date:
+      - Tue, 21 Oct 2025 14:14:50 GMT
+      Server:
+      - nginx/1.21.6
+      Strict-Transport-Security:
+      - max-age=31536000
+      Vary:
+      - Accept-Encoding
+    status:
+      code: 200
+      message: OK
+version: 1

compose_runner/tests/test_lambda_handlers.py

@@ -4,6 +4,8 @@ import json
 from datetime import datetime, timezone
 from typing import Any, Dict
 
+import pytest
+
 from compose_runner.aws_lambda import log_poll_handler, results_handler, run_handler, status_handler
 
 
@@ -23,6 +25,28 @@ def _make_http_event(payload: Dict[str, Any]) -> Dict[str, Any]:
     }
 
 
+def test_requires_large_task_detection():
+    spec = {"corrector": {"type": "FWECorrector", "args": {"method": "montecarlo"}}}
+    assert run_handler._requires_large_task(spec)
+
+
+def test_requires_large_task_false_when_method_differs():
+    spec = {"corrector": {"type": "FWECorrector", "args": {"method": "bonferroni"}}}
+    assert run_handler._requires_large_task(spec) is False
+
+
+@pytest.mark.vcr(record_mode="once")
+def test_select_task_size_uses_large_for_montecarlo():
+    task_size = run_handler._select_task_size("ZPSvyvhZAopz", "staging", "artifact-test")
+    assert task_size == "large"
+
+
+@pytest.mark.vcr(record_mode="once")
+def test_select_task_size_uses_standard_for_fdr():
+    task_size = run_handler._select_task_size("VtFZJFniCKvG", "staging", "artifact-test")
+    assert task_size == "standard"
+
+
 def test_run_handler_http_success(monkeypatch, tmp_path):
     captured = {}
 
@@ -36,6 +60,7 @@ def test_run_handler_http_success(monkeypatch, tmp_path):
                 ...
 
     monkeypatch.setattr(run_handler, "_SFN_CLIENT", FakeSFN())
+    monkeypatch.setattr(run_handler, "_select_task_size", lambda *args: "standard")
     monkeypatch.setenv("STATE_MACHINE_ARN", "arn:aws:states:state-machine")
     monkeypatch.setenv("RESULTS_BUCKET", "bucket")
     monkeypatch.setenv("RESULTS_PREFIX", "prefix")
@@ -63,6 +88,32 @@ def test_run_handler_http_success(monkeypatch, tmp_path):
     assert input_doc["results"]["prefix"] == "prefix"
     assert input_doc["nsc_key"] == "nsc"
     assert input_doc["nv_key"] == "nv"
+    assert input_doc["task_size"] == "standard"
+
+
+def test_run_handler_http_uses_large_task(monkeypatch):
+    captured = {}
+
+    class FakeSFN:
+        def start_execution(self, **kwargs):
+            captured.update(kwargs)
+            return {"executionArn": "arn:aws:states:us-east-1:123:execution:state-machine:run-456"}
+
+        class exceptions:
+            class ExecutionAlreadyExists(Exception):
+                ...
+
+    monkeypatch.setattr(run_handler, "_SFN_CLIENT", FakeSFN())
+    monkeypatch.setattr(run_handler, "_select_task_size", lambda *args: "large")
+    monkeypatch.setenv("STATE_MACHINE_ARN", "arn:aws:states:state-machine")
+    monkeypatch.setenv("RESULTS_BUCKET", "bucket")
+    monkeypatch.setenv("RESULTS_PREFIX", "prefix")
+
+    event = _make_http_event({"meta_analysis_id": "abc123"})
+    response = run_handler.handler(event, DummyContext())
+    assert response["statusCode"] == 202
+    input_doc = json.loads(captured["input"])
+    assert input_doc["task_size"] == "large"
 
 
 def test_run_handler_missing_meta_analysis(monkeypatch):

{compose_runner-0.6.3.dist-info → compose_runner-0.6.4rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compose-runner
-Version: 0.6.3
+Version: 0.6.4rc1
 Summary: A package for running neurosynth-compose analyses
 Project-URL: Repository, https://github.com/neurostuff/compose-runner
 Author-email: James Kent <jamesdkent21@gmail.com>
@@ -67,7 +67,7 @@ The deployed architecture works like this:
    Pass `-c resultsBucketName=<bucket>` to use an existing S3 bucket, or omit it
    to let the stack create and retain a dedicated bucket. Additional knobs:
 
-   - `-c stateMachineTimeoutSeconds=7200` to control the max wall clock per run
+  - `-c stateMachineTimeoutSeconds=32400` to control the max wall clock per run
    - `-c submitTimeoutSeconds` / `-c statusTimeoutSeconds` / `-c pollTimeoutSeconds`
      to tune Lambda timeouts
    - `-c taskEphemeralStorageGiB` if the default 21 GiB scratch volume is insufficient

{compose_runner-0.6.3.dist-info → compose_runner-0.6.4rc1.dist-info}/RECORD

@@ -1,27 +1,29 @@
 compose_runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-compose_runner/_version.py,sha256=zzZgi3c1knPwH9UzpR0B77qIKc0mxKIzSp9QvD8yW8o,704
+compose_runner/_version.py,sha256=jh9xfbKA06VsDeDHwVjKWWJSjT7budydFj-4rgqpW7M,714
 compose_runner/cli.py,sha256=1tkxFgEe8Yk7VkzE8qxGmCGqLU7UbGin2VaP0AiZkVg,1101
-compose_runner/ecs_task.py,sha256=0s7pQvchNTSOvGNdp5qW7BFNjmIjzkApdZdHLK8rJEM,5154
+compose_runner/ecs_task.py,sha256=15CyLsaf2xrvWM-gGPDOXj_Hq8eJJPiH7xyRas93zn0,5352
 compose_runner/run.py,sha256=yIh8Fj8dfVKvahRl483qGOsDUoAS1FdsYrKZp_HknGo,18525
 compose_runner/sentry.py,sha256=pjqwsZrXrKB0cCy-TL-_2eYJIqUU0aV-8e0SWUk-9Xw,320
 compose_runner/aws_lambda/__init__.py,sha256=yZNXXv7gCPSrtLCEX5Qf4cnzSTS3fHPV6k-SyZwiZIA,48
 compose_runner/aws_lambda/common.py,sha256=cA2G5lO4P8uVBqJaYcU6Y3P3t3syoTmk4SpLKZhAFo8,1688
 compose_runner/aws_lambda/log_poll_handler.py,sha256=eEU-Ra_-17me3e4eqSTd2Nv_qoaOl7zi3kIxD58Tbek,1905
 compose_runner/aws_lambda/results_handler.py,sha256=vSxs4nbWyBmkFFKRGIp5-T4W2hPh9zgj7uNH-e18aW8,2107
-compose_runner/aws_lambda/run_handler.py,sha256=iZW35xqa9FBZQTxRBH0JOpYhQ3Si1eTkxFHB5VJ5drA,3981
+compose_runner/aws_lambda/run_handler.py,sha256=WbaoRp5hsWNqAgDamW9BPihp1tfVVTM94_HKHL4Uawg,6562
 compose_runner/aws_lambda/status_handler.py,sha256=K_VDyPYY3ExiyalDyf35nXi3UZzqj4AenWmlxkzWNXo,3423
 compose_runner/tests/conftest.py,sha256=ijb1iw724izKMxrvclt5x7LljTGoBfHwSS-jIEUe-sQ,191
 compose_runner/tests/test_cli.py,sha256=G3Kz7Nbl2voJ_luXPL7E6slkRNF9lmcpZ-nHBAqeL-M,290
 compose_runner/tests/test_ecs_task.py,sha256=kY_3mPXjUTiKlLMiJpCmNIQP7cWWw047aKVB8-Rr8Ws,463
-compose_runner/tests/test_lambda_handlers.py,sha256=A13q8KEMzxFPdqQYOIEqhYvEwKN1-3SPtGN9nRvLvtU,7115
+compose_runner/tests/test_lambda_handlers.py,sha256=WDoOFR5r-RXTXzasAqGcpVI_G736g1YGdB90WetIZBo,9028
 compose_runner/tests/test_run.py,sha256=Nhx7wz8XxQuxy3kT5yoE_S1Hw0Mgmfn8TWYOZXm1_Gg,1795
+compose_runner/tests/cassettes/test_lambda_handlers/test_select_task_size_uses_large_for_montecarlo.yaml,sha256=Btgzi8zzN9AwK7EbjJYdLIwv2ep3IiJlIHimcyCfrYw,2748
+compose_runner/tests/cassettes/test_lambda_handlers/test_select_task_size_uses_standard_for_fdr.yaml,sha256=40wd30GbtwmhWgfz9cxPdRC8fZiSZOy9Fdc8Eiovacg,2591
 compose_runner/tests/cassettes/test_run/test_download_bundle.yaml,sha256=vgdGDqirjBHosQsspkaN5Ty6XqJbkYUAbGtdImym5xI,79304
 compose_runner/tests/cassettes/test_run/test_run_database_workflow.yaml,sha256=ay0aHtU-nmVWvbmN_EIgO9MMkC4ZeQljKU8nkTXOoDw,8724312
 compose_runner/tests/cassettes/test_run/test_run_group_comparison_workflow.yaml,sha256=FaZpMdcaM7TMgyueyZBGftm6ywUh1HhtGmCegXUmRFA,4029712
 compose_runner/tests/cassettes/test_run/test_run_string_group_comparison_workflow.yaml,sha256=pcn6tQwrimhDtP8yJ3jFlsfEOnk8FWybYQr9IQ5A_KA,3233839
 compose_runner/tests/cassettes/test_run/test_run_workflow.yaml,sha256=0Nk7eJWAmgYALG2ODrezbRhpYsc00JiuYVjXt3TUm5c,3857234
-compose_runner-0.6.3.dist-info/METADATA,sha256=xtlq7SjONY4WD85TbtD9dTdeaBP9WVDS-6383-zBsYo,3432
-compose_runner-0.6.3.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
-compose_runner-0.6.3.dist-info/entry_points.txt,sha256=TyPmB9o2tSWw8L3mcach9r2EL7inRVXE9ew3_XReMIY,55
-compose_runner-0.6.3.dist-info/licenses/LICENSE,sha256=PeiWxrrRme2rIpPMV9vjgGe7UHEKCIcTb0KagYhnyqo,1313
-compose_runner-0.6.3.dist-info/RECORD,,
+compose_runner-0.6.4rc1.dist-info/METADATA,sha256=XcUqv6A8lHXIOh_xQSMHTnl6HnnXUfd09kNpVZMJaaQ,3435
+compose_runner-0.6.4rc1.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
+compose_runner-0.6.4rc1.dist-info/entry_points.txt,sha256=TyPmB9o2tSWw8L3mcach9r2EL7inRVXE9ew3_XReMIY,55
+compose_runner-0.6.4rc1.dist-info/licenses/LICENSE,sha256=PeiWxrrRme2rIpPMV9vjgGe7UHEKCIcTb0KagYhnyqo,1313
+compose_runner-0.6.4rc1.dist-info/RECORD,,