chemsift 1.2.0__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chemsift-1.2.0 → chemsift-1.3.0}/PKG-INFO +27 -1
- {chemsift-1.2.0 → chemsift-1.3.0}/README.md +25 -0
- {chemsift-1.2.0 → chemsift-1.3.0}/pyproject.toml +1 -1
- chemsift-1.3.0/src/chemsift_cloud/add_feature_space.py +145 -0
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/config.py +19 -0
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/download.py +6 -3
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/environments.py +7 -0
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/list.py +15 -11
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/main.py +2 -0
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/map.py +3 -3
- chemsift-1.3.0/src/chemsift_cloud/pool.py +196 -0
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/query.py +44 -15
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/register.py +2 -2
- chemsift-1.3.0/src/chemsift_cloud/standardization.py +89 -0
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/status.py +2 -2
- chemsift-1.3.0/src/chemsift_cloud/supervised.py +217 -0
- chemsift-1.2.0/src/chemsift_cloud/submit.py +0 -95
- chemsift-1.2.0/src/chemsift_cloud/supervised.py +0 -134
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/__init__.py +0 -0
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/env.py +0 -0
- {chemsift-1.2.0 → chemsift-1.3.0}/src/chemsift_cloud/login.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: chemsift
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary:
|
|
5
5
|
Author: Dave Martinez
|
|
6
6
|
Author-email: 34991082+dkm-coder@users.noreply.github.com
|
|
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.13
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.14
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
15
16
|
Requires-Dist: boto3 (>=1.34.135,<2.0.0)
|
|
16
17
|
Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
|
|
17
18
|
Requires-Dist: requests (>=2.32.5,<3.0.0)
|
|
@@ -38,6 +39,8 @@ pip install chemsift
|
|
|
38
39
|
- `chemsift map <register-job-id>`: Run the mapped pipeline against a completed register job
|
|
39
40
|
- `chemsift query <register-job-id> --molecules <csv> --yaml <yml>`: Submit a query job
|
|
40
41
|
- `chemsift supervised <register-job-id> --yaml <yml> [--state <csv>]`: Submit one round of active-learning screening
|
|
42
|
+
- `chemsift supervised --pool <pool-id> --yaml <yml> [--state <csv>]`: The same, against a shared demo pool instead of your own registration
|
|
43
|
+
- `chemsift env list` / `chemsift env use <dev|prod>`: Show or switch the active environment
|
|
41
44
|
- `chemsift status <job-id>`: Check status of a job
|
|
42
45
|
- `chemsift download <job-id> <output-dir>`: Download results of a completed job
|
|
43
46
|
- `chemsift list`: List all job IDs for the current user
|
|
@@ -56,6 +59,29 @@ chemsift download <job-id> .
|
|
|
56
59
|
chemsift supervised <register-job-id> --yaml supervised.yml --state state.csv
|
|
57
60
|
```
|
|
58
61
|
|
|
62
|
+
### Screening a shared pool
|
|
63
|
+
|
|
64
|
+
A pool is a library registered once that every authenticated user of a stack can
|
|
65
|
+
screen, so a new account does not have to register anything before it can run a
|
|
66
|
+
round. Pass `--pool` instead of a register job id — everything else is identical:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
chemsift supervised --pool ecbd-luciferase-v1 --yaml supervised.yml
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
The pool is read-only and lives in its own bucket; everything a round writes still
|
|
73
|
+
goes to your own prefix. `--pool-bucket` overrides the bucket, which defaults to
|
|
74
|
+
the active environment's.
|
|
75
|
+
|
|
76
|
+
Two things the pool form does for you. It names the pool's library in the uploaded
|
|
77
|
+
config when your config did not, so a campaign records what it screened (CS-08).
|
|
78
|
+
And on a resumed round it refuses a state CSV that was started against a different
|
|
79
|
+
library, rather than silently scoring one library's molecules with another's
|
|
80
|
+
ordering.
|
|
81
|
+
|
|
82
|
+
A pool publishes one pinned library with nothing to score it against, so pools are
|
|
83
|
+
screenable with `supervised` but are not valid targets for `query`.
|
|
84
|
+
|
|
59
85
|
## Development
|
|
60
86
|
|
|
61
87
|
This project uses [Poetry](https://python-poetry.org/) for dependency management and packaging.
|
|
@@ -15,6 +15,8 @@ pip install chemsift
|
|
|
15
15
|
- `chemsift map <register-job-id>`: Run the mapped pipeline against a completed register job
|
|
16
16
|
- `chemsift query <register-job-id> --molecules <csv> --yaml <yml>`: Submit a query job
|
|
17
17
|
- `chemsift supervised <register-job-id> --yaml <yml> [--state <csv>]`: Submit one round of active-learning screening
|
|
18
|
+
- `chemsift supervised --pool <pool-id> --yaml <yml> [--state <csv>]`: The same, against a shared demo pool instead of your own registration
|
|
19
|
+
- `chemsift env list` / `chemsift env use <dev|prod>`: Show or switch the active environment
|
|
18
20
|
- `chemsift status <job-id>`: Check status of a job
|
|
19
21
|
- `chemsift download <job-id> <output-dir>`: Download results of a completed job
|
|
20
22
|
- `chemsift list`: List all job IDs for the current user
|
|
@@ -33,6 +35,29 @@ chemsift download <job-id> .
|
|
|
33
35
|
chemsift supervised <register-job-id> --yaml supervised.yml --state state.csv
|
|
34
36
|
```
|
|
35
37
|
|
|
38
|
+
### Screening a shared pool
|
|
39
|
+
|
|
40
|
+
A pool is a library registered once that every authenticated user of a stack can
|
|
41
|
+
screen, so a new account does not have to register anything before it can run a
|
|
42
|
+
round. Pass `--pool` instead of a register job id — everything else is identical:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
chemsift supervised --pool ecbd-luciferase-v1 --yaml supervised.yml
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The pool is read-only and lives in its own bucket; everything a round writes still
|
|
49
|
+
goes to your own prefix. `--pool-bucket` overrides the bucket, which defaults to
|
|
50
|
+
the active environment's.
|
|
51
|
+
|
|
52
|
+
Two things the pool form does for you. It names the pool's library in the uploaded
|
|
53
|
+
config when your config did not, so a campaign records what it screened (CS-08).
|
|
54
|
+
And on a resumed round it refuses a state CSV that was started against a different
|
|
55
|
+
library, rather than silently scoring one library's molecules with another's
|
|
56
|
+
ordering.
|
|
57
|
+
|
|
58
|
+
A pool publishes one pinned library with nothing to score it against, so pools are
|
|
59
|
+
screenable with `supervised` but are not valid targets for `query`.
|
|
60
|
+
|
|
36
61
|
## Development
|
|
37
62
|
|
|
38
63
|
This project uses [Poetry](https://python-poetry.org/) for dependency management and packaging.
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import typer
|
|
2
|
+
import uuid
|
|
3
|
+
import datetime
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import List, Optional
|
|
7
|
+
from rich import print
|
|
8
|
+
from rich.progress import track
|
|
9
|
+
from chemsift_cloud import config
|
|
10
|
+
from chemsift_cloud.register import get_session
|
|
11
|
+
from chemsift_cloud.map import discover_s3_prefixes
|
|
12
|
+
|
|
13
|
+
app = typer.Typer()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@app.command("add-feature-space")
|
|
17
|
+
def add_feature_space(
|
|
18
|
+
register_job_id: str = typer.Argument(help="Register job ID to add a feature space to."),
|
|
19
|
+
feature_space_dir: Path = typer.Argument(
|
|
20
|
+
help="Local directory for the new feature space (must contain metadata.json).",
|
|
21
|
+
exists=True,
|
|
22
|
+
file_okay=False,
|
|
23
|
+
resolve_path=True,
|
|
24
|
+
),
|
|
25
|
+
library: Optional[List[str]] = typer.Option(
|
|
26
|
+
None,
|
|
27
|
+
"--library",
|
|
28
|
+
help="Limit to this library (repeatable). Default: every library in the register job.",
|
|
29
|
+
),
|
|
30
|
+
bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
|
|
31
|
+
table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)"),
|
|
32
|
+
):
|
|
33
|
+
"""
|
|
34
|
+
Add a new feature space to an already-registered library.
|
|
35
|
+
|
|
36
|
+
Featurizes, indexes, clusters, and projects the new space only — the
|
|
37
|
+
register job's other feature spaces and its pre-registration are left
|
|
38
|
+
untouched.
|
|
39
|
+
"""
|
|
40
|
+
cfg = config.load()
|
|
41
|
+
bucket, table_name = config.resolve_target(cfg, bucket, table_name)
|
|
42
|
+
config.show_target(cfg)
|
|
43
|
+
session = get_session(cfg)
|
|
44
|
+
s3 = session.client('s3')
|
|
45
|
+
db = session.resource('dynamodb')
|
|
46
|
+
table = db.Table(table_name)
|
|
47
|
+
identity_id = cfg['identity_id']
|
|
48
|
+
|
|
49
|
+
if not (feature_space_dir / "metadata.json").exists():
|
|
50
|
+
print(f"[red]Error: 'metadata.json' not found in {feature_space_dir}.[/red]")
|
|
51
|
+
raise typer.Exit(code=1)
|
|
52
|
+
fs_name = feature_space_dir.name
|
|
53
|
+
|
|
54
|
+
# Look up source register job
|
|
55
|
+
resp = table.get_item(Key={'user_id': identity_id, 'job_id': register_job_id})
|
|
56
|
+
source_item = resp.get('Item')
|
|
57
|
+
if not source_item:
|
|
58
|
+
print(f"[red]Register job {register_job_id} not found.[/red]")
|
|
59
|
+
raise typer.Exit(code=1)
|
|
60
|
+
|
|
61
|
+
if source_item.get('job_type', 'register') != 'register':
|
|
62
|
+
print(f"[red]Job {register_job_id} is not a register job (type: {source_item.get('job_type')}).[/red]")
|
|
63
|
+
raise typer.Exit(code=1)
|
|
64
|
+
|
|
65
|
+
if source_item.get('status') != 'SUCCEEDED':
|
|
66
|
+
print(f"[yellow]Warning: source job status is '{source_item.get('status')}'. Proceeding anyway.[/yellow]")
|
|
67
|
+
|
|
68
|
+
# Discover libraries from registry_output
|
|
69
|
+
registry_output_prefix = f"users/{identity_id}/{register_job_id}/registry_output/"
|
|
70
|
+
all_libraries = discover_s3_prefixes(s3, bucket, registry_output_prefix)
|
|
71
|
+
if not all_libraries:
|
|
72
|
+
print(f"[red]No libraries found under {registry_output_prefix}. Is the register job complete?[/red]")
|
|
73
|
+
raise typer.Exit(code=1)
|
|
74
|
+
|
|
75
|
+
if library:
|
|
76
|
+
missing = sorted(set(library) - set(all_libraries))
|
|
77
|
+
if missing:
|
|
78
|
+
print(f"[red]Unknown librar{'y' if len(missing) == 1 else 'ies'}: {missing}. "
|
|
79
|
+
f"Available: {all_libraries}[/red]")
|
|
80
|
+
raise typer.Exit(code=1)
|
|
81
|
+
libraries = list(library)
|
|
82
|
+
else:
|
|
83
|
+
libraries = all_libraries
|
|
84
|
+
|
|
85
|
+
# Refuse to silently clobber a feature space that already exists.
|
|
86
|
+
collisions = [
|
|
87
|
+
lib for lib in libraries
|
|
88
|
+
if fs_name in discover_s3_prefixes(s3, bucket, f"{registry_output_prefix}{lib}/standalone/")
|
|
89
|
+
]
|
|
90
|
+
if collisions:
|
|
91
|
+
print(f"[red]Feature space '{fs_name}' already exists for: {collisions}. "
|
|
92
|
+
f"Choose a different name or remove it first.[/red]")
|
|
93
|
+
raise typer.Exit(code=1)
|
|
94
|
+
|
|
95
|
+
job_id = str(uuid.uuid4())
|
|
96
|
+
output_prefix = f"users/{identity_id}/{register_job_id}/registry_output"
|
|
97
|
+
input_prefix = f"users/{identity_id}/{job_id}/add_feature_space_input"
|
|
98
|
+
|
|
99
|
+
# Upload the new feature space's files (metadata.json + any sibling files, e.g. a normalizer)
|
|
100
|
+
files = [
|
|
101
|
+
p for p in feature_space_dir.rglob("*")
|
|
102
|
+
if p.is_file() and p.name != ".DS_Store"
|
|
103
|
+
]
|
|
104
|
+
for file_path in track(files, description=f"Uploading {fs_name}..."):
|
|
105
|
+
relative_path = file_path.relative_to(feature_space_dir)
|
|
106
|
+
s3_key = f"{input_prefix}/feature_spaces/{fs_name}/{relative_path}"
|
|
107
|
+
with open(file_path, 'rb') as f:
|
|
108
|
+
s3.put_object(Bucket=bucket, Key=s3_key, Body=f)
|
|
109
|
+
|
|
110
|
+
# Register in DynamoDB
|
|
111
|
+
table.put_item(Item={
|
|
112
|
+
'user_id': identity_id,
|
|
113
|
+
'job_id': job_id,
|
|
114
|
+
'username': cfg['username'],
|
|
115
|
+
'status': 'SUBMITTED',
|
|
116
|
+
'job_type': 'add_feature_space',
|
|
117
|
+
'source_job_id': register_job_id,
|
|
118
|
+
'created_at': datetime.datetime.utcnow().isoformat(),
|
|
119
|
+
's3_path': f"s3://{bucket}/{output_prefix}",
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
sfn_input = {
|
|
123
|
+
'user_id': identity_id,
|
|
124
|
+
'job_id': job_id,
|
|
125
|
+
's3_bucket': bucket,
|
|
126
|
+
'input_prefix': input_prefix,
|
|
127
|
+
'output_prefix': output_prefix,
|
|
128
|
+
'libraries': [{'name': lib} for lib in libraries],
|
|
129
|
+
'feature_spaces': [{'name': fs_name}],
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
# Upload trigger file — EventBridge watches for this key and invokes the
|
|
133
|
+
# add-feature-space Lambda.
|
|
134
|
+
trigger_key = f"{input_prefix}/add_feature_space.json"
|
|
135
|
+
s3.put_object(
|
|
136
|
+
Bucket=bucket,
|
|
137
|
+
Key=trigger_key,
|
|
138
|
+
Body=json.dumps(sfn_input),
|
|
139
|
+
ContentType="application/json",
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
print(f"[green]Add-feature-space job submitted![/green]")
|
|
143
|
+
print(f"Job ID: [bold]{job_id}[/bold]")
|
|
144
|
+
print(f"Feature space: {fs_name}")
|
|
145
|
+
print(f"Libraries: {libraries}")
|
|
@@ -210,6 +210,25 @@ def resolve_target(cfg: dict, bucket: str | None, table_name: str | None) -> tup
|
|
|
210
210
|
return bucket or cfg["bucket"], table_name or cfg["table"]
|
|
211
211
|
|
|
212
212
|
|
|
213
|
+
def resolve_pool_bucket(cfg: dict, pool_bucket: str | None) -> str:
|
|
214
|
+
"""The bucket holding shared demo pools for the active environment.
|
|
215
|
+
|
|
216
|
+
Separate from `resolve_target` because it is a different bucket with a
|
|
217
|
+
different purpose: pools are read-only published data granted by the pool
|
|
218
|
+
bucket's own policy, while `bucket` is where this user's own job inputs and
|
|
219
|
+
outputs live. Conflating them would send a round's output into a bucket the
|
|
220
|
+
caller has no write on, and the failure would surface inside Batch.
|
|
221
|
+
"""
|
|
222
|
+
resolved = pool_bucket or cfg.get("pool_bucket")
|
|
223
|
+
if not resolved:
|
|
224
|
+
print(
|
|
225
|
+
f"[red][bold]Error[/bold]: environment '{cfg.get('environment', '?')}' declares no "
|
|
226
|
+
f"pool bucket.[/red] Pass [blue]--pool-bucket[/blue] explicitly."
|
|
227
|
+
)
|
|
228
|
+
raise typer.Exit(code=1)
|
|
229
|
+
return resolved
|
|
230
|
+
|
|
231
|
+
|
|
213
232
|
def show_target(cfg: dict) -> None:
|
|
214
233
|
"""One line naming the stack a command is about to act on."""
|
|
215
234
|
env = cfg.get("environment", "?")
|
|
@@ -11,13 +11,13 @@ app = typer.Typer()
|
|
|
11
11
|
|
|
12
12
|
@app.command()
|
|
13
13
|
def download(
|
|
14
|
-
job_id: Annotated[str, typer.Argument(help="ID
|
|
15
|
-
output_dir: Annotated[Path, typer.Argument(help="
|
|
14
|
+
job_id: Annotated[str, typer.Argument(help="Job ID to download.")],
|
|
15
|
+
output_dir: Annotated[Path, typer.Argument(help="Local directory to download into.")],
|
|
16
16
|
bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
|
|
17
17
|
table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)")
|
|
18
18
|
):
|
|
19
19
|
"""
|
|
20
|
-
Download
|
|
20
|
+
Download a job's results.
|
|
21
21
|
"""
|
|
22
22
|
cfg = config.load()
|
|
23
23
|
bucket, table_name = config.resolve_target(cfg, bucket, table_name)
|
|
@@ -39,6 +39,9 @@ def download(
|
|
|
39
39
|
elif job_type == 'map':
|
|
40
40
|
source_job_id = item.get('source_job_id', job_id)
|
|
41
41
|
prefix = f"users/{cfg['identity_id']}/{source_job_id}/registry_output/"
|
|
42
|
+
elif job_type == 'add_feature_space':
|
|
43
|
+
source_job_id = item.get('source_job_id', job_id)
|
|
44
|
+
prefix = f"users/{cfg['identity_id']}/{source_job_id}/registry_output/"
|
|
42
45
|
else:
|
|
43
46
|
# register (default)
|
|
44
47
|
prefix = f"users/{cfg['identity_id']}/{job_id}/registry_output/"
|
|
@@ -37,6 +37,12 @@ ENVIRONMENTS: dict[str, dict] = {
|
|
|
37
37
|
"cognito_domain": "chemsift-register-pipeline-prod.auth.eu-west-1.amazoncognito.com",
|
|
38
38
|
"bucket": "chemsift-register-pipeline",
|
|
39
39
|
"table": "chemsift-jobs",
|
|
40
|
+
# Shared demo pools — libraries registered once that every authenticated
|
|
41
|
+
# user of this stack may screen against. A fourth resource rather than a
|
|
42
|
+
# flag for the same reason the other three live here: a session and the
|
|
43
|
+
# resources it can reach must not drift apart. Per-environment, so a dev
|
|
44
|
+
# pool can be rebuilt without touching what prod's users are screening.
|
|
45
|
+
"pool_bucket": "chemsift-demo-pool",
|
|
40
46
|
},
|
|
41
47
|
"dev": {
|
|
42
48
|
"name": "dev",
|
|
@@ -48,6 +54,7 @@ ENVIRONMENTS: dict[str, dict] = {
|
|
|
48
54
|
"cognito_domain": "chemsift-register-pipeline-dev.auth.eu-west-1.amazoncognito.com",
|
|
49
55
|
"bucket": "chemsift-register-pipeline-dev",
|
|
50
56
|
"table": "chemsift-jobs-dev",
|
|
57
|
+
"pool_bucket": "chemsift-demo-pool-dev",
|
|
51
58
|
},
|
|
52
59
|
}
|
|
53
60
|
|
|
@@ -7,6 +7,8 @@ from chemsift_cloud.register import get_session
|
|
|
7
7
|
|
|
8
8
|
app = typer.Typer()
|
|
9
9
|
|
|
10
|
+
JOBS_BY_CREATED_AT_INDEX = "user_id-created_at-index"
|
|
11
|
+
|
|
10
12
|
def _compact_ts(ts: str) -> str:
|
|
11
13
|
"""Shorten ISO timestamp to 'YYYY-MM-DD HH:MM'."""
|
|
12
14
|
if not ts or ts == 'N/A':
|
|
@@ -16,10 +18,10 @@ def _compact_ts(ts: str) -> str:
|
|
|
16
18
|
@app.command()
|
|
17
19
|
def list(
|
|
18
20
|
table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)"),
|
|
19
|
-
limit: int = typer.Option(50, help="Maximum number of most recent jobs to show")
|
|
21
|
+
limit: int = typer.Option(50, help="Maximum number of most recent jobs to show.")
|
|
20
22
|
):
|
|
21
23
|
"""
|
|
22
|
-
|
|
24
|
+
List your jobs.
|
|
23
25
|
"""
|
|
24
26
|
cfg = config.load()
|
|
25
27
|
_, table_name = config.resolve_target(cfg, None, table_name)
|
|
@@ -30,29 +32,31 @@ def list(
|
|
|
30
32
|
|
|
31
33
|
print(f"Retrieving jobs for {cfg['username']}...")
|
|
32
34
|
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
35
|
+
# Read newest-first straight from the created_at GSI. The table's own sort
|
|
36
|
+
# key is job_id (a random UUID), so querying the table can't answer "most
|
|
37
|
+
# recent N" without reading the whole partition and sorting client-side.
|
|
38
|
+
# Rows without created_at are absent from the (sparse) index — see
|
|
39
|
+
# docs/runbooks/created-at-gsi-rollout.md.
|
|
38
40
|
items = []
|
|
39
41
|
query_kwargs = {
|
|
40
|
-
"
|
|
42
|
+
"IndexName": JOBS_BY_CREATED_AT_INDEX,
|
|
43
|
+
"KeyConditionExpression": boto3.dynamodb.conditions.Key('user_id').eq(cfg['identity_id']),
|
|
44
|
+
"ScanIndexForward": False,
|
|
45
|
+
"Limit": limit,
|
|
41
46
|
}
|
|
42
|
-
while
|
|
47
|
+
while len(items) < limit:
|
|
43
48
|
resp = table.query(**query_kwargs)
|
|
44
49
|
items.extend(resp.get('Items', []))
|
|
45
50
|
last_key = resp.get('LastEvaluatedKey')
|
|
46
51
|
if not last_key:
|
|
47
52
|
break
|
|
48
53
|
query_kwargs['ExclusiveStartKey'] = last_key
|
|
54
|
+
items = items[:limit]
|
|
49
55
|
|
|
50
56
|
if not items:
|
|
51
57
|
print("[yellow]No jobs found.[/yellow]")
|
|
52
58
|
return
|
|
53
59
|
|
|
54
|
-
items = sorted(items, key=lambda x: x.get('created_at', ''), reverse=True)[:limit]
|
|
55
|
-
|
|
56
60
|
table_ui = Table(title=f"Your Jobs (most recent {len(items)})")
|
|
57
61
|
table_ui.add_column("Job ID", style="cyan", no_wrap=True, min_width=36)
|
|
58
62
|
table_ui.add_column("Type", style="yellow")
|
|
@@ -4,6 +4,7 @@ from chemsift_cloud.env import app as env_app
|
|
|
4
4
|
from chemsift_cloud.register import app as register_app
|
|
5
5
|
from chemsift_cloud.map import app as map_app
|
|
6
6
|
from chemsift_cloud.query import app as query_app
|
|
7
|
+
from chemsift_cloud.add_feature_space import app as add_feature_space_app
|
|
7
8
|
from chemsift_cloud.supervised import app as supervised_app
|
|
8
9
|
from chemsift_cloud.status import app as status_app
|
|
9
10
|
from chemsift_cloud.download import app as download_app
|
|
@@ -16,6 +17,7 @@ app.add_typer(env_app)
|
|
|
16
17
|
app.add_typer(register_app)
|
|
17
18
|
app.add_typer(map_app)
|
|
18
19
|
app.add_typer(query_app)
|
|
20
|
+
app.add_typer(add_feature_space_app)
|
|
19
21
|
app.add_typer(supervised_app)
|
|
20
22
|
app.add_typer(status_app)
|
|
21
23
|
app.add_typer(download_app)
|
|
@@ -22,13 +22,13 @@ def discover_s3_prefixes(s3, bucket, prefix):
|
|
|
22
22
|
|
|
23
23
|
@app.command()
|
|
24
24
|
def map(
|
|
25
|
-
register_job_id: str = typer.Argument(help="Register job ID to
|
|
26
|
-
reference_library: str = typer.Argument(help="Name of the reference library
|
|
25
|
+
register_job_id: str = typer.Argument(help="Register job ID to map."),
|
|
26
|
+
reference_library: str = typer.Argument(help="Name of the reference library to map into."),
|
|
27
27
|
bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
|
|
28
28
|
table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)"),
|
|
29
29
|
):
|
|
30
30
|
"""
|
|
31
|
-
|
|
31
|
+
Map a registered library into a reference library's space.
|
|
32
32
|
"""
|
|
33
33
|
cfg = config.load()
|
|
34
34
|
bucket, table_name = config.resolve_target(cfg, bucket, table_name)
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""Shared demo pools: resolving one, and refusing to screen it incoherently.
|
|
2
|
+
|
|
3
|
+
A pool is a library registered once that every authenticated user of a stack may
|
|
4
|
+
screen against, so a new account does not have to register a library before it can
|
|
5
|
+
run anything. It is read-only; everything a round writes goes to the caller's own
|
|
6
|
+
prefix.
|
|
7
|
+
|
|
8
|
+
This module is deliberately free of `typer` — it raises `PoolError` and lets the
|
|
9
|
+
command layer turn that into output and an exit code. Two reasons, and the second
|
|
10
|
+
is the one that matters: presentation is not policy, and a module that imports
|
|
11
|
+
typer cannot be tested from the pipeline's own environment, which does not have it
|
|
12
|
+
(`cli/` is a separate Poetry package). The same split is why
|
|
13
|
+
`chemsift_cloud/standardization.py` is testable from `test/invariants/`.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import csv
|
|
19
|
+
import json
|
|
20
|
+
import tempfile
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
import yaml as pyyaml
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class PoolError(Exception):
|
|
27
|
+
"""Something about the pool, the state file, or the config does not add up."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# ---------------------------------------------------------------------------
|
|
31
|
+
# State CSV
|
|
32
|
+
# ---------------------------------------------------------------------------
|
|
33
|
+
#
|
|
34
|
+
# csv, not pandas. pandas is NOT a declared dependency of this package
|
|
35
|
+
# (cli/pyproject.toml lists typer, urllib3, requests, boto3, rich, pyyaml,
|
|
36
|
+
# typing-extensions), so the `import pandas` the supervised command used to do for
|
|
37
|
+
# a single max() crashed `--state` on a clean `pip install chemsift`. Nothing here
|
|
38
|
+
# needs a dataframe.
|
|
39
|
+
|
|
40
|
+
REQUIRED_STATE_COLUMNS = ("round", "idx", "smiles")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def read_state(path: Path) -> list[dict]:
|
|
44
|
+
with open(path, newline="", encoding="utf-8") as fh:
|
|
45
|
+
rows = list(csv.DictReader(fh))
|
|
46
|
+
if not rows:
|
|
47
|
+
raise PoolError(f"{path} has no rows.")
|
|
48
|
+
missing = [c for c in REQUIRED_STATE_COLUMNS if c not in rows[0]]
|
|
49
|
+
if missing:
|
|
50
|
+
raise PoolError(
|
|
51
|
+
f"{path} has no {', '.join(repr(m) for m in missing)} column(s) — "
|
|
52
|
+
f"is this a supervised state CSV?"
|
|
53
|
+
)
|
|
54
|
+
return rows
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def next_round(rows: list[dict]) -> int:
|
|
58
|
+
values = [r["round"] for r in rows if r.get("round", "") != ""]
|
|
59
|
+
if not values:
|
|
60
|
+
raise PoolError("The 'round' column is empty in every row.")
|
|
61
|
+
try:
|
|
62
|
+
return max(int(float(v)) for v in values) + 1
|
|
63
|
+
except ValueError:
|
|
64
|
+
raise PoolError("The 'round' column holds a value that is not a number.")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def read_library_smiles(s3, bucket: str, prefix: str, library: str) -> list[str]:
|
|
68
|
+
"""The library's molecules in row order — the definition of `idx`.
|
|
69
|
+
|
|
70
|
+
`mols.smiles` is a one-column CSV *with a header*, read here the way the
|
|
71
|
+
pipeline reads it (`library.py`). Counting lines instead would shift every
|
|
72
|
+
index by one and attach every label to the next molecule along.
|
|
73
|
+
"""
|
|
74
|
+
key = f"{prefix.rstrip('/')}/{library}/mols.smiles"
|
|
75
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
76
|
+
local = Path(tmp) / "mols.smiles"
|
|
77
|
+
try:
|
|
78
|
+
s3.download_file(bucket, key, str(local))
|
|
79
|
+
except Exception as exc:
|
|
80
|
+
raise PoolError(
|
|
81
|
+
f"Could not read s3://{bucket}/{key} to verify the state file "
|
|
82
|
+
f"({type(exc).__name__})."
|
|
83
|
+
)
|
|
84
|
+
with open(local, newline="", encoding="utf-8") as fh:
|
|
85
|
+
return [r["smiles"] for r in csv.DictReader(fh)]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def state_library_mismatches(rows: list[dict], library_smiles: list[str],
|
|
89
|
+
limit: int = 5) -> list[str]:
|
|
90
|
+
"""Rows whose `idx` does not denote the molecule they claim.
|
|
91
|
+
|
|
92
|
+
`idx` is a row offset into the library's `mols.smiles`, so a state produced
|
|
93
|
+
against one registry is meaningless against another — and both failure modes
|
|
94
|
+
are silent. Resume a campaign against a different pool, or against a library
|
|
95
|
+
re-registered in a different order, and every label attaches to the wrong
|
|
96
|
+
structure while the job succeeds and the hit rates stay plausible. That is
|
|
97
|
+
CS-03/CS-08 at the submission boundary.
|
|
98
|
+
|
|
99
|
+
The check needs no new metadata, because the state already carries both halves
|
|
100
|
+
of the claim: the `idx` and the `smiles` it is supposed to denote. Comparing
|
|
101
|
+
them verifies the meaning rather than a label for it, which is why this is
|
|
102
|
+
preferred over matching a recorded digest.
|
|
103
|
+
"""
|
|
104
|
+
out: list[str] = []
|
|
105
|
+
for row in rows:
|
|
106
|
+
raw = row.get("idx", "")
|
|
107
|
+
if raw == "":
|
|
108
|
+
continue
|
|
109
|
+
claimed = row.get("smiles", "")
|
|
110
|
+
try:
|
|
111
|
+
idx = int(float(raw))
|
|
112
|
+
except ValueError:
|
|
113
|
+
out.append(f"idx {raw!r}: not a number")
|
|
114
|
+
else:
|
|
115
|
+
if not 0 <= idx < len(library_smiles):
|
|
116
|
+
out.append(f"idx {idx}: out of range for a "
|
|
117
|
+
f"{len(library_smiles)}-molecule library")
|
|
118
|
+
elif library_smiles[idx] != claimed:
|
|
119
|
+
out.append(f"idx {idx}: state says {claimed!r}, "
|
|
120
|
+
f"library holds {library_smiles[idx]!r}")
|
|
121
|
+
if len(out) >= limit:
|
|
122
|
+
break
|
|
123
|
+
return out
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def verify_state_matches_library(s3, bucket: str, prefix: str, library: str,
|
|
127
|
+
rows: list[dict]) -> None:
|
|
128
|
+
mismatches = state_library_mismatches(rows, read_library_smiles(s3, bucket, prefix, library))
|
|
129
|
+
if mismatches:
|
|
130
|
+
raise PoolError(
|
|
131
|
+
"This state file does not belong to that library:\n "
|
|
132
|
+
+ "\n ".join(mismatches)
|
|
133
|
+
+ f"\n `idx` is a row offset into {library}/mols.smiles, so submitting this would "
|
|
134
|
+
f"train on labels attached to the wrong molecules (CS-08), and nothing would fail."
|
|
135
|
+
f"\n Use the library the campaign was started against, or start a new campaign."
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# ---------------------------------------------------------------------------
|
|
140
|
+
# The pool itself
|
|
141
|
+
# ---------------------------------------------------------------------------
|
|
142
|
+
|
|
143
|
+
def load_manifest(s3, pool_bucket: str, pool_id: str) -> dict:
|
|
144
|
+
"""Read a pool's manifest, which doubles as the authorization check.
|
|
145
|
+
|
|
146
|
+
This replaces the DynamoDB `get_item` the register-job path uses, and proves
|
|
147
|
+
strictly more: that row proved a record existed in the caller's own partition,
|
|
148
|
+
while reading `pool.json` proves the pool exists AND that these credentials
|
|
149
|
+
carry the pool bucket's read grant. Without it, the first sign of a missing
|
|
150
|
+
grant is a download failure inside Batch, minutes later, in a log nobody is
|
|
151
|
+
watching.
|
|
152
|
+
"""
|
|
153
|
+
key = f"{pool_id}/pool.json"
|
|
154
|
+
try:
|
|
155
|
+
body = s3.get_object(Bucket=pool_bucket, Key=key)["Body"].read()
|
|
156
|
+
except Exception as exc:
|
|
157
|
+
raise PoolError(
|
|
158
|
+
f"Pool {pool_id!r} is not readable at s3://{pool_bucket}/{key} "
|
|
159
|
+
f"({type(exc).__name__}).\n"
|
|
160
|
+
f" Either the pool id is wrong, or this environment's users have no read grant "
|
|
161
|
+
f"on {pool_bucket}."
|
|
162
|
+
)
|
|
163
|
+
try:
|
|
164
|
+
manifest = json.loads(body)
|
|
165
|
+
except json.JSONDecodeError as exc:
|
|
166
|
+
raise PoolError(f"s3://{pool_bucket}/{key} is not valid JSON: {exc}")
|
|
167
|
+
if not manifest.get("library"):
|
|
168
|
+
raise PoolError(f"s3://{pool_bucket}/{key} declares no 'library'.")
|
|
169
|
+
return manifest
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def config_naming_library(yaml_path: Path, library: str, tmpdir: str) -> Path:
|
|
173
|
+
"""The config, with `library:` filled in from the pool if it was absent.
|
|
174
|
+
|
|
175
|
+
`select_pool_library` refuses to guess when a registry holds more than one
|
|
176
|
+
library, because the state file's `idx` is an offset into whichever one is
|
|
177
|
+
chosen and a different pick between rounds silently relabels every molecule
|
|
178
|
+
(CS-08). A pool's manifest already names its library, so writing it into the
|
|
179
|
+
config makes the choice a recorded fact rather than relying on there happening
|
|
180
|
+
to be a single candidate.
|
|
181
|
+
"""
|
|
182
|
+
with open(yaml_path, encoding="utf-8") as fh:
|
|
183
|
+
cfg = pyyaml.safe_load(fh) or {}
|
|
184
|
+
declared = cfg.get("library")
|
|
185
|
+
if declared and declared != library:
|
|
186
|
+
raise PoolError(
|
|
187
|
+
f"The config names library {declared!r} but this pool holds {library!r}."
|
|
188
|
+
)
|
|
189
|
+
if declared:
|
|
190
|
+
return yaml_path
|
|
191
|
+
cfg["library"] = library
|
|
192
|
+
out = Path(tmpdir) / "config.yaml"
|
|
193
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
194
|
+
with open(out, "w", encoding="utf-8") as fh:
|
|
195
|
+
pyyaml.safe_dump(cfg, fh, sort_keys=False)
|
|
196
|
+
return out
|
|
@@ -7,6 +7,10 @@ from rich import print
|
|
|
7
7
|
from rich.progress import track
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
from chemsift_cloud import config
|
|
10
|
+
from chemsift_cloud.standardization import (
|
|
11
|
+
PolicyConflict,
|
|
12
|
+
inherited_standardization,
|
|
13
|
+
)
|
|
10
14
|
from chemsift_cloud.register import get_session
|
|
11
15
|
|
|
12
16
|
app = typer.Typer()
|
|
@@ -25,7 +29,7 @@ def discover_s3_prefixes(s3, bucket, prefix):
|
|
|
25
29
|
|
|
26
30
|
@app.command()
|
|
27
31
|
def query(
|
|
28
|
-
register_job_id: str = typer.Argument(help="Register job ID
|
|
32
|
+
register_job_id: str = typer.Argument(help="Register job ID to query."),
|
|
29
33
|
molecules: Annotated[Path, typer.Option(
|
|
30
34
|
"--molecules", help="Local path to query molecules CSV",
|
|
31
35
|
exists=True, file_okay=True, dir_okay=False, resolve_path=True
|
|
@@ -40,7 +44,7 @@ def query(
|
|
|
40
44
|
precalculated_library_prefix: str = typer.Option(None, "--precalculated-library-prefix", help="S3 prefix of a single registered library directory (e.g. .../registry_output/<library_name>) whose precomputed features are used for the query molecules, skipping featurization"),
|
|
41
45
|
):
|
|
42
46
|
"""
|
|
43
|
-
Submit a query against a
|
|
47
|
+
Submit a rule-based query against a registered library.
|
|
44
48
|
"""
|
|
45
49
|
cfg = config.load()
|
|
46
50
|
bucket, table_name = config.resolve_target(cfg, bucket, table_name)
|
|
@@ -74,26 +78,15 @@ def query(
|
|
|
74
78
|
yaml_key = f"users/{identity_id}/{job_id}/query_input/query.yml"
|
|
75
79
|
library_metadata_key = f"users/{identity_id}/{job_id}/query_input/libraries/query_molecules/metadata.json"
|
|
76
80
|
|
|
77
|
-
library_metadata = {
|
|
78
|
-
"name": "query_molecules",
|
|
79
|
-
"structure_file": "molecules.csv",
|
|
80
|
-
"smiles_column": "smiles",
|
|
81
|
-
"is_reference": False
|
|
82
|
-
}
|
|
83
81
|
|
|
84
|
-
print(f"Uploading query inputs for job [blue]{job_id}[/blue]...")
|
|
85
82
|
import json as _json
|
|
83
|
+
|
|
84
|
+
print(f"Uploading query inputs for job [blue]{job_id}[/blue]...")
|
|
86
85
|
for local_path, s3_key in track(
|
|
87
86
|
[(str(molecules), molecules_key), (str(yaml), yaml_key)],
|
|
88
87
|
description="Uploading..."
|
|
89
88
|
):
|
|
90
89
|
s3.upload_file(local_path, bucket, s3_key)
|
|
91
|
-
s3.put_object(
|
|
92
|
-
Bucket=bucket,
|
|
93
|
-
Key=library_metadata_key,
|
|
94
|
-
Body=_json.dumps(library_metadata),
|
|
95
|
-
ContentType="application/json",
|
|
96
|
-
)
|
|
97
90
|
|
|
98
91
|
# Discover feature_spaces from local query YAML
|
|
99
92
|
import yaml as pyyaml
|
|
@@ -108,6 +101,42 @@ def query(
|
|
|
108
101
|
|
|
109
102
|
print(f"Discovered feature spaces from query YAML: {feature_spaces}")
|
|
110
103
|
|
|
104
|
+
# CS-83: the query molecules are registered as their OWN library, and the
|
|
105
|
+
# query job then slices its features instead of featurizing here. So that
|
|
106
|
+
# library has to be registered under the same standardization policy as the
|
|
107
|
+
# libraries the rules name (CS-78) — otherwise its vectors are built from
|
|
108
|
+
# different molecules than the index they are searched against, and
|
|
109
|
+
# run_query_pipeline refuses the job rather than scoring it.
|
|
110
|
+
#
|
|
111
|
+
# The policy is read from standardization.json, which registration writes
|
|
112
|
+
# beside mols.smiles and which records what ACTUALLY ran, not merely what
|
|
113
|
+
# was declared.
|
|
114
|
+
try:
|
|
115
|
+
standardization = inherited_standardization(
|
|
116
|
+
s3.get_object, bucket,
|
|
117
|
+
f"users/{identity_id}/{register_job_id}/registry_output", rules)
|
|
118
|
+
except PolicyConflict as exc:
|
|
119
|
+
print(f"[red]{exc}[/red]")
|
|
120
|
+
raise typer.Exit(code=1)
|
|
121
|
+
if standardization:
|
|
122
|
+
print(f"Inheriting standardization from the reference library: "
|
|
123
|
+
f"[blue]{standardization}[/blue]")
|
|
124
|
+
|
|
125
|
+
library_metadata = {
|
|
126
|
+
"name": "query_molecules",
|
|
127
|
+
"structure_file": "molecules.csv",
|
|
128
|
+
"smiles_column": "smiles",
|
|
129
|
+
"is_reference": False,
|
|
130
|
+
}
|
|
131
|
+
if standardization:
|
|
132
|
+
library_metadata["standardization"] = standardization
|
|
133
|
+
s3.put_object(
|
|
134
|
+
Bucket=bucket,
|
|
135
|
+
Key=library_metadata_key,
|
|
136
|
+
Body=_json.dumps(library_metadata),
|
|
137
|
+
ContentType="application/json",
|
|
138
|
+
)
|
|
139
|
+
|
|
111
140
|
output_prefix = f"users/{identity_id}/{job_id}/query_output"
|
|
112
141
|
|
|
113
142
|
# Register in DynamoDB
|
|
@@ -38,7 +38,7 @@ def get_session(cfg):
|
|
|
38
38
|
@app.command()
|
|
39
39
|
def register(
|
|
40
40
|
input_dir: Path = typer.Argument(
|
|
41
|
-
help="
|
|
41
|
+
help="Local registry_input folder (or its parent) — libraries/ and feature_spaces/.",
|
|
42
42
|
exists=True,
|
|
43
43
|
file_okay=False,
|
|
44
44
|
resolve_path=True
|
|
@@ -47,7 +47,7 @@ def register(
|
|
|
47
47
|
table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)")
|
|
48
48
|
):
|
|
49
49
|
"""
|
|
50
|
-
Submit
|
|
50
|
+
Submit a registration job for a library and its feature spaces.
|
|
51
51
|
"""
|
|
52
52
|
cfg = config.load()
|
|
53
53
|
bucket, table_name = config.resolve_target(cfg, bucket, table_name)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Which standardization policy a cloud query must encode under (CS-83).
|
|
2
|
+
|
|
3
|
+
Stdlib only, and deliberately free of typer/rich/boto3 imports, so the invariant
|
|
4
|
+
suite in the core repo can exercise it without installing the CLI's dependency
|
|
5
|
+
tree. The CLI translates the error into a typer exit; the rule itself lives here.
|
|
6
|
+
|
|
7
|
+
Why this exists at all. In the cloud the query molecules are uploaded as their
|
|
8
|
+
OWN library, registered and featurized by Step Functions, and the query job then
|
|
9
|
+
SLICES those features rather than featurizing anything itself. So the query's
|
|
10
|
+
vectors are only comparable with the index they are searched against if that
|
|
11
|
+
library was registered under the same standardization policy (CS-78).
|
|
12
|
+
|
|
13
|
+
The CLI used to hardcode that library's metadata with no policy at all, so any
|
|
14
|
+
reference library declaring one became unqueryable: `run_query_pipeline` saw
|
|
15
|
+
`None` on the query side and `chembl_parent_largest` on the reference side and
|
|
16
|
+
refused the job — correctly, since the query's fingerprints would have carried
|
|
17
|
+
counterions the index has never seen. Found on dev, not locally, because the
|
|
18
|
+
LOCAL query path featurizes the query itself and so never takes this branch.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import json
|
|
24
|
+
|
|
25
|
+
#: Written beside mols.smiles by registration. Read rather than metadata.json
|
|
26
|
+
#: because it records the policy that ACTUALLY ran, not the one declared.
|
|
27
|
+
RECORD_NAME = "standardization.json"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class PolicyConflict(ValueError):
|
|
31
|
+
"""The libraries a rule set names were registered under different policies."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def library_names(rules) -> list[str]:
|
|
35
|
+
"""The libraries a query's rules score against, deduplicated and ordered."""
|
|
36
|
+
return sorted({r["library"] for r in (rules or []) if r.get("library")})
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def read_policy(get_object, bucket: str, prefix: str, library: str):
|
|
40
|
+
"""The policy `library` was registered under, or None.
|
|
41
|
+
|
|
42
|
+
`get_object` is the boto3 client method, injected so this stays testable and
|
|
43
|
+
so a missing object is an ordinary None rather than a dependency on botocore
|
|
44
|
+
exception classes. A library registered before CS-78 has no record, and that
|
|
45
|
+
genuinely means no policy — not an error.
|
|
46
|
+
"""
|
|
47
|
+
key = f"{prefix.rstrip('/')}/{library}/{RECORD_NAME}"
|
|
48
|
+
try:
|
|
49
|
+
body = get_object(Bucket=bucket, Key=key)["Body"].read()
|
|
50
|
+
except Exception as exc:
|
|
51
|
+
# ONLY a missing object means "registered before CS-78, so no policy".
|
|
52
|
+
# A bare `except Exception` here turned an AccessDenied or a throttle
|
|
53
|
+
# into None, which registers the query library unstandardized and fails
|
|
54
|
+
# the job in Batch minutes later — precisely the outcome CS-83 exists to
|
|
55
|
+
# prevent, reached by the error handling rather than the logic.
|
|
56
|
+
name = type(exc).__name__
|
|
57
|
+
code = getattr(getattr(exc, "response", None), "get", lambda *_: {})("Error", {})
|
|
58
|
+
if name in ("NoSuchKey", "KeyError", "FileNotFoundError", "404") or \
|
|
59
|
+
(isinstance(code, dict) and code.get("Code") in ("NoSuchKey", "404", "NotFound")):
|
|
60
|
+
return None
|
|
61
|
+
raise
|
|
62
|
+
try:
|
|
63
|
+
return json.loads(body).get("policy")
|
|
64
|
+
except (ValueError, TypeError, AttributeError):
|
|
65
|
+
return None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def inherited_standardization(get_object, bucket: str, prefix: str, rules):
|
|
69
|
+
"""The single policy the query-molecules library must be registered under.
|
|
70
|
+
|
|
71
|
+
Raises PolicyConflict when the rule set names libraries registered under
|
|
72
|
+
different policies. Refusing here rather than picking one is the same
|
|
73
|
+
judgement run_query_pipeline makes server-side: a query is encoded once, so
|
|
74
|
+
choosing either policy silently mis-encodes it against the other half of the
|
|
75
|
+
rule set. Failing in the CLI is strictly better than failing in Batch five
|
|
76
|
+
minutes later, and says the same thing.
|
|
77
|
+
"""
|
|
78
|
+
policies = {lib: read_policy(get_object, bucket, prefix, lib)
|
|
79
|
+
for lib in library_names(rules)}
|
|
80
|
+
distinct = set(policies.values())
|
|
81
|
+
if len(distinct) > 1:
|
|
82
|
+
detail = ", ".join(f"{lib}={pol!r}" for lib, pol in sorted(policies.items()))
|
|
83
|
+
raise PolicyConflict(
|
|
84
|
+
f"the libraries this query names were registered under different "
|
|
85
|
+
f"standardization policies ({detail}). A query is encoded once and scored "
|
|
86
|
+
f"against all of them, so they must agree — query them separately, or "
|
|
87
|
+
f"re-register the odd one out (CS-83)."
|
|
88
|
+
)
|
|
89
|
+
return next(iter(distinct), None)
|
|
@@ -11,11 +11,11 @@ app = typer.Typer()
|
|
|
11
11
|
|
|
12
12
|
@app.command()
|
|
13
13
|
def status(
|
|
14
|
-
job_id: Annotated[str, typer.Argument(help="ID
|
|
14
|
+
job_id: Annotated[str, typer.Argument(help="Job ID to check.")],
|
|
15
15
|
table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)")
|
|
16
16
|
):
|
|
17
17
|
"""
|
|
18
|
-
Check
|
|
18
|
+
Check a job's status.
|
|
19
19
|
"""
|
|
20
20
|
cfg = config.load()
|
|
21
21
|
_, table_name = config.resolve_target(cfg, None, table_name)
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import json
|
|
3
|
+
import tempfile
|
|
4
|
+
import uuid
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from rich import print
|
|
10
|
+
from rich.panel import Panel
|
|
11
|
+
from typing_extensions import Annotated
|
|
12
|
+
|
|
13
|
+
from chemsift_cloud import config, pool as poollib
|
|
14
|
+
from chemsift_cloud.pool import PoolError
|
|
15
|
+
from chemsift_cloud.register import get_session
|
|
16
|
+
|
|
17
|
+
app = typer.Typer()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _or_exit(fn, *args, **kwargs):
|
|
21
|
+
"""Run a pool-module call, turning its PoolError into CLI output.
|
|
22
|
+
|
|
23
|
+
The policy lives in chemsift_cloud.pool, which raises; presentation lives
|
|
24
|
+
here. That split is what lets the checks be tested from the pipeline's own
|
|
25
|
+
environment, which has no typer.
|
|
26
|
+
"""
|
|
27
|
+
try:
|
|
28
|
+
return fn(*args, **kwargs)
|
|
29
|
+
except PoolError as exc:
|
|
30
|
+
print(f"[red]{exc}[/red]")
|
|
31
|
+
raise typer.Exit(code=1)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@app.command()
|
|
35
|
+
def supervised(
|
|
36
|
+
register_job_id: Annotated[Optional[str], typer.Argument(
|
|
37
|
+
help="Register job ID whose library is the active-learning pool. "
|
|
38
|
+
"Omit when using --pool."
|
|
39
|
+
)] = None,
|
|
40
|
+
yaml: Annotated[Path, typer.Option(
|
|
41
|
+
"--yaml", help="Local path to supervised query YAML config.",
|
|
42
|
+
exists=True, file_okay=True, dir_okay=False, resolve_path=True,
|
|
43
|
+
)] = None,
|
|
44
|
+
state: Annotated[Optional[Path], typer.Option(
|
|
45
|
+
"--state",
|
|
46
|
+
help="Local state CSV. Omit on round 0. On subsequent rounds pass the CSV "
|
|
47
|
+
"with the 'label' column filled in (1 active, 0 inactive, -1 tested "
|
|
48
|
+
"but inconclusive).",
|
|
49
|
+
file_okay=True, dir_okay=False, resolve_path=True,
|
|
50
|
+
)] = None,
|
|
51
|
+
pool: Annotated[Optional[str], typer.Option(
|
|
52
|
+
"--pool",
|
|
53
|
+
help="Shared demo pool id to screen against (e.g. ecbd-luciferase-v1), "
|
|
54
|
+
"instead of one of your own register jobs.",
|
|
55
|
+
)] = None,
|
|
56
|
+
pool_bucket: str = typer.Option(None, help="Pool bucket (default: active environment)"),
|
|
57
|
+
bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
|
|
58
|
+
table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)"),
|
|
59
|
+
):
|
|
60
|
+
"""
|
|
61
|
+
Submit one round of a supervised query (active-learning screening).
|
|
62
|
+
|
|
63
|
+
Screens either one of your own registered libraries, or a shared demo pool:
|
|
64
|
+
|
|
65
|
+
chemsift supervised <register_job_id> --yaml config.yaml
|
|
66
|
+
chemsift supervised --pool ecbd-luciferase-v1 --yaml config.yaml
|
|
67
|
+
|
|
68
|
+
A pool is a library registered once that every authenticated user may screen
|
|
69
|
+
against, so a new account does not have to register a library before it can
|
|
70
|
+
run anything. It is READ-ONLY: everything this command writes — the config,
|
|
71
|
+
the state, the round's output — goes to your own prefix, exactly as it does
|
|
72
|
+
for your own registrations.
|
|
73
|
+
|
|
74
|
+
Round 0 (first call — no --state): randomly samples n_samples molecules and
|
|
75
|
+
writes them to S3 with an empty 'label' column.
|
|
76
|
+
|
|
77
|
+
Subsequent rounds: upload the state CSV with labels filled in, and the model
|
|
78
|
+
selects the next batch using the RF/NB/ChemSift ensemble.
|
|
79
|
+
|
|
80
|
+
After each round:
|
|
81
|
+
1. chemsift status <job_id> — wait for SUCCEEDED
|
|
82
|
+
2. chemsift download <job_id> . — downloads state.csv with the new round
|
|
83
|
+
3. Fill the 'label' column (1 = active, 0 = inactive,
|
|
84
|
+
-1 = tested but inconclusive — never retested, never trains the model)
|
|
85
|
+
4. Re-run with --state state.csv
|
|
86
|
+
"""
|
|
87
|
+
if yaml is None:
|
|
88
|
+
print("[red]--yaml is required.[/red]")
|
|
89
|
+
raise typer.Exit(code=1)
|
|
90
|
+
# Exactly one source. Accepting both would leave the prefix ambiguous, and
|
|
91
|
+
# accepting neither used to be impossible only because the argument was
|
|
92
|
+
# required.
|
|
93
|
+
if bool(register_job_id) == bool(pool):
|
|
94
|
+
print("[red]Pass exactly one of <register_job_id> or --pool.[/red]")
|
|
95
|
+
raise typer.Exit(code=1)
|
|
96
|
+
|
|
97
|
+
cfg = config.load()
|
|
98
|
+
bucket, table_name = config.resolve_target(cfg, bucket, table_name)
|
|
99
|
+
config.show_target(cfg)
|
|
100
|
+
session = get_session(cfg)
|
|
101
|
+
s3 = session.client("s3")
|
|
102
|
+
db = session.resource("dynamodb")
|
|
103
|
+
table = db.Table(table_name)
|
|
104
|
+
|
|
105
|
+
identity_id = cfg["identity_id"]
|
|
106
|
+
job_id = str(uuid.uuid4())
|
|
107
|
+
|
|
108
|
+
rows = _or_exit(poollib.read_state, state) if state and Path(state).exists() else None
|
|
109
|
+
round_number = _or_exit(poollib.next_round, rows) if rows else 0
|
|
110
|
+
|
|
111
|
+
# ------------------------------------------------------------------
|
|
112
|
+
# Resolve what is being screened: a shared pool, or one of your own
|
|
113
|
+
# register jobs. These differ in WHERE the library is read from and in how
|
|
114
|
+
# the right to read it is established — nothing else.
|
|
115
|
+
# ------------------------------------------------------------------
|
|
116
|
+
source_bucket = bucket # the library's bucket; same as `bucket` unless pooled
|
|
117
|
+
manifest: dict = {}
|
|
118
|
+
if pool:
|
|
119
|
+
source_bucket = config.resolve_pool_bucket(cfg, pool_bucket)
|
|
120
|
+
manifest = _or_exit(poollib.load_manifest, s3, source_bucket, pool)
|
|
121
|
+
library = manifest["library"]
|
|
122
|
+
registry_output_prefix = f"{pool}/registry_output"
|
|
123
|
+
source_label = f"pool {pool} ({library}, {manifest.get('pool_size', '?')} molecules)"
|
|
124
|
+
else:
|
|
125
|
+
resp = table.get_item(Key={"user_id": identity_id, "job_id": register_job_id})
|
|
126
|
+
source = resp.get("Item")
|
|
127
|
+
if not source:
|
|
128
|
+
print(f"[red]Register job {register_job_id} not found.[/red]")
|
|
129
|
+
raise typer.Exit(code=1)
|
|
130
|
+
if source.get("status") != "SUCCEEDED":
|
|
131
|
+
print(f"[yellow]Warning: register job status is '{source.get('status')}'. "
|
|
132
|
+
"Pool library may be incomplete.[/yellow]")
|
|
133
|
+
library = None
|
|
134
|
+
registry_output_prefix = f"users/{identity_id}/{register_job_id}/registry_output"
|
|
135
|
+
source_label = register_job_id
|
|
136
|
+
|
|
137
|
+
input_prefix = f"users/{identity_id}/{job_id}/supervised_input"
|
|
138
|
+
output_prefix = f"users/{identity_id}/{job_id}/supervised_output"
|
|
139
|
+
|
|
140
|
+
# A resumed campaign must be resumed against the library it was started on.
|
|
141
|
+
if rows and library:
|
|
142
|
+
_or_exit(poollib.verify_state_matches_library,
|
|
143
|
+
s3, source_bucket, registry_output_prefix, library, rows)
|
|
144
|
+
|
|
145
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
146
|
+
# Upload YAML — naming the pool's library when the config did not (CS-08)
|
|
147
|
+
upload_yaml = (_or_exit(poollib.config_naming_library, yaml, library, tmp)
|
|
148
|
+
if library else yaml)
|
|
149
|
+
yaml_key = f"{input_prefix}/config.yaml"
|
|
150
|
+
s3.upload_file(str(upload_yaml), bucket, yaml_key)
|
|
151
|
+
|
|
152
|
+
# Upload state CSV if present (rounds > 0)
|
|
153
|
+
state_key = ""
|
|
154
|
+
if rows:
|
|
155
|
+
state_key = f"{input_prefix}/state.csv"
|
|
156
|
+
s3.upload_file(str(state), bucket, state_key)
|
|
157
|
+
|
|
158
|
+
# Register in DynamoDB before uploading trigger (state machine will update to RUNNING)
|
|
159
|
+
item = {
|
|
160
|
+
"user_id": identity_id,
|
|
161
|
+
"job_id": job_id,
|
|
162
|
+
"username": cfg["username"],
|
|
163
|
+
"status": "SUBMITTED",
|
|
164
|
+
"job_type": "supervised_query",
|
|
165
|
+
"round_number": round_number,
|
|
166
|
+
"created_at": datetime.datetime.utcnow().isoformat(),
|
|
167
|
+
"s3_path": f"s3://{bucket}/{output_prefix}",
|
|
168
|
+
}
|
|
169
|
+
if pool:
|
|
170
|
+
# No source_job_id: there is no register job in THIS user's partition to
|
|
171
|
+
# point at, and inventing one would make a pooled campaign look like it
|
|
172
|
+
# came from a registration the user owns.
|
|
173
|
+
item["source_pool"] = pool
|
|
174
|
+
item["source_pool_bucket"] = source_bucket
|
|
175
|
+
if (manifest.get("source") or {}).get("registration_job_id"):
|
|
176
|
+
item["source_registration"] = manifest["source"]["registration_job_id"]
|
|
177
|
+
else:
|
|
178
|
+
item["source_job_id"] = register_job_id
|
|
179
|
+
table.put_item(Item=item)
|
|
180
|
+
|
|
181
|
+
# Upload trigger file last — EventBridge watches for this key and invokes the Lambda
|
|
182
|
+
trigger = {
|
|
183
|
+
"user_id": identity_id,
|
|
184
|
+
"job_id": job_id,
|
|
185
|
+
"s3_bucket": bucket,
|
|
186
|
+
# Where the library is READ from. The Lambda defaults this to s3_bucket
|
|
187
|
+
# when absent, so an older client keeps working; sending it explicitly is
|
|
188
|
+
# what lets the pool live in its own read-only bucket.
|
|
189
|
+
"registry_output_bucket": source_bucket,
|
|
190
|
+
"registry_output_prefix": registry_output_prefix,
|
|
191
|
+
"yaml_key": yaml_key,
|
|
192
|
+
"state_key": state_key,
|
|
193
|
+
"output_prefix": output_prefix,
|
|
194
|
+
}
|
|
195
|
+
s3.put_object(
|
|
196
|
+
Bucket=bucket,
|
|
197
|
+
Key=f"{input_prefix}/supervised.json",
|
|
198
|
+
Body=json.dumps(trigger),
|
|
199
|
+
ContentType="application/json",
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
resubmit = (f"chemsift supervised --pool {pool} --yaml {yaml.name} --state state.csv"
|
|
203
|
+
if pool else
|
|
204
|
+
f"chemsift supervised {register_job_id} --yaml {yaml.name} --state state.csv")
|
|
205
|
+
print(Panel(
|
|
206
|
+
f"[bold]Job ID:[/bold] {job_id}\n"
|
|
207
|
+
f"[bold]Round:[/bold] {round_number}\n"
|
|
208
|
+
f"[bold]Screening:[/bold] {source_label}\n"
|
|
209
|
+
f"[bold]Reading from:[/bold] s3://{source_bucket}/{registry_output_prefix}\n\n"
|
|
210
|
+
f"[bold]Next steps:[/bold]\n"
|
|
211
|
+
f" chemsift status {job_id}\n"
|
|
212
|
+
f" chemsift download {job_id} .\n"
|
|
213
|
+
f" # fill 'label' column in state.csv (1=active, 0=inactive, -1=inconclusive)\n"
|
|
214
|
+
f" {resubmit}",
|
|
215
|
+
title="[green]Supervised round submitted[/green]",
|
|
216
|
+
expand=False,
|
|
217
|
+
))
|
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
import typer
|
|
2
|
-
from rich import print
|
|
3
|
-
from rich.progress import track
|
|
4
|
-
from chemsift_cloud import config
|
|
5
|
-
from chemsift_cloud.register import get_session
|
|
6
|
-
from pathlib import Path
|
|
7
|
-
import uuid
|
|
8
|
-
import datetime
|
|
9
|
-
|
|
10
|
-
app = typer.Typer()
|
|
11
|
-
|
|
12
|
-
@app.command()
|
|
13
|
-
def submit(
|
|
14
|
-
input_dir: Path = typer.Argument(
|
|
15
|
-
help="Directory to submit for processing",
|
|
16
|
-
exists=True,
|
|
17
|
-
file_okay=False,
|
|
18
|
-
resolve_path=True
|
|
19
|
-
),
|
|
20
|
-
bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
|
|
21
|
-
table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)")
|
|
22
|
-
):
|
|
23
|
-
"""
|
|
24
|
-
Submit input (folder) to S3 and register job in DynamoDB
|
|
25
|
-
"""
|
|
26
|
-
cfg = config.load()
|
|
27
|
-
bucket, table_name = config.resolve_target(cfg, bucket, table_name)
|
|
28
|
-
config.show_target(cfg)
|
|
29
|
-
session = get_session(cfg)
|
|
30
|
-
s3 = session.client('s3')
|
|
31
|
-
db = session.resource('dynamodb')
|
|
32
|
-
table = db.Table(table_name)
|
|
33
|
-
|
|
34
|
-
job_id = str(uuid.uuid4())
|
|
35
|
-
identity_id = cfg['identity_id']
|
|
36
|
-
username = cfg['username']
|
|
37
|
-
|
|
38
|
-
# Path: users/{identity_id}/{job_id}/...
|
|
39
|
-
base_path = f"users/{identity_id}/{job_id}"
|
|
40
|
-
input_path = f"{base_path}/registry_input"
|
|
41
|
-
|
|
42
|
-
print(f"Submitting job [blue]{job_id}[/blue] for user [blue]{username}[/blue]...")
|
|
43
|
-
|
|
44
|
-
# Auto-detect if user pointed to parent of registry_input
|
|
45
|
-
if (input_dir / "registry_input").exists() and (input_dir / "registry_input").is_dir():
|
|
46
|
-
print(f"[yellow]Detected 'registry_input' subfolder. using {input_dir}/registry_input as root.[/yellow]")
|
|
47
|
-
upload_root = input_dir / "registry_input"
|
|
48
|
-
else:
|
|
49
|
-
upload_root = input_dir
|
|
50
|
-
|
|
51
|
-
# Validate structure
|
|
52
|
-
if not (upload_root / "libraries").exists():
|
|
53
|
-
print(f"[red]Error: 'libraries' folder not found in {upload_root}. Structure should be libraries/ and feature_spaces/.[/red]")
|
|
54
|
-
raise typer.Exit(code=1)
|
|
55
|
-
|
|
56
|
-
files = [
|
|
57
|
-
p for p in upload_root.rglob("*")
|
|
58
|
-
if p.is_file() and p.name != ".DS_Store"
|
|
59
|
-
]
|
|
60
|
-
|
|
61
|
-
# 1. Upload files to S3
|
|
62
|
-
for file_path in track(files, description="Uploading files..."):
|
|
63
|
-
relative_path = file_path.relative_to(upload_root)
|
|
64
|
-
s3_key = f"{input_path}/{relative_path}"
|
|
65
|
-
|
|
66
|
-
with open(file_path, 'rb') as f:
|
|
67
|
-
s3.put_object(
|
|
68
|
-
Bucket=bucket,
|
|
69
|
-
Key=s3_key,
|
|
70
|
-
Body=f
|
|
71
|
-
)
|
|
72
|
-
|
|
73
|
-
# 2. Upload trigger file (input.json) if it doesn't exist in the list
|
|
74
|
-
# Assuming input_dir *is* the registry_input content
|
|
75
|
-
trigger_key = f"{input_path}/input.json"
|
|
76
|
-
s3.put_object(
|
|
77
|
-
Bucket=bucket,
|
|
78
|
-
Key=trigger_key,
|
|
79
|
-
Body="{}" # Simple trigger
|
|
80
|
-
)
|
|
81
|
-
|
|
82
|
-
# 3. Register job in DynamoDB
|
|
83
|
-
table.put_item(
|
|
84
|
-
Item={
|
|
85
|
-
'user_id': identity_id,
|
|
86
|
-
'job_id': job_id,
|
|
87
|
-
'username': username,
|
|
88
|
-
'status': 'SUBMITTED',
|
|
89
|
-
'created_at': datetime.datetime.utcnow().isoformat(),
|
|
90
|
-
's3_path': f"s3://{bucket}/{base_path}"
|
|
91
|
-
}
|
|
92
|
-
)
|
|
93
|
-
|
|
94
|
-
print("[green]Job submitted successfully![/green]")
|
|
95
|
-
print(f"Job ID: [bold]{job_id}[/bold]")
|
|
@@ -1,134 +0,0 @@
|
|
|
1
|
-
import datetime
|
|
2
|
-
import json
|
|
3
|
-
import uuid
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
from typing import Optional
|
|
6
|
-
|
|
7
|
-
import typer
|
|
8
|
-
from rich import print
|
|
9
|
-
from rich.panel import Panel
|
|
10
|
-
from typing_extensions import Annotated
|
|
11
|
-
|
|
12
|
-
from chemsift_cloud import config
|
|
13
|
-
from chemsift_cloud.register import get_session
|
|
14
|
-
|
|
15
|
-
app = typer.Typer()
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
@app.command()
|
|
19
|
-
def supervised(
|
|
20
|
-
register_job_id: Annotated[str, typer.Argument(
|
|
21
|
-
help="Register job ID whose registry_output contains the pre-registered pool library."
|
|
22
|
-
)],
|
|
23
|
-
yaml: Annotated[Path, typer.Option(
|
|
24
|
-
"--yaml", help="Local path to supervised query YAML config.",
|
|
25
|
-
exists=True, file_okay=True, dir_okay=False, resolve_path=True,
|
|
26
|
-
)],
|
|
27
|
-
state: Annotated[Optional[Path], typer.Option(
|
|
28
|
-
"--state",
|
|
29
|
-
help="Local state CSV. Omit on round 0. On subsequent rounds pass the CSV "
|
|
30
|
-
"with the 'label' column filled in.",
|
|
31
|
-
file_okay=True, dir_okay=False, resolve_path=True,
|
|
32
|
-
)] = None,
|
|
33
|
-
bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
|
|
34
|
-
table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)"),
|
|
35
|
-
):
|
|
36
|
-
"""
|
|
37
|
-
Submit one round of supervised active-learning screening.
|
|
38
|
-
|
|
39
|
-
Round 0 (first call — no --state): randomly samples n_samples molecules and
|
|
40
|
-
writes them to S3 with an empty 'label' column.
|
|
41
|
-
|
|
42
|
-
Subsequent rounds: upload the state CSV with labels filled in, and the model
|
|
43
|
-
selects the next batch using the RF/NB/ChemSift ensemble.
|
|
44
|
-
|
|
45
|
-
After each round:
|
|
46
|
-
1. chemsift status <job_id> — wait for SUCCEEDED
|
|
47
|
-
2. chemsift download <job_id> . — downloads state.csv with the new round
|
|
48
|
-
3. Fill the 'label' column (0 = inactive, 1 = active)
|
|
49
|
-
4. Re-run: chemsift supervised <register_job_id> --yaml ... --state state.csv
|
|
50
|
-
"""
|
|
51
|
-
cfg = config.load()
|
|
52
|
-
bucket, table_name = config.resolve_target(cfg, bucket, table_name)
|
|
53
|
-
config.show_target(cfg)
|
|
54
|
-
session = get_session(cfg)
|
|
55
|
-
s3 = session.client("s3")
|
|
56
|
-
db = session.resource("dynamodb")
|
|
57
|
-
table = db.Table(table_name)
|
|
58
|
-
|
|
59
|
-
identity_id = cfg["identity_id"]
|
|
60
|
-
job_id = str(uuid.uuid4())
|
|
61
|
-
|
|
62
|
-
# Determine round number from existing state file
|
|
63
|
-
round_number = 0
|
|
64
|
-
if state and Path(state).exists():
|
|
65
|
-
import pandas as pd
|
|
66
|
-
df = pd.read_csv(state)
|
|
67
|
-
round_number = int(df["round"].max()) + 1
|
|
68
|
-
|
|
69
|
-
# Validate register job exists
|
|
70
|
-
resp = table.get_item(Key={"user_id": identity_id, "job_id": register_job_id})
|
|
71
|
-
source = resp.get("Item")
|
|
72
|
-
if not source:
|
|
73
|
-
print(f"[red]Register job {register_job_id} not found.[/red]")
|
|
74
|
-
raise typer.Exit(code=1)
|
|
75
|
-
if source.get("status") != "SUCCEEDED":
|
|
76
|
-
print(f"[yellow]Warning: register job status is '{source.get('status')}'. "
|
|
77
|
-
"Pool library may be incomplete.[/yellow]")
|
|
78
|
-
|
|
79
|
-
registry_output_prefix = f"users/{identity_id}/{register_job_id}/registry_output"
|
|
80
|
-
input_prefix = f"users/{identity_id}/{job_id}/supervised_input"
|
|
81
|
-
output_prefix = f"users/{identity_id}/{job_id}/supervised_output"
|
|
82
|
-
|
|
83
|
-
# Upload YAML
|
|
84
|
-
yaml_key = f"{input_prefix}/config.yaml"
|
|
85
|
-
s3.upload_file(str(yaml), bucket, yaml_key)
|
|
86
|
-
|
|
87
|
-
# Upload state CSV if present (rounds > 0)
|
|
88
|
-
state_key = ""
|
|
89
|
-
if state and Path(state).exists():
|
|
90
|
-
state_key = f"{input_prefix}/state.csv"
|
|
91
|
-
s3.upload_file(str(state), bucket, state_key)
|
|
92
|
-
|
|
93
|
-
# Register in DynamoDB before uploading trigger (state machine will update to RUNNING)
|
|
94
|
-
table.put_item(Item={
|
|
95
|
-
"user_id": identity_id,
|
|
96
|
-
"job_id": job_id,
|
|
97
|
-
"username": cfg["username"],
|
|
98
|
-
"status": "SUBMITTED",
|
|
99
|
-
"job_type": "supervised_query",
|
|
100
|
-
"source_job_id": register_job_id,
|
|
101
|
-
"round_number": round_number,
|
|
102
|
-
"created_at": datetime.datetime.utcnow().isoformat(),
|
|
103
|
-
"s3_path": f"s3://{bucket}/{output_prefix}",
|
|
104
|
-
})
|
|
105
|
-
|
|
106
|
-
# Upload trigger file last — EventBridge watches for this key and invokes the Lambda
|
|
107
|
-
trigger = {
|
|
108
|
-
"user_id": identity_id,
|
|
109
|
-
"job_id": job_id,
|
|
110
|
-
"s3_bucket": bucket,
|
|
111
|
-
"registry_output_prefix": registry_output_prefix,
|
|
112
|
-
"yaml_key": yaml_key,
|
|
113
|
-
"state_key": state_key,
|
|
114
|
-
"output_prefix": output_prefix,
|
|
115
|
-
}
|
|
116
|
-
s3.put_object(
|
|
117
|
-
Bucket=bucket,
|
|
118
|
-
Key=f"{input_prefix}/supervised.json",
|
|
119
|
-
Body=json.dumps(trigger),
|
|
120
|
-
ContentType="application/json",
|
|
121
|
-
)
|
|
122
|
-
|
|
123
|
-
print(Panel(
|
|
124
|
-
f"[bold]Job ID:[/bold] {job_id}\n"
|
|
125
|
-
f"[bold]Round:[/bold] {round_number}\n"
|
|
126
|
-
f"[bold]Pool library:[/bold] {register_job_id}\n\n"
|
|
127
|
-
f"[bold]Next steps:[/bold]\n"
|
|
128
|
-
f" chemsift status {job_id}\n"
|
|
129
|
-
f" chemsift download {job_id} .\n"
|
|
130
|
-
f" # fill 'label' column in state.csv (0=inactive, 1=active)\n"
|
|
131
|
-
f" chemsift supervised {register_job_id} --yaml {yaml.name} --state state.csv",
|
|
132
|
-
title="[green]Supervised round submitted[/green]",
|
|
133
|
-
expand=False,
|
|
134
|
-
))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|