chemsift 1.2.0__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: chemsift
3
- Version: 1.2.0
3
+ Version: 1.3.0
4
4
  Summary:
5
5
  Author: Dave Martinez
6
6
  Author-email: 34991082+dkm-coder@users.noreply.github.com
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.11
12
12
  Classifier: Programming Language :: Python :: 3.12
13
13
  Classifier: Programming Language :: Python :: 3.13
14
14
  Classifier: Programming Language :: Python :: 3.14
15
+ Classifier: Programming Language :: Python :: 3.15
15
16
  Requires-Dist: boto3 (>=1.34.135,<2.0.0)
16
17
  Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
17
18
  Requires-Dist: requests (>=2.32.5,<3.0.0)
@@ -38,6 +39,8 @@ pip install chemsift
38
39
  - `chemsift map <register-job-id>`: Run the mapped pipeline against a completed register job
39
40
  - `chemsift query <register-job-id> --molecules <csv> --yaml <yml>`: Submit a query job
40
41
  - `chemsift supervised <register-job-id> --yaml <yml> [--state <csv>]`: Submit one round of active-learning screening
42
+ - `chemsift supervised --pool <pool-id> --yaml <yml> [--state <csv>]`: The same, against a shared demo pool instead of your own registration
43
+ - `chemsift env list` / `chemsift env use <dev|prod>`: Show or switch the active environment
41
44
  - `chemsift status <job-id>`: Check status of a job
42
45
  - `chemsift download <job-id> <output-dir>`: Download results of a completed job
43
46
  - `chemsift list`: List all job IDs for the current user
@@ -56,6 +59,29 @@ chemsift download <job-id> .
56
59
  chemsift supervised <register-job-id> --yaml supervised.yml --state state.csv
57
60
  ```
58
61
 
62
+ ### Screening a shared pool
63
+
64
+ A pool is a library registered once that every authenticated user of a stack can
65
+ screen, so a new account does not have to register anything before it can run a
66
+ round. Pass `--pool` instead of a register job id — everything else is identical:
67
+
68
+ ```bash
69
+ chemsift supervised --pool ecbd-luciferase-v1 --yaml supervised.yml
70
+ ```
71
+
72
+ The pool is read-only and lives in its own bucket; everything a round writes still
73
+ goes to your own prefix. `--pool-bucket` overrides the bucket, which defaults to
74
+ the active environment's.
75
+
76
+ Two things the pool form does for you. It names the pool's library in the uploaded
77
+ config when your config did not, so a campaign records what it screened (CS-08).
78
+ And on a resumed round it refuses a state CSV that was started against a different
79
+ library, rather than silently scoring one library's molecules with another's
80
+ ordering.
81
+
82
+ A pool publishes one pinned library with nothing to score it against, so pools are
83
+ screenable with `supervised` but are not valid targets for `query`.
84
+
59
85
  ## Development
60
86
 
61
87
  This project uses [Poetry](https://python-poetry.org/) for dependency management and packaging.
@@ -15,6 +15,8 @@ pip install chemsift
15
15
  - `chemsift map <register-job-id>`: Run the mapped pipeline against a completed register job
16
16
  - `chemsift query <register-job-id> --molecules <csv> --yaml <yml>`: Submit a query job
17
17
  - `chemsift supervised <register-job-id> --yaml <yml> [--state <csv>]`: Submit one round of active-learning screening
18
+ - `chemsift supervised --pool <pool-id> --yaml <yml> [--state <csv>]`: The same, against a shared demo pool instead of your own registration
19
+ - `chemsift env list` / `chemsift env use <dev|prod>`: Show or switch the active environment
18
20
  - `chemsift status <job-id>`: Check status of a job
19
21
  - `chemsift download <job-id> <output-dir>`: Download results of a completed job
20
22
  - `chemsift list`: List all job IDs for the current user
@@ -33,6 +35,29 @@ chemsift download <job-id> .
33
35
  chemsift supervised <register-job-id> --yaml supervised.yml --state state.csv
34
36
  ```
35
37
 
38
+ ### Screening a shared pool
39
+
40
+ A pool is a library registered once that every authenticated user of a stack can
41
+ screen, so a new account does not have to register anything before it can run a
42
+ round. Pass `--pool` instead of a register job id — everything else is identical:
43
+
44
+ ```bash
45
+ chemsift supervised --pool ecbd-luciferase-v1 --yaml supervised.yml
46
+ ```
47
+
48
+ The pool is read-only and lives in its own bucket; everything a round writes still
49
+ goes to your own prefix. `--pool-bucket` overrides the bucket, which defaults to
50
+ the active environment's.
51
+
52
+ Two things the pool form does for you. It names the pool's library in the uploaded
53
+ config when your config did not, so a campaign records what it screened (CS-08).
54
+ And on a resumed round it refuses a state CSV that was started against a different
55
+ library, rather than silently scoring one library's molecules with another's
56
+ ordering.
57
+
58
+ A pool publishes one pinned library with nothing to score it against, so pools are
59
+ screenable with `supervised` but are not valid targets for `query`.
60
+
36
61
  ## Development
37
62
 
38
63
  This project uses [Poetry](https://python-poetry.org/) for dependency management and packaging.
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "chemsift"
3
- version = "1.2.0"
3
+ version = "1.3.0"
4
4
  description = ""
5
5
  authors = [
6
6
  {name = "Dave Martinez",email = "34991082+dkm-coder@users.noreply.github.com"}
@@ -0,0 +1,145 @@
1
+ import typer
2
+ import uuid
3
+ import datetime
4
+ import json
5
+ from pathlib import Path
6
+ from typing import List, Optional
7
+ from rich import print
8
+ from rich.progress import track
9
+ from chemsift_cloud import config
10
+ from chemsift_cloud.register import get_session
11
+ from chemsift_cloud.map import discover_s3_prefixes
12
+
13
+ app = typer.Typer()
14
+
15
+
16
+ @app.command("add-feature-space")
17
+ def add_feature_space(
18
+ register_job_id: str = typer.Argument(help="Register job ID to add a feature space to."),
19
+ feature_space_dir: Path = typer.Argument(
20
+ help="Local directory for the new feature space (must contain metadata.json).",
21
+ exists=True,
22
+ file_okay=False,
23
+ resolve_path=True,
24
+ ),
25
+ library: Optional[List[str]] = typer.Option(
26
+ None,
27
+ "--library",
28
+ help="Limit to this library (repeatable). Default: every library in the register job.",
29
+ ),
30
+ bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
31
+ table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)"),
32
+ ):
33
+ """
34
+ Add a new feature space to an already-registered library.
35
+
36
+ Featurizes, indexes, clusters, and projects the new space only — the
37
+ register job's other feature spaces and its pre-registration are left
38
+ untouched.
39
+ """
40
+ cfg = config.load()
41
+ bucket, table_name = config.resolve_target(cfg, bucket, table_name)
42
+ config.show_target(cfg)
43
+ session = get_session(cfg)
44
+ s3 = session.client('s3')
45
+ db = session.resource('dynamodb')
46
+ table = db.Table(table_name)
47
+ identity_id = cfg['identity_id']
48
+
49
+ if not (feature_space_dir / "metadata.json").exists():
50
+ print(f"[red]Error: 'metadata.json' not found in {feature_space_dir}.[/red]")
51
+ raise typer.Exit(code=1)
52
+ fs_name = feature_space_dir.name
53
+
54
+ # Look up source register job
55
+ resp = table.get_item(Key={'user_id': identity_id, 'job_id': register_job_id})
56
+ source_item = resp.get('Item')
57
+ if not source_item:
58
+ print(f"[red]Register job {register_job_id} not found.[/red]")
59
+ raise typer.Exit(code=1)
60
+
61
+ if source_item.get('job_type', 'register') != 'register':
62
+ print(f"[red]Job {register_job_id} is not a register job (type: {source_item.get('job_type')}).[/red]")
63
+ raise typer.Exit(code=1)
64
+
65
+ if source_item.get('status') != 'SUCCEEDED':
66
+ print(f"[yellow]Warning: source job status is '{source_item.get('status')}'. Proceeding anyway.[/yellow]")
67
+
68
+ # Discover libraries from registry_output
69
+ registry_output_prefix = f"users/{identity_id}/{register_job_id}/registry_output/"
70
+ all_libraries = discover_s3_prefixes(s3, bucket, registry_output_prefix)
71
+ if not all_libraries:
72
+ print(f"[red]No libraries found under {registry_output_prefix}. Is the register job complete?[/red]")
73
+ raise typer.Exit(code=1)
74
+
75
+ if library:
76
+ missing = sorted(set(library) - set(all_libraries))
77
+ if missing:
78
+ print(f"[red]Unknown librar{'y' if len(missing) == 1 else 'ies'}: {missing}. "
79
+ f"Available: {all_libraries}[/red]")
80
+ raise typer.Exit(code=1)
81
+ libraries = list(library)
82
+ else:
83
+ libraries = all_libraries
84
+
85
+ # Refuse to silently clobber a feature space that already exists.
86
+ collisions = [
87
+ lib for lib in libraries
88
+ if fs_name in discover_s3_prefixes(s3, bucket, f"{registry_output_prefix}{lib}/standalone/")
89
+ ]
90
+ if collisions:
91
+ print(f"[red]Feature space '{fs_name}' already exists for: {collisions}. "
92
+ f"Choose a different name or remove it first.[/red]")
93
+ raise typer.Exit(code=1)
94
+
95
+ job_id = str(uuid.uuid4())
96
+ output_prefix = f"users/{identity_id}/{register_job_id}/registry_output"
97
+ input_prefix = f"users/{identity_id}/{job_id}/add_feature_space_input"
98
+
99
+ # Upload the new feature space's files (metadata.json + any sibling files, e.g. a normalizer)
100
+ files = [
101
+ p for p in feature_space_dir.rglob("*")
102
+ if p.is_file() and p.name != ".DS_Store"
103
+ ]
104
+ for file_path in track(files, description=f"Uploading {fs_name}..."):
105
+ relative_path = file_path.relative_to(feature_space_dir)
106
+ s3_key = f"{input_prefix}/feature_spaces/{fs_name}/{relative_path}"
107
+ with open(file_path, 'rb') as f:
108
+ s3.put_object(Bucket=bucket, Key=s3_key, Body=f)
109
+
110
+ # Register in DynamoDB
111
+ table.put_item(Item={
112
+ 'user_id': identity_id,
113
+ 'job_id': job_id,
114
+ 'username': cfg['username'],
115
+ 'status': 'SUBMITTED',
116
+ 'job_type': 'add_feature_space',
117
+ 'source_job_id': register_job_id,
118
+ 'created_at': datetime.datetime.utcnow().isoformat(),
119
+ 's3_path': f"s3://{bucket}/{output_prefix}",
120
+ })
121
+
122
+ sfn_input = {
123
+ 'user_id': identity_id,
124
+ 'job_id': job_id,
125
+ 's3_bucket': bucket,
126
+ 'input_prefix': input_prefix,
127
+ 'output_prefix': output_prefix,
128
+ 'libraries': [{'name': lib} for lib in libraries],
129
+ 'feature_spaces': [{'name': fs_name}],
130
+ }
131
+
132
+ # Upload trigger file — EventBridge watches for this key and invokes the
133
+ # add-feature-space Lambda.
134
+ trigger_key = f"{input_prefix}/add_feature_space.json"
135
+ s3.put_object(
136
+ Bucket=bucket,
137
+ Key=trigger_key,
138
+ Body=json.dumps(sfn_input),
139
+ ContentType="application/json",
140
+ )
141
+
142
+ print(f"[green]Add-feature-space job submitted![/green]")
143
+ print(f"Job ID: [bold]{job_id}[/bold]")
144
+ print(f"Feature space: {fs_name}")
145
+ print(f"Libraries: {libraries}")
@@ -210,6 +210,25 @@ def resolve_target(cfg: dict, bucket: str | None, table_name: str | None) -> tup
210
210
  return bucket or cfg["bucket"], table_name or cfg["table"]
211
211
 
212
212
 
213
+ def resolve_pool_bucket(cfg: dict, pool_bucket: str | None) -> str:
214
+ """The bucket holding shared demo pools for the active environment.
215
+
216
+ Separate from `resolve_target` because it is a different bucket with a
217
+ different purpose: pools are read-only published data granted by the pool
218
+ bucket's own policy, while `bucket` is where this user's own job inputs and
219
+ outputs live. Conflating them would send a round's output into a bucket the
220
+ caller has no write on, and the failure would surface inside Batch.
221
+ """
222
+ resolved = pool_bucket or cfg.get("pool_bucket")
223
+ if not resolved:
224
+ print(
225
+ f"[red][bold]Error[/bold]: environment '{cfg.get('environment', '?')}' declares no "
226
+ f"pool bucket.[/red] Pass [blue]--pool-bucket[/blue] explicitly."
227
+ )
228
+ raise typer.Exit(code=1)
229
+ return resolved
230
+
231
+
213
232
  def show_target(cfg: dict) -> None:
214
233
  """One line naming the stack a command is about to act on."""
215
234
  env = cfg.get("environment", "?")
@@ -11,13 +11,13 @@ app = typer.Typer()
11
11
 
12
12
  @app.command()
13
13
  def download(
14
- job_id: Annotated[str, typer.Argument(help="ID of the job to download the results of")],
15
- output_dir: Annotated[Path, typer.Argument(help="Directory in which to download the job result")],
14
+ job_id: Annotated[str, typer.Argument(help="Job ID to download.")],
15
+ output_dir: Annotated[Path, typer.Argument(help="Local directory to download into.")],
16
16
  bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
17
17
  table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)")
18
18
  ):
19
19
  """
20
- Download the result of a given job ID
20
+ Download a job's results.
21
21
  """
22
22
  cfg = config.load()
23
23
  bucket, table_name = config.resolve_target(cfg, bucket, table_name)
@@ -39,6 +39,9 @@ def download(
39
39
  elif job_type == 'map':
40
40
  source_job_id = item.get('source_job_id', job_id)
41
41
  prefix = f"users/{cfg['identity_id']}/{source_job_id}/registry_output/"
42
+ elif job_type == 'add_feature_space':
43
+ source_job_id = item.get('source_job_id', job_id)
44
+ prefix = f"users/{cfg['identity_id']}/{source_job_id}/registry_output/"
42
45
  else:
43
46
  # register (default)
44
47
  prefix = f"users/{cfg['identity_id']}/{job_id}/registry_output/"
@@ -37,6 +37,12 @@ ENVIRONMENTS: dict[str, dict] = {
37
37
  "cognito_domain": "chemsift-register-pipeline-prod.auth.eu-west-1.amazoncognito.com",
38
38
  "bucket": "chemsift-register-pipeline",
39
39
  "table": "chemsift-jobs",
40
+ # Shared demo pools — libraries registered once that every authenticated
41
+ # user of this stack may screen against. A fourth resource rather than a
42
+ # flag for the same reason the other three live here: a session and the
43
+ # resources it can reach must not drift apart. Per-environment, so a dev
44
+ # pool can be rebuilt without touching what prod's users are screening.
45
+ "pool_bucket": "chemsift-demo-pool",
40
46
  },
41
47
  "dev": {
42
48
  "name": "dev",
@@ -48,6 +54,7 @@ ENVIRONMENTS: dict[str, dict] = {
48
54
  "cognito_domain": "chemsift-register-pipeline-dev.auth.eu-west-1.amazoncognito.com",
49
55
  "bucket": "chemsift-register-pipeline-dev",
50
56
  "table": "chemsift-jobs-dev",
57
+ "pool_bucket": "chemsift-demo-pool-dev",
51
58
  },
52
59
  }
53
60
 
@@ -7,6 +7,8 @@ from chemsift_cloud.register import get_session
7
7
 
8
8
  app = typer.Typer()
9
9
 
10
+ JOBS_BY_CREATED_AT_INDEX = "user_id-created_at-index"
11
+
10
12
  def _compact_ts(ts: str) -> str:
11
13
  """Shorten ISO timestamp to 'YYYY-MM-DD HH:MM'."""
12
14
  if not ts or ts == 'N/A':
@@ -16,10 +18,10 @@ def _compact_ts(ts: str) -> str:
16
18
  @app.command()
17
19
  def list(
18
20
  table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)"),
19
- limit: int = typer.Option(50, help="Maximum number of most recent jobs to show")
21
+ limit: int = typer.Option(50, help="Maximum number of most recent jobs to show.")
20
22
  ):
21
23
  """
22
- Get list of job IDs for the current user
24
+ List your jobs.
23
25
  """
24
26
  cfg = config.load()
25
27
  _, table_name = config.resolve_target(cfg, None, table_name)
@@ -30,29 +32,31 @@ def list(
30
32
 
31
33
  print(f"Retrieving jobs for {cfg['username']}...")
32
34
 
33
- # Query is unbounded by job count but capped at 1MB per page by DynamoDB,
34
- # and results come back in job_id (sort key) order, not by recency — so a
35
- # single page can't be trusted to contain the actual most recent jobs once
36
- # a user has enough history to span multiple pages. Page through
37
- # everything before sorting so "most recent N" is actually correct.
35
+ # Read newest-first straight from the created_at GSI. The table's own sort
36
+ # key is job_id (a random UUID), so querying the table can't answer "most
37
+ # recent N" without reading the whole partition and sorting client-side.
38
+ # Rows without created_at are absent from the (sparse) index — see
39
+ # docs/runbooks/created-at-gsi-rollout.md.
38
40
  items = []
39
41
  query_kwargs = {
40
- "KeyConditionExpression": boto3.dynamodb.conditions.Key('user_id').eq(cfg['identity_id'])
42
+ "IndexName": JOBS_BY_CREATED_AT_INDEX,
43
+ "KeyConditionExpression": boto3.dynamodb.conditions.Key('user_id').eq(cfg['identity_id']),
44
+ "ScanIndexForward": False,
45
+ "Limit": limit,
41
46
  }
42
- while True:
47
+ while len(items) < limit:
43
48
  resp = table.query(**query_kwargs)
44
49
  items.extend(resp.get('Items', []))
45
50
  last_key = resp.get('LastEvaluatedKey')
46
51
  if not last_key:
47
52
  break
48
53
  query_kwargs['ExclusiveStartKey'] = last_key
54
+ items = items[:limit]
49
55
 
50
56
  if not items:
51
57
  print("[yellow]No jobs found.[/yellow]")
52
58
  return
53
59
 
54
- items = sorted(items, key=lambda x: x.get('created_at', ''), reverse=True)[:limit]
55
-
56
60
  table_ui = Table(title=f"Your Jobs (most recent {len(items)})")
57
61
  table_ui.add_column("Job ID", style="cyan", no_wrap=True, min_width=36)
58
62
  table_ui.add_column("Type", style="yellow")
@@ -4,6 +4,7 @@ from chemsift_cloud.env import app as env_app
4
4
  from chemsift_cloud.register import app as register_app
5
5
  from chemsift_cloud.map import app as map_app
6
6
  from chemsift_cloud.query import app as query_app
7
+ from chemsift_cloud.add_feature_space import app as add_feature_space_app
7
8
  from chemsift_cloud.supervised import app as supervised_app
8
9
  from chemsift_cloud.status import app as status_app
9
10
  from chemsift_cloud.download import app as download_app
@@ -16,6 +17,7 @@ app.add_typer(env_app)
16
17
  app.add_typer(register_app)
17
18
  app.add_typer(map_app)
18
19
  app.add_typer(query_app)
20
+ app.add_typer(add_feature_space_app)
19
21
  app.add_typer(supervised_app)
20
22
  app.add_typer(status_app)
21
23
  app.add_typer(download_app)
@@ -22,13 +22,13 @@ def discover_s3_prefixes(s3, bucket, prefix):
22
22
 
23
23
  @app.command()
24
24
  def map(
25
- register_job_id: str = typer.Argument(help="Register job ID to run mapping on"),
26
- reference_library: str = typer.Argument(help="Name of the reference library in registry_output"),
25
+ register_job_id: str = typer.Argument(help="Register job ID to map."),
26
+ reference_library: str = typer.Argument(help="Name of the reference library to map into."),
27
27
  bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
28
28
  table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)"),
29
29
  ):
30
30
  """
31
- Trigger the mapped pipeline against a completed register job.
31
+ Map a registered library into a reference library's space.
32
32
  """
33
33
  cfg = config.load()
34
34
  bucket, table_name = config.resolve_target(cfg, bucket, table_name)
@@ -0,0 +1,196 @@
1
+ """Shared demo pools: resolving one, and refusing to screen it incoherently.
2
+
3
+ A pool is a library registered once that every authenticated user of a stack may
4
+ screen against, so a new account does not have to register a library before it can
5
+ run anything. It is read-only; everything a round writes goes to the caller's own
6
+ prefix.
7
+
8
+ This module is deliberately free of `typer` — it raises `PoolError` and lets the
9
+ command layer turn that into output and an exit code. Two reasons, and the second
10
+ is the one that matters: presentation is not policy, and a module that imports
11
+ typer cannot be tested from the pipeline's own environment, which does not have it
12
+ (`cli/` is a separate Poetry package). The same split is why
13
+ `chemsift_cloud/standardization.py` is testable from `test/invariants/`.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import csv
19
+ import json
20
+ import tempfile
21
+ from pathlib import Path
22
+
23
+ import yaml as pyyaml
24
+
25
+
26
+ class PoolError(Exception):
27
+ """Something about the pool, the state file, or the config does not add up."""
28
+
29
+
30
+ # ---------------------------------------------------------------------------
31
+ # State CSV
32
+ # ---------------------------------------------------------------------------
33
+ #
34
+ # csv, not pandas. pandas is NOT a declared dependency of this package
35
+ # (cli/pyproject.toml lists typer, urllib3, requests, boto3, rich, pyyaml,
36
+ # typing-extensions), so the `import pandas` the supervised command used to do for
37
+ # a single max() crashed `--state` on a clean `pip install chemsift`. Nothing here
38
+ # needs a dataframe.
39
+
40
+ REQUIRED_STATE_COLUMNS = ("round", "idx", "smiles")
41
+
42
+
43
+ def read_state(path: Path) -> list[dict]:
44
+ with open(path, newline="", encoding="utf-8") as fh:
45
+ rows = list(csv.DictReader(fh))
46
+ if not rows:
47
+ raise PoolError(f"{path} has no rows.")
48
+ missing = [c for c in REQUIRED_STATE_COLUMNS if c not in rows[0]]
49
+ if missing:
50
+ raise PoolError(
51
+ f"{path} has no {', '.join(repr(m) for m in missing)} column(s) — "
52
+ f"is this a supervised state CSV?"
53
+ )
54
+ return rows
55
+
56
+
57
+ def next_round(rows: list[dict]) -> int:
58
+ values = [r["round"] for r in rows if r.get("round", "") != ""]
59
+ if not values:
60
+ raise PoolError("The 'round' column is empty in every row.")
61
+ try:
62
+ return max(int(float(v)) for v in values) + 1
63
+ except ValueError:
64
+ raise PoolError("The 'round' column holds a value that is not a number.")
65
+
66
+
67
+ def read_library_smiles(s3, bucket: str, prefix: str, library: str) -> list[str]:
68
+ """The library's molecules in row order — the definition of `idx`.
69
+
70
+ `mols.smiles` is a one-column CSV *with a header*, read here the way the
71
+ pipeline reads it (`library.py`). Counting lines instead would shift every
72
+ index by one and attach every label to the next molecule along.
73
+ """
74
+ key = f"{prefix.rstrip('/')}/{library}/mols.smiles"
75
+ with tempfile.TemporaryDirectory() as tmp:
76
+ local = Path(tmp) / "mols.smiles"
77
+ try:
78
+ s3.download_file(bucket, key, str(local))
79
+ except Exception as exc:
80
+ raise PoolError(
81
+ f"Could not read s3://{bucket}/{key} to verify the state file "
82
+ f"({type(exc).__name__})."
83
+ )
84
+ with open(local, newline="", encoding="utf-8") as fh:
85
+ return [r["smiles"] for r in csv.DictReader(fh)]
86
+
87
+
88
+ def state_library_mismatches(rows: list[dict], library_smiles: list[str],
89
+ limit: int = 5) -> list[str]:
90
+ """Rows whose `idx` does not denote the molecule they claim.
91
+
92
+ `idx` is a row offset into the library's `mols.smiles`, so a state produced
93
+ against one registry is meaningless against another — and both failure modes
94
+ are silent. Resume a campaign against a different pool, or against a library
95
+ re-registered in a different order, and every label attaches to the wrong
96
+ structure while the job succeeds and the hit rates stay plausible. That is
97
+ CS-03/CS-08 at the submission boundary.
98
+
99
+ The check needs no new metadata, because the state already carries both halves
100
+ of the claim: the `idx` and the `smiles` it is supposed to denote. Comparing
101
+ them verifies the meaning rather than a label for it, which is why this is
102
+ preferred over matching a recorded digest.
103
+ """
104
+ out: list[str] = []
105
+ for row in rows:
106
+ raw = row.get("idx", "")
107
+ if raw == "":
108
+ continue
109
+ claimed = row.get("smiles", "")
110
+ try:
111
+ idx = int(float(raw))
112
+ except ValueError:
113
+ out.append(f"idx {raw!r}: not a number")
114
+ else:
115
+ if not 0 <= idx < len(library_smiles):
116
+ out.append(f"idx {idx}: out of range for a "
117
+ f"{len(library_smiles)}-molecule library")
118
+ elif library_smiles[idx] != claimed:
119
+ out.append(f"idx {idx}: state says {claimed!r}, "
120
+ f"library holds {library_smiles[idx]!r}")
121
+ if len(out) >= limit:
122
+ break
123
+ return out
124
+
125
+
126
+ def verify_state_matches_library(s3, bucket: str, prefix: str, library: str,
127
+ rows: list[dict]) -> None:
128
+ mismatches = state_library_mismatches(rows, read_library_smiles(s3, bucket, prefix, library))
129
+ if mismatches:
130
+ raise PoolError(
131
+ "This state file does not belong to that library:\n "
132
+ + "\n ".join(mismatches)
133
+ + f"\n `idx` is a row offset into {library}/mols.smiles, so submitting this would "
134
+ f"train on labels attached to the wrong molecules (CS-08), and nothing would fail."
135
+ f"\n Use the library the campaign was started against, or start a new campaign."
136
+ )
137
+
138
+
139
+ # ---------------------------------------------------------------------------
140
+ # The pool itself
141
+ # ---------------------------------------------------------------------------
142
+
143
+ def load_manifest(s3, pool_bucket: str, pool_id: str) -> dict:
144
+ """Read a pool's manifest, which doubles as the authorization check.
145
+
146
+ This replaces the DynamoDB `get_item` the register-job path uses, and proves
147
+ strictly more: that row proved a record existed in the caller's own partition,
148
+ while reading `pool.json` proves the pool exists AND that these credentials
149
+ carry the pool bucket's read grant. Without it, the first sign of a missing
150
+ grant is a download failure inside Batch, minutes later, in a log nobody is
151
+ watching.
152
+ """
153
+ key = f"{pool_id}/pool.json"
154
+ try:
155
+ body = s3.get_object(Bucket=pool_bucket, Key=key)["Body"].read()
156
+ except Exception as exc:
157
+ raise PoolError(
158
+ f"Pool {pool_id!r} is not readable at s3://{pool_bucket}/{key} "
159
+ f"({type(exc).__name__}).\n"
160
+ f" Either the pool id is wrong, or this environment's users have no read grant "
161
+ f"on {pool_bucket}."
162
+ )
163
+ try:
164
+ manifest = json.loads(body)
165
+ except json.JSONDecodeError as exc:
166
+ raise PoolError(f"s3://{pool_bucket}/{key} is not valid JSON: {exc}")
167
+ if not manifest.get("library"):
168
+ raise PoolError(f"s3://{pool_bucket}/{key} declares no 'library'.")
169
+ return manifest
170
+
171
+
172
+ def config_naming_library(yaml_path: Path, library: str, tmpdir: str) -> Path:
173
+ """The config, with `library:` filled in from the pool if it was absent.
174
+
175
+ `select_pool_library` refuses to guess when a registry holds more than one
176
+ library, because the state file's `idx` is an offset into whichever one is
177
+ chosen and a different pick between rounds silently relabels every molecule
178
+ (CS-08). A pool's manifest already names its library, so writing it into the
179
+ config makes the choice a recorded fact rather than relying on there happening
180
+ to be a single candidate.
181
+ """
182
+ with open(yaml_path, encoding="utf-8") as fh:
183
+ cfg = pyyaml.safe_load(fh) or {}
184
+ declared = cfg.get("library")
185
+ if declared and declared != library:
186
+ raise PoolError(
187
+ f"The config names library {declared!r} but this pool holds {library!r}."
188
+ )
189
+ if declared:
190
+ return yaml_path
191
+ cfg["library"] = library
192
+ out = Path(tmpdir) / "config.yaml"
193
+ out.parent.mkdir(parents=True, exist_ok=True)
194
+ with open(out, "w", encoding="utf-8") as fh:
195
+ pyyaml.safe_dump(cfg, fh, sort_keys=False)
196
+ return out
@@ -7,6 +7,10 @@ from rich import print
7
7
  from rich.progress import track
8
8
  from pathlib import Path
9
9
  from chemsift_cloud import config
10
+ from chemsift_cloud.standardization import (
11
+ PolicyConflict,
12
+ inherited_standardization,
13
+ )
10
14
  from chemsift_cloud.register import get_session
11
15
 
12
16
  app = typer.Typer()
@@ -25,7 +29,7 @@ def discover_s3_prefixes(s3, bucket, prefix):
25
29
 
26
30
  @app.command()
27
31
  def query(
28
- register_job_id: str = typer.Argument(help="Register job ID whose registry_output to query"),
32
+ register_job_id: str = typer.Argument(help="Register job ID to query."),
29
33
  molecules: Annotated[Path, typer.Option(
30
34
  "--molecules", help="Local path to query molecules CSV",
31
35
  exists=True, file_okay=True, dir_okay=False, resolve_path=True
@@ -40,7 +44,7 @@ def query(
40
44
  precalculated_library_prefix: str = typer.Option(None, "--precalculated-library-prefix", help="S3 prefix of a single registered library directory (e.g. .../registry_output/<library_name>) whose precomputed features are used for the query molecules, skipping featurization"),
41
45
  ):
42
46
  """
43
- Submit a query against a completed register job's registry_output.
47
+ Submit a rule-based query against a registered library.
44
48
  """
45
49
  cfg = config.load()
46
50
  bucket, table_name = config.resolve_target(cfg, bucket, table_name)
@@ -74,26 +78,15 @@ def query(
74
78
  yaml_key = f"users/{identity_id}/{job_id}/query_input/query.yml"
75
79
  library_metadata_key = f"users/{identity_id}/{job_id}/query_input/libraries/query_molecules/metadata.json"
76
80
 
77
- library_metadata = {
78
- "name": "query_molecules",
79
- "structure_file": "molecules.csv",
80
- "smiles_column": "smiles",
81
- "is_reference": False
82
- }
83
81
 
84
- print(f"Uploading query inputs for job [blue]{job_id}[/blue]...")
85
82
  import json as _json
83
+
84
+ print(f"Uploading query inputs for job [blue]{job_id}[/blue]...")
86
85
  for local_path, s3_key in track(
87
86
  [(str(molecules), molecules_key), (str(yaml), yaml_key)],
88
87
  description="Uploading..."
89
88
  ):
90
89
  s3.upload_file(local_path, bucket, s3_key)
91
- s3.put_object(
92
- Bucket=bucket,
93
- Key=library_metadata_key,
94
- Body=_json.dumps(library_metadata),
95
- ContentType="application/json",
96
- )
97
90
 
98
91
  # Discover feature_spaces from local query YAML
99
92
  import yaml as pyyaml
@@ -108,6 +101,42 @@ def query(
108
101
 
109
102
  print(f"Discovered feature spaces from query YAML: {feature_spaces}")
110
103
 
104
+ # CS-83: the query molecules are registered as their OWN library, and the
105
+ # query job then slices its features instead of featurizing here. So that
106
+ # library has to be registered under the same standardization policy as the
107
+ # libraries the rules name (CS-78) — otherwise its vectors are built from
108
+ # different molecules than the index they are searched against, and
109
+ # run_query_pipeline refuses the job rather than scoring it.
110
+ #
111
+ # The policy is read from standardization.json, which registration writes
112
+ # beside mols.smiles and which records what ACTUALLY ran, not merely what
113
+ # was declared.
114
+ try:
115
+ standardization = inherited_standardization(
116
+ s3.get_object, bucket,
117
+ f"users/{identity_id}/{register_job_id}/registry_output", rules)
118
+ except PolicyConflict as exc:
119
+ print(f"[red]{exc}[/red]")
120
+ raise typer.Exit(code=1)
121
+ if standardization:
122
+ print(f"Inheriting standardization from the reference library: "
123
+ f"[blue]{standardization}[/blue]")
124
+
125
+ library_metadata = {
126
+ "name": "query_molecules",
127
+ "structure_file": "molecules.csv",
128
+ "smiles_column": "smiles",
129
+ "is_reference": False,
130
+ }
131
+ if standardization:
132
+ library_metadata["standardization"] = standardization
133
+ s3.put_object(
134
+ Bucket=bucket,
135
+ Key=library_metadata_key,
136
+ Body=_json.dumps(library_metadata),
137
+ ContentType="application/json",
138
+ )
139
+
111
140
  output_prefix = f"users/{identity_id}/{job_id}/query_output"
112
141
 
113
142
  # Register in DynamoDB
@@ -38,7 +38,7 @@ def get_session(cfg):
38
38
  @app.command()
39
39
  def register(
40
40
  input_dir: Path = typer.Argument(
41
- help="Directory to submit for processing",
41
+ help="Local registry_input folder (or its parent) — libraries/ and feature_spaces/.",
42
42
  exists=True,
43
43
  file_okay=False,
44
44
  resolve_path=True
@@ -47,7 +47,7 @@ def register(
47
47
  table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)")
48
48
  ):
49
49
  """
50
- Submit input (folder) to S3 and register job in DynamoDB
50
+ Submit a registration job for a library and its feature spaces.
51
51
  """
52
52
  cfg = config.load()
53
53
  bucket, table_name = config.resolve_target(cfg, bucket, table_name)
@@ -0,0 +1,89 @@
1
+ """Which standardization policy a cloud query must encode under (CS-83).
2
+
3
+ Stdlib only, and deliberately free of typer/rich/boto3 imports, so the invariant
4
+ suite in the core repo can exercise it without installing the CLI's dependency
5
+ tree. The CLI translates the error into a typer exit; the rule itself lives here.
6
+
7
+ Why this exists at all. In the cloud the query molecules are uploaded as their
8
+ OWN library, registered and featurized by Step Functions, and the query job then
9
+ SLICES those features rather than featurizing anything itself. So the query's
10
+ vectors are only comparable with the index they are searched against if that
11
+ library was registered under the same standardization policy (CS-78).
12
+
13
+ The CLI used to hardcode that library's metadata with no policy at all, so any
14
+ reference library declaring one became unqueryable: `run_query_pipeline` saw
15
+ `None` on the query side and `chembl_parent_largest` on the reference side and
16
+ refused the job — correctly, since the query's fingerprints would have carried
17
+ counterions the index has never seen. Found on dev, not locally, because the
18
+ LOCAL query path featurizes the query itself and so never takes this branch.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import json
24
+
25
+ #: Written beside mols.smiles by registration. Read rather than metadata.json
26
+ #: because it records the policy that ACTUALLY ran, not the one declared.
27
+ RECORD_NAME = "standardization.json"
28
+
29
+
30
+ class PolicyConflict(ValueError):
31
+ """The libraries a rule set names were registered under different policies."""
32
+
33
+
34
+ def library_names(rules) -> list[str]:
35
+ """The libraries a query's rules score against, deduplicated and ordered."""
36
+ return sorted({r["library"] for r in (rules or []) if r.get("library")})
37
+
38
+
39
+ def read_policy(get_object, bucket: str, prefix: str, library: str):
40
+ """The policy `library` was registered under, or None.
41
+
42
+ `get_object` is the boto3 client method, injected so this stays testable and
43
+ so a missing object is an ordinary None rather than a dependency on botocore
44
+ exception classes. A library registered before CS-78 has no record, and that
45
+ genuinely means no policy — not an error.
46
+ """
47
+ key = f"{prefix.rstrip('/')}/{library}/{RECORD_NAME}"
48
+ try:
49
+ body = get_object(Bucket=bucket, Key=key)["Body"].read()
50
+ except Exception as exc:
51
+ # ONLY a missing object means "registered before CS-78, so no policy".
52
+ # A bare `except Exception` here turned an AccessDenied or a throttle
53
+ # into None, which registers the query library unstandardized and fails
54
+ # the job in Batch minutes later — precisely the outcome CS-83 exists to
55
+ # prevent, reached by the error handling rather than the logic.
56
+ name = type(exc).__name__
57
+ code = getattr(getattr(exc, "response", None), "get", lambda *_: {})("Error", {})
58
+ if name in ("NoSuchKey", "KeyError", "FileNotFoundError", "404") or \
59
+ (isinstance(code, dict) and code.get("Code") in ("NoSuchKey", "404", "NotFound")):
60
+ return None
61
+ raise
62
+ try:
63
+ return json.loads(body).get("policy")
64
+ except (ValueError, TypeError, AttributeError):
65
+ return None
66
+
67
+
68
+ def inherited_standardization(get_object, bucket: str, prefix: str, rules):
69
+ """The single policy the query-molecules library must be registered under.
70
+
71
+ Raises PolicyConflict when the rule set names libraries registered under
72
+ different policies. Refusing here rather than picking one is the same
73
+ judgement run_query_pipeline makes server-side: a query is encoded once, so
74
+ choosing either policy silently mis-encodes it against the other half of the
75
+ rule set. Failing in the CLI is strictly better than failing in Batch five
76
+ minutes later, and says the same thing.
77
+ """
78
+ policies = {lib: read_policy(get_object, bucket, prefix, lib)
79
+ for lib in library_names(rules)}
80
+ distinct = set(policies.values())
81
+ if len(distinct) > 1:
82
+ detail = ", ".join(f"{lib}={pol!r}" for lib, pol in sorted(policies.items()))
83
+ raise PolicyConflict(
84
+ f"the libraries this query names were registered under different "
85
+ f"standardization policies ({detail}). A query is encoded once and scored "
86
+ f"against all of them, so they must agree — query them separately, or "
87
+ f"re-register the odd one out (CS-83)."
88
+ )
89
+ return next(iter(distinct), None)
@@ -11,11 +11,11 @@ app = typer.Typer()
11
11
 
12
12
  @app.command()
13
13
  def status(
14
- job_id: Annotated[str, typer.Argument(help="ID of the job to get the status of")],
14
+ job_id: Annotated[str, typer.Argument(help="Job ID to check.")],
15
15
  table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)")
16
16
  ):
17
17
  """
18
- Check status of a job ID
18
+ Check a job's status.
19
19
  """
20
20
  cfg = config.load()
21
21
  _, table_name = config.resolve_target(cfg, None, table_name)
@@ -0,0 +1,217 @@
1
+ import datetime
2
+ import json
3
+ import tempfile
4
+ import uuid
5
+ from pathlib import Path
6
+ from typing import Optional
7
+
8
+ import typer
9
+ from rich import print
10
+ from rich.panel import Panel
11
+ from typing_extensions import Annotated
12
+
13
+ from chemsift_cloud import config, pool as poollib
14
+ from chemsift_cloud.pool import PoolError
15
+ from chemsift_cloud.register import get_session
16
+
17
+ app = typer.Typer()
18
+
19
+
20
+ def _or_exit(fn, *args, **kwargs):
21
+ """Run a pool-module call, turning its PoolError into CLI output.
22
+
23
+ The policy lives in chemsift_cloud.pool, which raises; presentation lives
24
+ here. That split is what lets the checks be tested from the pipeline's own
25
+ environment, which has no typer.
26
+ """
27
+ try:
28
+ return fn(*args, **kwargs)
29
+ except PoolError as exc:
30
+ print(f"[red]{exc}[/red]")
31
+ raise typer.Exit(code=1)
32
+
33
+
34
+ @app.command()
35
+ def supervised(
36
+ register_job_id: Annotated[Optional[str], typer.Argument(
37
+ help="Register job ID whose library is the active-learning pool. "
38
+ "Omit when using --pool."
39
+ )] = None,
40
+ yaml: Annotated[Path, typer.Option(
41
+ "--yaml", help="Local path to supervised query YAML config.",
42
+ exists=True, file_okay=True, dir_okay=False, resolve_path=True,
43
+ )] = None,
44
+ state: Annotated[Optional[Path], typer.Option(
45
+ "--state",
46
+ help="Local state CSV. Omit on round 0. On subsequent rounds pass the CSV "
47
+ "with the 'label' column filled in (1 active, 0 inactive, -1 tested "
48
+ "but inconclusive).",
49
+ file_okay=True, dir_okay=False, resolve_path=True,
50
+ )] = None,
51
+ pool: Annotated[Optional[str], typer.Option(
52
+ "--pool",
53
+ help="Shared demo pool id to screen against (e.g. ecbd-luciferase-v1), "
54
+ "instead of one of your own register jobs.",
55
+ )] = None,
56
+ pool_bucket: str = typer.Option(None, help="Pool bucket (default: active environment)"),
57
+ bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
58
+ table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)"),
59
+ ):
60
+ """
61
+ Submit one round of a supervised query (active-learning screening).
62
+
63
+ Screens either one of your own registered libraries, or a shared demo pool:
64
+
65
+ chemsift supervised <register_job_id> --yaml config.yaml
66
+ chemsift supervised --pool ecbd-luciferase-v1 --yaml config.yaml
67
+
68
+ A pool is a library registered once that every authenticated user may screen
69
+ against, so a new account does not have to register a library before it can
70
+ run anything. It is READ-ONLY: everything this command writes — the config,
71
+ the state, the round's output — goes to your own prefix, exactly as it does
72
+ for your own registrations.
73
+
74
+ Round 0 (first call — no --state): randomly samples n_samples molecules and
75
+ writes them to S3 with an empty 'label' column.
76
+
77
+ Subsequent rounds: upload the state CSV with labels filled in, and the model
78
+ selects the next batch using the RF/NB/ChemSift ensemble.
79
+
80
+ After each round:
81
+ 1. chemsift status <job_id> — wait for SUCCEEDED
82
+ 2. chemsift download <job_id> . — downloads state.csv with the new round
83
+ 3. Fill the 'label' column (1 = active, 0 = inactive,
84
+ -1 = tested but inconclusive — never retested, never trains the model)
85
+ 4. Re-run with --state state.csv
86
+ """
87
+ if yaml is None:
88
+ print("[red]--yaml is required.[/red]")
89
+ raise typer.Exit(code=1)
90
+ # Exactly one source. Accepting both would leave the prefix ambiguous, and
91
+ # accepting neither used to be impossible only because the argument was
92
+ # required.
93
+ if bool(register_job_id) == bool(pool):
94
+ print("[red]Pass exactly one of <register_job_id> or --pool.[/red]")
95
+ raise typer.Exit(code=1)
96
+
97
+ cfg = config.load()
98
+ bucket, table_name = config.resolve_target(cfg, bucket, table_name)
99
+ config.show_target(cfg)
100
+ session = get_session(cfg)
101
+ s3 = session.client("s3")
102
+ db = session.resource("dynamodb")
103
+ table = db.Table(table_name)
104
+
105
+ identity_id = cfg["identity_id"]
106
+ job_id = str(uuid.uuid4())
107
+
108
+ rows = _or_exit(poollib.read_state, state) if state and Path(state).exists() else None
109
+ round_number = _or_exit(poollib.next_round, rows) if rows else 0
110
+
111
+ # ------------------------------------------------------------------
112
+ # Resolve what is being screened: a shared pool, or one of your own
113
+ # register jobs. These differ in WHERE the library is read from and in how
114
+ # the right to read it is established — nothing else.
115
+ # ------------------------------------------------------------------
116
+ source_bucket = bucket # the library's bucket; same as `bucket` unless pooled
117
+ manifest: dict = {}
118
+ if pool:
119
+ source_bucket = config.resolve_pool_bucket(cfg, pool_bucket)
120
+ manifest = _or_exit(poollib.load_manifest, s3, source_bucket, pool)
121
+ library = manifest["library"]
122
+ registry_output_prefix = f"{pool}/registry_output"
123
+ source_label = f"pool {pool} ({library}, {manifest.get('pool_size', '?')} molecules)"
124
+ else:
125
+ resp = table.get_item(Key={"user_id": identity_id, "job_id": register_job_id})
126
+ source = resp.get("Item")
127
+ if not source:
128
+ print(f"[red]Register job {register_job_id} not found.[/red]")
129
+ raise typer.Exit(code=1)
130
+ if source.get("status") != "SUCCEEDED":
131
+ print(f"[yellow]Warning: register job status is '{source.get('status')}'. "
132
+ "Pool library may be incomplete.[/yellow]")
133
+ library = None
134
+ registry_output_prefix = f"users/{identity_id}/{register_job_id}/registry_output"
135
+ source_label = register_job_id
136
+
137
+ input_prefix = f"users/{identity_id}/{job_id}/supervised_input"
138
+ output_prefix = f"users/{identity_id}/{job_id}/supervised_output"
139
+
140
+ # A resumed campaign must be resumed against the library it was started on.
141
+ if rows and library:
142
+ _or_exit(poollib.verify_state_matches_library,
143
+ s3, source_bucket, registry_output_prefix, library, rows)
144
+
145
+ with tempfile.TemporaryDirectory() as tmp:
146
+ # Upload YAML — naming the pool's library when the config did not (CS-08)
147
+ upload_yaml = (_or_exit(poollib.config_naming_library, yaml, library, tmp)
148
+ if library else yaml)
149
+ yaml_key = f"{input_prefix}/config.yaml"
150
+ s3.upload_file(str(upload_yaml), bucket, yaml_key)
151
+
152
+ # Upload state CSV if present (rounds > 0)
153
+ state_key = ""
154
+ if rows:
155
+ state_key = f"{input_prefix}/state.csv"
156
+ s3.upload_file(str(state), bucket, state_key)
157
+
158
+ # Register in DynamoDB before uploading trigger (state machine will update to RUNNING)
159
+ item = {
160
+ "user_id": identity_id,
161
+ "job_id": job_id,
162
+ "username": cfg["username"],
163
+ "status": "SUBMITTED",
164
+ "job_type": "supervised_query",
165
+ "round_number": round_number,
166
+ "created_at": datetime.datetime.utcnow().isoformat(),
167
+ "s3_path": f"s3://{bucket}/{output_prefix}",
168
+ }
169
+ if pool:
170
+ # No source_job_id: there is no register job in THIS user's partition to
171
+ # point at, and inventing one would make a pooled campaign look like it
172
+ # came from a registration the user owns.
173
+ item["source_pool"] = pool
174
+ item["source_pool_bucket"] = source_bucket
175
+ if (manifest.get("source") or {}).get("registration_job_id"):
176
+ item["source_registration"] = manifest["source"]["registration_job_id"]
177
+ else:
178
+ item["source_job_id"] = register_job_id
179
+ table.put_item(Item=item)
180
+
181
+ # Upload trigger file last — EventBridge watches for this key and invokes the Lambda
182
+ trigger = {
183
+ "user_id": identity_id,
184
+ "job_id": job_id,
185
+ "s3_bucket": bucket,
186
+ # Where the library is READ from. The Lambda defaults this to s3_bucket
187
+ # when absent, so an older client keeps working; sending it explicitly is
188
+ # what lets the pool live in its own read-only bucket.
189
+ "registry_output_bucket": source_bucket,
190
+ "registry_output_prefix": registry_output_prefix,
191
+ "yaml_key": yaml_key,
192
+ "state_key": state_key,
193
+ "output_prefix": output_prefix,
194
+ }
195
+ s3.put_object(
196
+ Bucket=bucket,
197
+ Key=f"{input_prefix}/supervised.json",
198
+ Body=json.dumps(trigger),
199
+ ContentType="application/json",
200
+ )
201
+
202
+ resubmit = (f"chemsift supervised --pool {pool} --yaml {yaml.name} --state state.csv"
203
+ if pool else
204
+ f"chemsift supervised {register_job_id} --yaml {yaml.name} --state state.csv")
205
+ print(Panel(
206
+ f"[bold]Job ID:[/bold] {job_id}\n"
207
+ f"[bold]Round:[/bold] {round_number}\n"
208
+ f"[bold]Screening:[/bold] {source_label}\n"
209
+ f"[bold]Reading from:[/bold] s3://{source_bucket}/{registry_output_prefix}\n\n"
210
+ f"[bold]Next steps:[/bold]\n"
211
+ f" chemsift status {job_id}\n"
212
+ f" chemsift download {job_id} .\n"
213
+ f" # fill 'label' column in state.csv (1=active, 0=inactive, -1=inconclusive)\n"
214
+ f" {resubmit}",
215
+ title="[green]Supervised round submitted[/green]",
216
+ expand=False,
217
+ ))
@@ -1,95 +0,0 @@
1
- import typer
2
- from rich import print
3
- from rich.progress import track
4
- from chemsift_cloud import config
5
- from chemsift_cloud.register import get_session
6
- from pathlib import Path
7
- import uuid
8
- import datetime
9
-
10
- app = typer.Typer()
11
-
12
- @app.command()
13
- def submit(
14
- input_dir: Path = typer.Argument(
15
- help="Directory to submit for processing",
16
- exists=True,
17
- file_okay=False,
18
- resolve_path=True
19
- ),
20
- bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
21
- table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)")
22
- ):
23
- """
24
- Submit input (folder) to S3 and register job in DynamoDB
25
- """
26
- cfg = config.load()
27
- bucket, table_name = config.resolve_target(cfg, bucket, table_name)
28
- config.show_target(cfg)
29
- session = get_session(cfg)
30
- s3 = session.client('s3')
31
- db = session.resource('dynamodb')
32
- table = db.Table(table_name)
33
-
34
- job_id = str(uuid.uuid4())
35
- identity_id = cfg['identity_id']
36
- username = cfg['username']
37
-
38
- # Path: users/{identity_id}/{job_id}/...
39
- base_path = f"users/{identity_id}/{job_id}"
40
- input_path = f"{base_path}/registry_input"
41
-
42
- print(f"Submitting job [blue]{job_id}[/blue] for user [blue]{username}[/blue]...")
43
-
44
- # Auto-detect if user pointed to parent of registry_input
45
- if (input_dir / "registry_input").exists() and (input_dir / "registry_input").is_dir():
46
- print(f"[yellow]Detected 'registry_input' subfolder. using {input_dir}/registry_input as root.[/yellow]")
47
- upload_root = input_dir / "registry_input"
48
- else:
49
- upload_root = input_dir
50
-
51
- # Validate structure
52
- if not (upload_root / "libraries").exists():
53
- print(f"[red]Error: 'libraries' folder not found in {upload_root}. Structure should be libraries/ and feature_spaces/.[/red]")
54
- raise typer.Exit(code=1)
55
-
56
- files = [
57
- p for p in upload_root.rglob("*")
58
- if p.is_file() and p.name != ".DS_Store"
59
- ]
60
-
61
- # 1. Upload files to S3
62
- for file_path in track(files, description="Uploading files..."):
63
- relative_path = file_path.relative_to(upload_root)
64
- s3_key = f"{input_path}/{relative_path}"
65
-
66
- with open(file_path, 'rb') as f:
67
- s3.put_object(
68
- Bucket=bucket,
69
- Key=s3_key,
70
- Body=f
71
- )
72
-
73
- # 2. Upload trigger file (input.json) if it doesn't exist in the list
74
- # Assuming input_dir *is* the registry_input content
75
- trigger_key = f"{input_path}/input.json"
76
- s3.put_object(
77
- Bucket=bucket,
78
- Key=trigger_key,
79
- Body="{}" # Simple trigger
80
- )
81
-
82
- # 3. Register job in DynamoDB
83
- table.put_item(
84
- Item={
85
- 'user_id': identity_id,
86
- 'job_id': job_id,
87
- 'username': username,
88
- 'status': 'SUBMITTED',
89
- 'created_at': datetime.datetime.utcnow().isoformat(),
90
- 's3_path': f"s3://{bucket}/{base_path}"
91
- }
92
- )
93
-
94
- print("[green]Job submitted successfully![/green]")
95
- print(f"Job ID: [bold]{job_id}[/bold]")
@@ -1,134 +0,0 @@
1
- import datetime
2
- import json
3
- import uuid
4
- from pathlib import Path
5
- from typing import Optional
6
-
7
- import typer
8
- from rich import print
9
- from rich.panel import Panel
10
- from typing_extensions import Annotated
11
-
12
- from chemsift_cloud import config
13
- from chemsift_cloud.register import get_session
14
-
15
- app = typer.Typer()
16
-
17
-
18
- @app.command()
19
- def supervised(
20
- register_job_id: Annotated[str, typer.Argument(
21
- help="Register job ID whose registry_output contains the pre-registered pool library."
22
- )],
23
- yaml: Annotated[Path, typer.Option(
24
- "--yaml", help="Local path to supervised query YAML config.",
25
- exists=True, file_okay=True, dir_okay=False, resolve_path=True,
26
- )],
27
- state: Annotated[Optional[Path], typer.Option(
28
- "--state",
29
- help="Local state CSV. Omit on round 0. On subsequent rounds pass the CSV "
30
- "with the 'label' column filled in.",
31
- file_okay=True, dir_okay=False, resolve_path=True,
32
- )] = None,
33
- bucket: str = typer.Option(None, help="S3 bucket (default: active environment)"),
34
- table_name: str = typer.Option(None, help="DynamoDB jobs table (default: active environment)"),
35
- ):
36
- """
37
- Submit one round of supervised active-learning screening.
38
-
39
- Round 0 (first call — no --state): randomly samples n_samples molecules and
40
- writes them to S3 with an empty 'label' column.
41
-
42
- Subsequent rounds: upload the state CSV with labels filled in, and the model
43
- selects the next batch using the RF/NB/ChemSift ensemble.
44
-
45
- After each round:
46
- 1. chemsift status <job_id> — wait for SUCCEEDED
47
- 2. chemsift download <job_id> . — downloads state.csv with the new round
48
- 3. Fill the 'label' column (0 = inactive, 1 = active)
49
- 4. Re-run: chemsift supervised <register_job_id> --yaml ... --state state.csv
50
- """
51
- cfg = config.load()
52
- bucket, table_name = config.resolve_target(cfg, bucket, table_name)
53
- config.show_target(cfg)
54
- session = get_session(cfg)
55
- s3 = session.client("s3")
56
- db = session.resource("dynamodb")
57
- table = db.Table(table_name)
58
-
59
- identity_id = cfg["identity_id"]
60
- job_id = str(uuid.uuid4())
61
-
62
- # Determine round number from existing state file
63
- round_number = 0
64
- if state and Path(state).exists():
65
- import pandas as pd
66
- df = pd.read_csv(state)
67
- round_number = int(df["round"].max()) + 1
68
-
69
- # Validate register job exists
70
- resp = table.get_item(Key={"user_id": identity_id, "job_id": register_job_id})
71
- source = resp.get("Item")
72
- if not source:
73
- print(f"[red]Register job {register_job_id} not found.[/red]")
74
- raise typer.Exit(code=1)
75
- if source.get("status") != "SUCCEEDED":
76
- print(f"[yellow]Warning: register job status is '{source.get('status')}'. "
77
- "Pool library may be incomplete.[/yellow]")
78
-
79
- registry_output_prefix = f"users/{identity_id}/{register_job_id}/registry_output"
80
- input_prefix = f"users/{identity_id}/{job_id}/supervised_input"
81
- output_prefix = f"users/{identity_id}/{job_id}/supervised_output"
82
-
83
- # Upload YAML
84
- yaml_key = f"{input_prefix}/config.yaml"
85
- s3.upload_file(str(yaml), bucket, yaml_key)
86
-
87
- # Upload state CSV if present (rounds > 0)
88
- state_key = ""
89
- if state and Path(state).exists():
90
- state_key = f"{input_prefix}/state.csv"
91
- s3.upload_file(str(state), bucket, state_key)
92
-
93
- # Register in DynamoDB before uploading trigger (state machine will update to RUNNING)
94
- table.put_item(Item={
95
- "user_id": identity_id,
96
- "job_id": job_id,
97
- "username": cfg["username"],
98
- "status": "SUBMITTED",
99
- "job_type": "supervised_query",
100
- "source_job_id": register_job_id,
101
- "round_number": round_number,
102
- "created_at": datetime.datetime.utcnow().isoformat(),
103
- "s3_path": f"s3://{bucket}/{output_prefix}",
104
- })
105
-
106
- # Upload trigger file last — EventBridge watches for this key and invokes the Lambda
107
- trigger = {
108
- "user_id": identity_id,
109
- "job_id": job_id,
110
- "s3_bucket": bucket,
111
- "registry_output_prefix": registry_output_prefix,
112
- "yaml_key": yaml_key,
113
- "state_key": state_key,
114
- "output_prefix": output_prefix,
115
- }
116
- s3.put_object(
117
- Bucket=bucket,
118
- Key=f"{input_prefix}/supervised.json",
119
- Body=json.dumps(trigger),
120
- ContentType="application/json",
121
- )
122
-
123
- print(Panel(
124
- f"[bold]Job ID:[/bold] {job_id}\n"
125
- f"[bold]Round:[/bold] {round_number}\n"
126
- f"[bold]Pool library:[/bold] {register_job_id}\n\n"
127
- f"[bold]Next steps:[/bold]\n"
128
- f" chemsift status {job_id}\n"
129
- f" chemsift download {job_id} .\n"
130
- f" # fill 'label' column in state.csv (0=inactive, 1=active)\n"
131
- f" chemsift supervised {register_job_id} --yaml {yaml.name} --state state.csv",
132
- title="[green]Supervised round submitted[/green]",
133
- expand=False,
134
- ))