workbench 0.8.162__py3-none-any.whl → 0.8.202__py3-none-any.whl

workbench/scripts/monitor_cloud_watch.py

@@ -4,8 +4,10 @@ import sys
 import time
 import argparse
 from datetime import datetime, timedelta, timezone
-from workbench.core.cloud_platform.aws.aws_account_clamp import AWSAccountClamp
+
+# Workbench Imports
 from workbench.utils.repl_utils import cprint, Spinner
+from workbench.utils.cloudwatch_utils import get_cloudwatch_client, get_active_log_streams, stream_log_events
 
 # Define the log levels to include all log levels above the specified level
 log_level_map = {
@@ -33,64 +35,6 @@ def date_display(dt):
         return dt.strftime("%Y-%m-%d %I:%M%p") + "(UTC)"
 
 
-def get_cloudwatch_client():
-    """Get the CloudWatch Logs client using the Workbench assumed role session."""
-    session = AWSAccountClamp().boto3_session
-    return session.client("logs")
-
-
-def get_active_log_streams(client, log_group_name, start_time_ms, stream_filter=None):
-    """Retrieve log streams that have events after the specified start time."""
-
-    # Get all the streams in the log group
-    active_streams = []
-    stream_params = {
-        "logGroupName": log_group_name,
-        "orderBy": "LastEventTime",
-        "descending": True,
-    }
-
-    # Loop to retrieve all log streams (maximum 50 per call)
-    while True:
-        response = client.describe_log_streams(**stream_params)
-        log_streams = response.get("logStreams", [])
-
-        for log_stream in log_streams:
-            log_stream_name = log_stream["logStreamName"]
-            last_event_timestamp = log_stream.get("lastEventTimestamp")
-
-            # Include streams with events since the specified start time
-            # Note: There's some issue where the last event timestamp is 'off'
-            #       so we're going to add 60 minutes from the last event timestamp
-            last_event_timestamp += 60 * 60 * 1000
-            if last_event_timestamp >= start_time_ms:
-                active_streams.append(log_stream_name)
-            else:
-                break  # Stop if we reach streams older than the start time
-
-        # Check if there are more streams to retrieve
-        if "nextToken" in response:
-            stream_params["nextToken"] = response["nextToken"]
-        else:
-            break
-
-    # Sort and report the active log streams
-    active_streams.sort()
-    if active_streams:
-        print("Active log streams:", len(active_streams))
-
-    # Filter the active streams by a substring if provided
-    if stream_filter and active_streams:
-        print(f"Filtering active log streams by '{stream_filter}'...")
-        active_streams = [stream for stream in active_streams if stream_filter in stream]
-
-    for stream in active_streams:
-        print(f"\t - {stream}")
-
-    # Return the active log streams
-    return active_streams
-
-
 def get_latest_log_events(client, log_group_name, start_time, end_time=None, stream_filter=None):
     """Retrieve the latest log events from the active/filtered log streams in a CloudWatch Logs group."""
 
@@ -99,11 +43,15 @@ def get_latest_log_events(client, log_group_name, start_time, end_time=None, str
         get_latest_log_events.first_run = True
 
     log_events = []
-    start_time_ms = int(start_time.timestamp() * 1000)  # Convert start_time to milliseconds
+    start_time_ms = int(start_time.timestamp() * 1000)
+
+    # Use the util function to get active streams
+    active_streams = get_active_log_streams(log_group_name, start_time_ms, stream_filter, client)
 
-    # Get the active log streams with events since start_time
-    active_streams = get_active_log_streams(client, log_group_name, start_time_ms, stream_filter)
     if active_streams:
+        print(f"Active log streams: {len(active_streams)}")
+        for stream in active_streams:
+            print(f"\t - {stream}")
         print(f"Processing log events from {date_display(start_time)} on {len(active_streams)} active log streams...")
         get_latest_log_events.first_run = False
     else:
@@ -114,50 +62,22 @@ def get_latest_log_events(client, log_group_name, start_time, end_time=None, str
             print("Monitoring for new events...")
         return log_events
 
-    # Iterate over the active streams and fetch log events
+    # Use the util function to stream events from each log stream
     for log_stream_name in active_streams:
-        params = {
-            "logGroupName": log_group_name,
-            "logStreamName": log_stream_name,
-            "startTime": start_time_ms,  # Use start_time in milliseconds
-            "startFromHead": True,  # Start from the nearest event to start_time
-        }
-        next_event_token = None
-        if end_time is not None:
-            params["endTime"] = int(end_time.timestamp() * 1000)
-
-        # Process the log events from this log stream
         spinner = Spinner("lightpurple", f"Pulling events from {log_stream_name}:")
         spinner.start()
         log_stream_events = 0
 
-        # Get the log events for the active log stream
-        while True:
-            if next_event_token:
-                params["nextToken"] = next_event_token
-                params.pop("startTime", None)  # Remove startTime when using nextToken
+        # Stream events using the util function
+        for event in stream_log_events(
+            log_group_name, log_stream_name, start_time, end_time, follow=False, client=client
+        ):
+            log_stream_events += 1
+            log_events.append(event)
 
-            # Fetch the log events (this call takes a while: optimize if we can)
-            events_response = client.get_log_events(**params)
+        spinner.stop()
+        print(f"Processed {log_stream_events} events from {log_stream_name} (Total: {len(log_events)})")
 
-            events = events_response.get("events", [])
-            for event in events:
-                event["logStreamName"] = log_stream_name
-
-            # Add the log stream events to our list of all log events
-            log_stream_events += len(events)
-            log_events.extend(events)
-
-            # Handle pagination for log events
-            next_event_token = events_response.get("nextForwardToken")
-
-            # Break the loop if there are no more events to fetch
-            if not next_event_token or next_event_token == params.get("nextToken"):
-                spinner.stop()
-                print(f"Processed {log_stream_events} events from {log_stream_name} (Total: {len(log_events)})")
-                break
-
-    # Return the log events
     return log_events
 
 
@@ -206,6 +126,7 @@ def monitor_log_group(
     print(f"Monitoring log group: {log_group_name} from {date_display(start_time)}")
     print(f"Log levels: {log_levels}")
     print(f"Search terms: {search_terms}")
+
     while True:
         # Get the latest log events with stream filtering if provided
         all_log_events = get_latest_log_events(client, log_group_name, start_time, end_time, stream_filter)
@@ -218,7 +139,6 @@ def monitor_log_group(
 
                 # Check the search terms
                 if not search_terms or any(term in event["message"].lower() for term in search_terms):
-
                     # Calculate the start and end index for this match
                     start_index = max(i - before, 0)
                     end_index = min(i + after, len(all_log_events) - 1)

workbench/utils/aws_utils.py

@@ -55,7 +55,8 @@ def aws_throttle(func=None, retry_intervals=None):
     if func is None:
         return lambda f: aws_throttle(f, retry_intervals=retry_intervals)
 
-    service_hold_time = 2  # Seconds to wait before calling AWS function
+    # This is currently commented out (we might want to use it later)
+    # service_hold_time = 2  # Seconds to wait before calling AWS function
     default_intervals = [2**i for i in range(1, 9)]  # Default exponential backoff: 2, 4, 8... 256 seconds
     intervals = retry_intervals or default_intervals
 
@@ -64,8 +65,8 @@ def aws_throttle(func=None, retry_intervals=None):
         for attempt, delay in enumerate(intervals, start=1):
             try:
                 # Add sleep before calling AWS func if running as a service
-                if cm.running_as_service:
-                    time.sleep(service_hold_time)
+                # if cm.running_as_service:
+                #    time.sleep(service_hold_time)
                 return func(*args, **kwargs)
             except ClientError as e:
                 if e.response["Error"]["Code"] == "ThrottlingException":

workbench/utils/chem_utils/fingerprints.py

@@ -0,0 +1,134 @@
+"""Molecular fingerprint computation utilities"""
+
+import logging
+import pandas as pd
+
+# Molecular Descriptor Imports
+from rdkit import Chem
+from rdkit.Chem import rdFingerprintGenerator
+from rdkit.Chem.MolStandardize import rdMolStandardize
+
+# Set up the logger
+log = logging.getLogger("workbench")
+
+
+def compute_morgan_fingerprints(df: pd.DataFrame, radius=2, n_bits=2048, counts=True) -> pd.DataFrame:
+    """Compute and add Morgan fingerprints to the DataFrame.
+
+    Args:
+        df (pd.DataFrame): Input DataFrame containing SMILES strings.
+        radius (int): Radius for the Morgan fingerprint.
+        n_bits (int): Number of bits for the fingerprint.
+        counts (bool): Count simulation for the fingerprint.
+
+    Returns:
+        pd.DataFrame: The input DataFrame with the Morgan fingerprints added as bit strings.
+
+    Note:
+        See: https://greglandrum.github.io/rdkit-blog/posts/2021-07-06-simulating-counts.html
+    """
+    delete_mol_column = False
+
+    # Check for the SMILES column (case-insensitive)
+    smiles_column = next((col for col in df.columns if col.lower() == "smiles"), None)
+    if smiles_column is None:
+        raise ValueError("Input DataFrame must have a 'smiles' column")
+
+    # Sanity check the molecule column (sometimes it gets serialized, which doesn't work)
+    if "molecule" in df.columns and df["molecule"].dtype == "string":
+        log.warning("Detected serialized molecules in 'molecule' column. Removing...")
+        del df["molecule"]
+
+    # Convert SMILES to RDKit molecule objects (vectorized)
+    if "molecule" not in df.columns:
+        log.info("Converting SMILES to RDKit Molecules...")
+        delete_mol_column = True
+        df["molecule"] = df[smiles_column].apply(Chem.MolFromSmiles)
+        # Make sure our molecules are not None
+        failed_smiles = df[df["molecule"].isnull()][smiles_column].tolist()
+        if failed_smiles:
+            log.error(f"Failed to convert the following SMILES to molecules: {failed_smiles}")
+        df = df.dropna(subset=["molecule"])
+
+    # If we have fragments in our compounds, get the largest fragment before computing fingerprints
+    largest_frags = df["molecule"].apply(
+        lambda mol: rdMolStandardize.LargestFragmentChooser().choose(mol) if mol else None
+    )
+
+    # Create a Morgan fingerprint generator
+    if counts:
+        n_bits *= 4  # Multiply by 4 to simulate counts
+    morgan_generator = rdFingerprintGenerator.GetMorganGenerator(radius=radius, fpSize=n_bits, countSimulation=counts)
+
+    # Compute Morgan fingerprints (vectorized)
+    fingerprints = largest_frags.apply(
+        lambda mol: (morgan_generator.GetFingerprint(mol).ToBitString() if mol else pd.NA)
+    )
+
+    # Add the fingerprints to the DataFrame
+    df["fingerprint"] = fingerprints
+
+    # Drop the intermediate 'molecule' column if it was added
+    if delete_mol_column:
+        del df["molecule"]
+    return df
+
+
+if __name__ == "__main__":
+    print("Running molecular fingerprint tests...")
+    print("Note: This requires molecular_screening module to be available")
+
+    # Test molecules
+    test_molecules = {
+        "aspirin": "CC(=O)OC1=CC=CC=C1C(=O)O",
+        "caffeine": "CN1C=NC2=C1C(=O)N(C(=O)N2C)C",
+        "glucose": "C([C@@H]1[C@H]([C@@H]([C@H](C(O1)O)O)O)O)O",  # With stereochemistry
+        "sodium_acetate": "CC(=O)[O-].[Na+]",  # Salt
+        "benzene": "c1ccccc1",
+        "butene_e": "C/C=C/C",  # E-butene
+        "butene_z": "C/C=C\\C",  # Z-butene
+    }
+
+    # Test 1: Morgan Fingerprints
+    print("\n1. Testing Morgan fingerprint generation...")
+
+    test_df = pd.DataFrame({"SMILES": list(test_molecules.values()), "name": list(test_molecules.keys())})
+
+    fp_df = compute_morgan_fingerprints(test_df.copy(), radius=2, n_bits=512, counts=False)
+
+    print("   Fingerprint generation results:")
+    for _, row in fp_df.iterrows():
+        fp = row.get("fingerprint", "N/A")
+        fp_len = len(fp) if fp != "N/A" else 0
+        print(f"   {row['name']:15} → {fp_len} bits")
+
+    # Test 2: Different fingerprint parameters
+    print("\n2. Testing different fingerprint parameters...")
+
+    # Test with counts enabled
+    fp_counts_df = compute_morgan_fingerprints(test_df.copy(), radius=3, n_bits=256, counts=True)
+
+    print("   With count simulation (256 bits * 4):")
+    for _, row in fp_counts_df.iterrows():
+        fp = row.get("fingerprint", "N/A")
+        fp_len = len(fp) if fp != "N/A" else 0
+        print(f"   {row['name']:15} → {fp_len} bits")
+
+    # Test 3: Edge cases
+    print("\n3. Testing edge cases...")
+
+    # Invalid SMILES
+    invalid_df = pd.DataFrame({"SMILES": ["INVALID", ""]})
+    try:
+        fp_invalid = compute_morgan_fingerprints(invalid_df.copy())
+        print(f"   ✓ Invalid SMILES handled: {len(fp_invalid)} valid molecules")
+    except Exception as e:
+        print(f"   ✓ Invalid SMILES properly raised error: {type(e).__name__}")
+
+    # Test with pre-existing molecule column
+    mol_df = test_df.copy()
+    mol_df["molecule"] = mol_df["SMILES"].apply(Chem.MolFromSmiles)
+    fp_with_mol = compute_morgan_fingerprints(mol_df)
+    print(f"   ✓ Pre-existing molecule column handled: {len(fp_with_mol)} fingerprints generated")
+
+    print("\n✅ All fingerprint tests completed!")

workbench/utils/chem_utils/misc.py

@@ -0,0 +1,194 @@
+"""Miscellaneous processing functions for molecular data."""
+
+import logging
+import numpy as np
+import pandas as pd
+from typing import List, Optional
+
+# Set up the logger
+log = logging.getLogger("workbench")
+
+
+def geometric_mean(series: pd.Series) -> float:
+    """Computes the geometric mean manually to avoid using scipy."""
+    return np.exp(np.log(series).mean())
+
+
+def rollup_experimental_data(
+    df: pd.DataFrame, id: str, time: str, target: str, use_gmean: bool = False
+) -> pd.DataFrame:
+    """
+    Rolls up a dataset by selecting the largest time per unique ID and averaging the target value
+    if multiple records exist at that time. Supports both arithmetic and geometric mean.
+
+    Parameters:
+        df (pd.DataFrame): Input dataframe.
+        id (str): Column representing the unique molecule ID.
+        time (str): Column representing the time.
+        target (str): Column representing the target value.
+        use_gmean (bool): Whether to use the geometric mean instead of the arithmetic mean.
+
+    Returns:
+        pd.DataFrame: Rolled-up dataframe with all original columns retained.
+    """
+    # Find the max time per unique ID
+    max_time_df = df.groupby(id)[time].transform("max")
+    filtered_df = df[df[time] == max_time_df]
+
+    # Define aggregation function
+    agg_func = geometric_mean if use_gmean else np.mean
+
+    # Perform aggregation on all columns
+    agg_dict = {col: "first" for col in df.columns if col not in [target, id, time]}
+    agg_dict[target] = lambda x: agg_func(x) if len(x) > 1 else x.iloc[0]  # Apply mean or gmean
+
+    rolled_up_df = filtered_df.groupby([id, time]).agg(agg_dict).reset_index()
+    return rolled_up_df
+
+
+def micromolar_to_log(series_µM: pd.Series) -> pd.Series:
+    """
+    Convert a pandas Series of concentrations in µM (micromolar) to their logarithmic values (log10).
+
+    Parameters:
+    series_uM (pd.Series): Series of concentrations in micromolar.
+
+    Returns:
+    pd.Series: Series of logarithmic values (log10).
+    """
+    # Replace 0 or negative values with a small number to avoid log errors
+    adjusted_series = series_µM.clip(lower=1e-9)  # Alignment with another project
+
+    series_mol_per_l = adjusted_series * 1e-6  # Convert µM/L to mol/L
+    log_series = np.log10(series_mol_per_l)
+    return log_series
+
+
+def log_to_micromolar(log_series: pd.Series) -> pd.Series:
+    """
+    Convert a pandas Series of logarithmic values (log10) back to concentrations in µM (micromolar).
+
+    Parameters:
+    log_series (pd.Series): Series of logarithmic values (log10).
+
+    Returns:
+    pd.Series: Series of concentrations in micromolar.
+    """
+    series_mol_per_l = 10**log_series  # Convert log10 back to mol/L
+    series_µM = series_mol_per_l * 1e6  # Convert mol/L to µM
+    return series_µM
+
+
+def feature_resolution_issues(df: pd.DataFrame, features: List[str], show_cols: Optional[List[str]] = None) -> None:
+    """
+    Identify and print groups in a DataFrame where the given features have more than one unique SMILES,
+    sorted by group size (largest number of unique SMILES first).
+
+    Args:
+        df (pd.DataFrame): Input DataFrame containing SMILES strings.
+        features (List[str]): List of features to check.
+        show_cols (Optional[List[str]]): Columns to display; defaults to all columns.
+    """
+    # Check for the 'smiles' column (case-insensitive)
+    smiles_column = next((col for col in df.columns if col.lower() == "smiles"), None)
+    if smiles_column is None:
+        raise ValueError("Input DataFrame must have a 'smiles' column")
+
+    show_cols = show_cols if show_cols is not None else df.columns.tolist()
+
+    # Drop duplicates to keep only unique SMILES for each feature combination
+    unique_df = df.drop_duplicates(subset=[smiles_column] + features)
+
+    # Find groups with more than one unique SMILES
+    group_counts = unique_df.groupby(features).size()
+    collision_groups = group_counts[group_counts > 1].sort_values(ascending=False)
+
+    # Print each group in order of size (largest first)
+    for group, count in collision_groups.items():
+        # Get the rows for this group
+        if isinstance(group, tuple):
+            group_mask = (unique_df[features] == group).all(axis=1)
+        else:
+            group_mask = unique_df[features[0]] == group
+
+        group_df = unique_df[group_mask]
+
+        print(f"Feature Group (unique SMILES: {count}):")
+        print(group_df[show_cols])
+        print("\n")
+
+
+if __name__ == "__main__":
+    print("Running molecular processing and transformation tests...")
+    print("Note: This requires the molecular_filters module to be available")
+
+    # Test 1: Concentration conversions
+    print("\n1. Testing concentration conversions...")
+
+    # Test micromolar to log
+    test_conc = pd.Series([1.0, 10.0, 100.0, 1000.0, 0.001])
+    log_values = micromolar_to_log(test_conc)
+    back_to_uM = log_to_micromolar(log_values)
+
+    print("   µM → log10 → µM:")
+    for orig, log_val, back in zip(test_conc, log_values, back_to_uM):
+        print(f"   {orig:8.3f} µM → {log_val:6.2f} → {back:8.3f} µM")
+
+    # Test 2: Geometric mean
+    print("\n2. Testing geometric mean...")
+    test_series = pd.Series([2, 4, 8, 16])
+    geo_mean = geometric_mean(test_series)
+    arith_mean = np.mean(test_series)
+    print(f"   Series: {list(test_series)}")
+    print(f"   Arithmetic mean: {arith_mean:.2f}")
+    print(f"   Geometric mean: {geo_mean:.2f}")
+
+    # Test 3: Experimental data rollup
+    print("\n3. Testing experimental data rollup...")
+
+    # Create test data with multiple timepoints and replicates
+    test_data = pd.DataFrame(
+        {
+            "compound_id": ["A", "A", "A", "B", "B", "C", "C", "C"],
+            "time": [1, 2, 2, 1, 2, 1, 1, 2],
+            "activity": [10, 20, 22, 5, 8, 100, 110, 200],
+            "assay": ["kinase", "kinase", "kinase", "kinase", "kinase", "cell", "cell", "cell"],
+        }
+    )
+
+    # Rollup with arithmetic mean
+    rolled_arith = rollup_experimental_data(test_data, "compound_id", "time", "activity", use_gmean=False)
+    print("   Arithmetic mean rollup:")
+    print(rolled_arith[["compound_id", "time", "activity"]])
+
+    # Rollup with geometric mean
+    rolled_geo = rollup_experimental_data(test_data, "compound_id", "time", "activity", use_gmean=True)
+    print("\n   Geometric mean rollup:")
+    print(rolled_geo[["compound_id", "time", "activity"]])
+
+    # Test 4: Feature resolution issues
+    print("\n4. Testing feature resolution identification...")
+
+    # Create data with some duplicate features but different SMILES
+    resolution_df = pd.DataFrame(
+        {
+            "smiles": ["CCO", "C(C)O", "CC(C)O", "CCC(C)O", "CCCO"],
+            "assay_id": ["A1", "A1", "A2", "A2", "A3"],
+            "value": [1.0, 1.5, 2.0, 2.2, 3.0],
+        }
+    )
+
+    print("   Checking for feature collisions in 'assay_id':")
+    feature_resolution_issues(resolution_df, ["assay_id"], show_cols=["smiles", "assay_id", "value"])
+
+    # Test 7: Edge cases
+    print("\n7. Testing edge cases...")
+
+    # Zero and negative concentrations
+    edge_conc = pd.Series([0, -1, 1e-10])
+    edge_log = micromolar_to_log(edge_conc)
+    print("   Edge concentration handling:")
+    for c, l in zip(edge_conc, edge_log):
+        print(f"      {c:6.2e} µM → {l:6.2f}")
+
+    print("\n✅ All molecular processing tests completed!")