workbench 0.8.162__py3-none-any.whl → 0.8.220__py3-none-any.whl

workbench/utils/execution_environment.py

@@ -1,211 +0,0 @@
-"""ExecutionEnvironment provides logic/functionality to figure out the current execution environment"""
-
-import os
-import sys
-import logging
-import requests
-from typing import Union
-import boto3
-from datetime import datetime, timezone
-
-# Workbench imports
-from workbench.utils.glue_utils import get_resolved_options
-from workbench.utils.deprecated_utils import deprecated
-
-# Set up the logger
-log = logging.getLogger("workbench")
-
-
-def running_on_glue():
-    """
-    Check if the current execution environment is an AWS Glue job.
-
-    Returns:
-        bool: True if running in AWS Glue environment, False otherwise.
-    """
-    # Check if GLUE_VERSION or GLUE_PYTHON_VERSION is in the environment
-    if "GLUE_VERSION" in os.environ or "GLUE_PYTHON_VERSION" in os.environ:
-        log.info("Running in AWS Glue Environment...")
-        return True
-    else:
-        return False
-
-
-def running_on_lambda():
-    """
-    Check if the current execution environment is an AWS Lambda function.
-
-    Returns:
-        bool: True if running in AWS Lambda environment, False otherwise.
-    """
-    if "AWS_LAMBDA_FUNCTION_NAME" in os.environ:
-        log.info("Running in AWS Lambda Environment...")
-        return True
-    else:
-        return False
-
-
-def running_on_docker() -> bool:
-    """Check if the current environment is running on a Docker container.
-
-    Returns:
-        bool: True if running in a Docker container, False otherwise.
-    """
-    try:
-        # Docker creates a .dockerenv file at the root of the directory tree inside the container.
-        # If this file exists, it is very likely that we are running inside a Docker container.
-        with open("/.dockerenv") as f:
-            return True
-    except FileNotFoundError:
-        pass
-
-    try:
-        # Another method is to check the contents of /proc/self/cgroup which should be different
-        # inside a Docker container.
-        with open("/proc/self/cgroup") as f:
-            if any("docker" in line for line in f):
-                return True
-    except FileNotFoundError:
-        pass
-
-    # Check if we are running on ECS
-    if running_on_ecs():
-        return True
-
-    # Probably not running in a Docker container
-    return False
-
-
-def running_on_ecs() -> bool:
-    """
-    Check if the current environment is running on AWS ECS.
-
-    Returns:
-        bool: True if running on AWS ECS, False otherwise.
-    """
-    indicators = [
-        "ECS_SERVICE_NAME",
-        "ECS_CONTAINER_METADATA_URI",
-        "ECS_CONTAINER_METADATA_URI_V4",
-        "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
-        "AWS_EXECUTION_ENV",
-    ]
-    return any(indicator in os.environ for indicator in indicators)
-
-
-def running_as_service() -> bool:
-    """
-    Check if the current environment is running as a service (e.g. Docker, ECS, Glue, Lambda).
-
-    Returns:
-        bool: True if running as a service, False otherwise.
-    """
-    return running_on_docker() or running_on_glue() or running_on_lambda()
-
-
-def _glue_job_from_script_name(args):
-    """Get the Glue Job Name from the script name"""
-    try:
-        script_name = args["scriptLocation"]
-        return os.path.splitext(os.path.basename(script_name))[0]
-    except Exception:
-        return "unknown"
-
-
-def glue_job_name():
-    """Get the Glue Job Name from the environment or script name"""
-    # Define the required argument
-    args = get_resolved_options(sys.argv)
-
-    # Get the job name
-    job_name = args.get("JOB_NAME") or _glue_job_from_script_name(args)
-    return job_name
-
-
-@deprecated(version=0.9)
-def glue_job_run_id(job_name: str, session: boto3.Session) -> Union[str, None]:
-    """Retrieve the Glue Job Run ID closest to the current time for the given job name.
-    Note: This mostly doesn't work, it will grab A glue job id but often not the correct one.
-          For now, I would just skip using this
-    """
-    try:
-        # Set current time in UTC
-        current_time = datetime.now(timezone.utc)
-
-        job_runs = session.client("glue").get_job_runs(JobName=job_name)
-        if job_runs["JobRuns"]:
-            # Find the job run with the StartedOn time closest to the current time
-            closest_job_run = min(job_runs["JobRuns"], key=lambda run: abs(run["StartedOn"] - current_time))
-            job_id = closest_job_run["Id"]
-            return job_id[:9]  # Shorten the Job Run ID to 9 characters
-
-        log.error(f"No runs found for Glue Job '{job_name}', returning None for Job Run ID.")
-        return None
-
-    except session.client("glue").exceptions.EntityNotFoundException:
-        log.error(f"Glue Job '{job_name}' not found, returning None for Job Run ID.")
-        return None
-    except Exception as e:
-        log.error(f"An error occurred while retrieving job run ID: {e}")
-        return None
-
-
-def ecs_job_name():
-    """Get the ECS Job Name from the metadata endpoint or environment variables."""
-    # Attempt to get the job name from ECS metadata
-    ecs_metadata_uri = os.environ.get("ECS_CONTAINER_METADATA_URI_V4")
-
-    if ecs_metadata_uri:
-        try:
-            response = requests.get(f"{ecs_metadata_uri}/task")
-            if response.status_code == 200:
-                metadata = response.json()
-                job_name = metadata.get("Family")  # 'Family' represents the ECS task definition family name
-                if job_name:
-                    return job_name
-        except requests.RequestException as e:
-            # Log the error or handle it as needed
-            log.error(f"Failed to fetch ECS metadata: {e}")
-
-    # Fallback to environment variables if metadata is not available
-    job_name = os.environ.get("ECS_SERVICE_NAME", "unknown")
-    return job_name
-
-
-if __name__ == "__main__":
-    """Test the Execution Environment utilities"""
-
-    # Test running_on_glue
-    assert running_on_glue() is False
-    os.environ["GLUE_VERSION"] = "1.0"
-    assert running_on_glue() is True
-    del os.environ["GLUE_VERSION"]
-
-    # Test running_on_lambda
-    assert running_on_lambda() is False
-    os.environ["AWS_LAMBDA_FUNCTION_NAME"] = "my_lambda_function"
-    assert running_on_lambda() is True
-    del os.environ["AWS_LAMBDA_FUNCTION_NAME"]
-
-    # Test running_on_docker
-    assert running_on_docker() is False
-    os.environ["ECS_CONTAINER_METADATA_URI"] = "http://localhost:8080"
-    assert running_on_docker() is True
-    del os.environ["ECS_CONTAINER_METADATA_URI"]
-
-    # Test running_on_ecs
-    assert running_on_ecs() is False
-    os.environ["ECS_CONTAINER_METADATA_URI"] = "http://localhost:8080"
-    assert running_on_ecs() is True
-    del os.environ["ECS_CONTAINER_METADATA_URI"]
-
-    # Test getting the Glue Job Name
-    print(glue_job_name())
-
-    # Test getting the Glue Job Run ID
-    from workbench.core.cloud_platform.aws.aws_session import AWSSession
-
-    session = AWSSession().boto3_session
-    print(glue_job_run_id("Test_Workbench_Shell", session))
-
-    print("All tests passed!")

workbench/utils/fast_inference.py

@@ -1,167 +0,0 @@
-"""Fast Inference on SageMaker Endpoints"""
-
-import pandas as pd
-from io import StringIO
-import logging
-from concurrent.futures import ThreadPoolExecutor
-
-# Sagemaker Imports
-import sagemaker
-from sagemaker.serializers import CSVSerializer
-from sagemaker.deserializers import CSVDeserializer
-from sagemaker import Predictor
-
-log = logging.getLogger("workbench")
-
-_CACHED_SM_SESSION = None
-
-
-def get_or_create_sm_session():
-    global _CACHED_SM_SESSION
-    if _CACHED_SM_SESSION is None:
-        _CACHED_SM_SESSION = sagemaker.Session()
-    return _CACHED_SM_SESSION
-
-
-def fast_inference(endpoint_name: str, eval_df: pd.DataFrame, sm_session=None, threads: int = 4) -> pd.DataFrame:
-    """Run inference on the Endpoint using the provided DataFrame
-
-    Args:
-        endpoint_name (str): The name of the Endpoint
-        eval_df (pd.DataFrame): The DataFrame to run predictions on
-        sm_session (sagemaker.session.Session, optional): SageMaker Session. If None, a cached session is created.
-        threads (int): The number of threads to use (default: 4)
-
-    Returns:
-        pd.DataFrame: The DataFrame with predictions
-    """
-    # Use cached session if none is provided
-    if sm_session is None:
-        sm_session = get_or_create_sm_session()
-
-    predictor = Predictor(
-        endpoint_name,
-        sagemaker_session=sm_session,
-        serializer=CSVSerializer(),
-        deserializer=CSVDeserializer(),
-    )
-
-    total_rows = len(eval_df)
-
-    def process_chunk(chunk_df: pd.DataFrame, start_index: int) -> pd.DataFrame:
-        log.info(f"Processing {start_index}:{min(start_index + chunk_size, total_rows)} out of {total_rows} rows...")
-        csv_buffer = StringIO()
-        chunk_df.to_csv(csv_buffer, index=False)
-        response = predictor.predict(csv_buffer.getvalue())
-        # CSVDeserializer returns a nested list: first row is headers
-        return pd.DataFrame.from_records(response[1:], columns=response[0])
-
-    # Sagemaker has a connection pool limit of 10
-    if threads > 10:
-        log.warning("Sagemaker has a connection pool limit of 10. Reducing threads to 10.")
-        threads = 10
-
-    # Compute the chunk size (divide number of threads)
-    chunk_size = max(1, total_rows // threads)
-
-    # We also need to ensure that the chunk size is not too big
-    if chunk_size > 100:
-        chunk_size = 100
-
-    # Split DataFrame into chunks and process them concurrently
-    chunks = [(eval_df[i : i + chunk_size], i) for i in range(0, total_rows, chunk_size)]
-    with ThreadPoolExecutor(max_workers=threads) as executor:
-        df_list = list(executor.map(lambda p: process_chunk(*p), chunks))
-
-    combined_df = pd.concat(df_list, ignore_index=True)
-
-    # Convert the types of the dataframe
-    combined_df = df_type_conversions(combined_df)
-    return combined_df
-
-
-def df_type_conversions(df: pd.DataFrame) -> pd.DataFrame:
-    """Convert the types of the dataframe that we get from an endpoint
-
-    Args:
-        df (pd.DataFrame): DataFrame to convert
-
-    Returns:
-        pd.DataFrame: Converted DataFrame
-    """
-    # Some endpoints will put in "N/A" values (for CSV serialization)
-    # We need to convert these to NaN and the run the conversions below
-    # Report on the number of N/A values in each column in the DataFrame
-    # For any count above 0 list the column name and the number of N/A values
-    na_counts = df.isin(["N/A"]).sum()
-    for column, count in na_counts.items():
-        if count > 0:
-            log.warning(f"{column} has {count} N/A values, converting to NaN")
-    pd.set_option("future.no_silent_downcasting", True)
-    df = df.replace("N/A", float("nan"))
-
-    # Convert data to numeric
-    # Note: Since we're using CSV serializers numeric columns often get changed to generic 'object' types
-
-    # Hard Conversion
-    # Note: We explicitly catch exceptions for columns that cannot be converted to numeric
-    for column in df.columns:
-        try:
-            df[column] = pd.to_numeric(df[column])
-        except ValueError:
-            # If a ValueError is raised, the column cannot be converted to numeric, so we keep it as is
-            pass
-        except TypeError:
-            # This typically means a duplicated column name, so confirm duplicate (more than 1) and log it
-            column_count = (df.columns == column).sum()
-            log.critical(f"{column} occurs {column_count} times in the DataFrame.")
-            pass
-
-    # Soft Conversion
-    # Convert columns to the best possible dtype that supports the pd.NA missing value.
-    df = df.convert_dtypes()
-
-    # Convert pd.NA placeholders to pd.NA
-    # Note: CSV serialization converts pd.NA to blank strings, so we have to put in placeholders
-    df.replace("__NA__", pd.NA, inplace=True)
-
-    # Check for True/False values in the string columns
-    for column in df.select_dtypes(include=["string"]).columns:
-        if df[column].str.lower().isin(["true", "false"]).all():
-            df[column] = df[column].str.lower().map({"true": True, "false": False})
-
-    # Return the Dataframe
-    return df
-
-
-if __name__ == "__main__":
-    """Exercise the Endpoint Utilities"""
-    import time
-    from workbench.api.endpoint import Endpoint
-    from workbench.utils.endpoint_utils import fs_training_data, fs_evaluation_data
-
-    # Create an Endpoint
-    my_endpoint_name = "abalone-regression"
-    my_endpoint = Endpoint(my_endpoint_name)
-    if not my_endpoint.exists():
-        print(f"Endpoint {my_endpoint_name} does not exist.")
-        exit(1)
-
-    # Get the training data
-    my_train_df = fs_training_data(my_endpoint)
-    print(my_train_df)
-
-    # Run Fast Inference and time it
-    my_sm_session = my_endpoint.sm_session
-    my_eval_df = fs_evaluation_data(my_endpoint)
-    start_time = time.time()
-    my_results_df = fast_inference(my_endpoint_name, my_eval_df, my_sm_session)
-    end_time = time.time()
-    print(f"Fast Inference took {end_time - start_time} seconds")
-    print(my_results_df)
-    print(my_results_df.info())
-
-    # Test with no session
-    my_results_df = fast_inference(my_endpoint_name, my_eval_df)
-    print(my_results_df)
-    print(my_results_df.info())

workbench/utils/resource_utils.py

@@ -1,39 +0,0 @@
-"""Resource utilities for Workbench"""
-
-import sys
-import importlib.resources as resources
-import pathlib
-import pkg_resources
-
-
-def get_resource_path(package: str, resource: str) -> pathlib.Path:
-    """Get the path to a resource file, compatible with Python 3.9 and higher.
-
-    Args:
-        package (str): The package where the resource is located.
-        resource (str): The name of the resource file.
-
-    Returns:
-        pathlib.Path: The path to the resource file.
-    """
-    if sys.version_info >= (3, 10):
-        # Python 3.10 and higher: use importlib.resources.path
-        with resources.path(package, resource) as path:
-            return path
-    else:
-        # Python 3.9 and lower: manually construct the path based on package location
-        # Get the location of the installed package
-        package_location = pathlib.Path(pkg_resources.get_distribution(package.split(".")[0]).location)
-        resource_path = package_location / package.replace(".", "/") / resource
-
-        if resource_path.exists():
-            return resource_path
-        else:
-            raise FileNotFoundError(f"Resource '{resource}' not found in package '{package}'.")
-
-
-if __name__ == "__main__":
-    # Test the resource utilities
-    with get_resource_path("workbench.resources", "open_source_api.key") as open_source_key_path:
-        with open(open_source_key_path, "r") as key_file:
-            print(key_file.read().strip())

workbench-0.8.162.dist-info/entry_points.txt

@@ -1,5 +0,0 @@
-[console_scripts]
-cloud_watch = workbench.scripts.monitor_cloud_watch:main
-glue_launcher = workbench.scripts.glue_launcher:main
-workbench = workbench.repl.workbench_shell:launch_shell
-workbench_config = workbench.scripts.show_config:main