aibs-informatics-aws-utils 0.0.2__tar.gz

aibs_informatics_aws_utils-0.0.2/LICENSE

@@ -0,0 +1,33 @@
+Allen Institute Software License – This software license is the 2-clause BSD 
+license plus a third clause that prohibits redistribution and use for 
+commercial purposes without further permission. 
+
+Copyright © 2024. Allen Institute.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without 
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this 
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, 
+this list of conditions and the following disclaimer in the documentation 
+and/or other materials provided with the distribution.
+
+3. Redistributions and use for commercial purposes are not permitted without 
+the Allen Institute’s written permission. For purposes of this license, 
+commercial purposes are the incorporation of the Allen Institute's software 
+into anything for which you will charge fees or other compensation or use of 
+the software to perform a commercial service for a third party. Contact 
+terms@alleninstitute.org for commercial licensing opportunities.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

aibs_informatics_aws_utils-0.0.2/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.1
+Name: aibs-informatics-aws-utils
+Version: 0.0.2
+Summary: Library of AWS utility code for informatics projects at the Allen Institute for Brain Science
+Project-URL: Documentation, https://.github.io/aibs-informatics-aws-utils/
+Project-URL: Homepage, https://github.com/AllenInstitute/aibs-informatics-aws-utils/
+Project-URL: Issues, https://github.com/AllenInstitute/aibs-informatics-aws-utils/issues
+Project-URL: Repository, https://github.com/AllenInstitute/aibs-informatics-aws-utils/
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: boto3~=1.35
+Requires-Dist: aibs-informatics-core
+Provides-Extra: dev
+Requires-Dist: aibs-informatics-test-resources; extra == "dev"
+Requires-Dist: boto3-stubs[apigateway,athena,batch,ecr,ecs,efs,essential,fsx,logs,secretsmanager,ses,sns,ssm,stepfunctions,sts]; extra == "dev"
+Requires-Dist: moto[all]~=5.0; extra == "dev"
+Provides-Extra: release
+Requires-Dist: build; extra == "release"
+Requires-Dist: bump-my-version; extra == "release"
+Requires-Dist: wheel; extra == "release"
+
+# AIBS Informatics AWS Utils
+
+[![Build Status](https://github.com/AllenInstitute/aibs-informatics-aws-utils/actions/workflows/build.yml/badge.svg)](https://github.com/AllenInstitute/aibs-informatics-aws-utils/actions/workflows/build.yml)
+
+---
+
+## Overview
+
+The AIBS Informatics AWS Utils library provides a collection of utilities and tools for working with AWS services. This library includes functionalities for interacting with AWS S3, ECR, Lambda, and other AWS services, making it easier to integrate AWS capabilities into various projects at the Allen Institute for Brain Science.
+
+## Contributing
+
+Any and all PRs are welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for more information.
+
+## Licensing
+
+This software is licensed under the Allen Institute Software License, which is the 2-clause BSD license plus a third clause that prohibits redistribution and use for commercial purposes without further permission. For more information, please visit [Allen Institute Terms of Use](https://alleninstitute.org/terms-of-use/).

aibs_informatics_aws_utils-0.0.2/README.md

@@ -0,0 +1,17 @@
+# AIBS Informatics AWS Utils
+
+[![Build Status](https://github.com/AllenInstitute/aibs-informatics-aws-utils/actions/workflows/build.yml/badge.svg)](https://github.com/AllenInstitute/aibs-informatics-aws-utils/actions/workflows/build.yml)
+
+---
+
+## Overview
+
+The AIBS Informatics AWS Utils library provides a collection of utilities and tools for working with AWS services. This library includes functionalities for interacting with AWS S3, ECR, Lambda, and other AWS services, making it easier to integrate AWS capabilities into various projects at the Allen Institute for Brain Science.
+
+## Contributing
+
+Any and all PRs are welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for more information.
+
+## Licensing
+
+This software is licensed under the Allen Institute Software License, which is the 2-clause BSD license plus a third clause that prohibits redistribution and use for commercial purposes without further permission. For more information, please visit [Allen Institute Terms of Use](https://alleninstitute.org/terms-of-use/).

aibs_informatics_aws_utils-0.0.2/pyproject.toml

@@ -0,0 +1,240 @@
+# -----------------------------------------------------------------------------
+##  Build System Configurations
+#   https://setuptools.pypa.io/en/latest/setuptools.html#building-and-distributing-packages-with-setuptools
+# -----------------------------------------------------------------------------
+
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+
+# -----------------------------------------------------------------------------
+[project]
+name = "aibs-informatics-aws-utils"
+description = "Library of AWS utility code for informatics projects at the Allen Institute for Brain Science"
+readme = "README.md"
+dynamic = [ "version"]
+requires-python = ">=3.9"
+
+dependencies = [
+    "boto3~=1.35",
+    "aibs-informatics-core",
+]
+
+[project.optional-dependencies]
+dev = [
+    "aibs-informatics-test-resources",
+    "boto3-stubs[athena,apigateway,batch,ecr,ecs,efs,essential,fsx,logs,secretsmanager,ses,sns,ssm,sts,stepfunctions]",
+    "moto[all] ~= 5.0",
+]
+
+release = [
+    "build",
+    "bump-my-version",
+    "wheel",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+"*" = ['py.typed']
+
+[tool.setuptools.dynamic]
+version = {attr = "aibs_informatics_aws_utils._version.__version__"}
+
+[project.urls]
+Documentation = "https://.github.io/aibs-informatics-aws-utils/"
+Homepage = "https://github.com/AllenInstitute/aibs-informatics-aws-utils/"
+Issues = "https://github.com/AllenInstitute/aibs-informatics-aws-utils/issues"
+Repository = "https://github.com/AllenInstitute/aibs-informatics-aws-utils/"
+
+
+# -----------------------------------------------------------------------------
+##  Pyright Configurations
+#   https://github.com/microsoft/pyright/blob/main/docs/configuration.md
+# -----------------------------------------------------------------------------
+
+[tool.pyright]
+reportGeneralTypeIssues = false
+typeCheckingMode = "basic"
+
+# -----------------------------------------------------------------------------
+##  Coverage Configurations 
+#   https://coverage.readthedocs.io/en/7.0.4/config.html#
+# -----------------------------------------------------------------------------
+
+[tool.coverage]
+# Note: we use pytest-cov to generate coverage reports
+#       when running tests. but these coverage configs
+#       are read when doing so.  
+[tool.coverage.run]
+branch = true
+command_line = "coverage -m pytest"
+data_file = "build/.coverage"
+source = [
+    "src/"
+]
+
+[tool.coverage.report]
+omit = [
+    "test/*"
+]
+exclude_lines = [
+    'pragma: no cover',
+    'raise NotImplementedError',
+    'if TYPE_CHECKING:',
+    'if typing.TYPE_CHECKING:',
+    '@overload',
+    '@typing.overload',
+    '\(Protocol\):$',
+    'typing.assert_never',
+    'assert_never',
+]
+skip_empty = true
+
+[tool.coverage.html]
+directory = "build/documentation/coverage"
+
+[tool.coverage.xml]
+output = "build/documentation/coverage.xml"
+
+# -----------------------------------------------------------------------------
+##  pytest Configurations
+#   https://docs.pytest.org/en/7.2.x/reference/customize.html
+# -----------------------------------------------------------------------------
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = [
+    "-ra", 
+    "--verbose",
+    "--ignore=build/private",
+    # Coverage options should be managed in the .coveragerc file.
+    # The below configurations simply enable coverage and reporting.  
+    "--cov", 
+    "--cov-report=term-missing",
+    "--cov-report=html",
+    "--cov-report=xml",
+    "--cov-fail-under=0",
+    "--color=yes",
+] 
+testpaths = [
+    "test",
+]
+cache_dir = "build/.pytest_cache"
+  
+# -----------------------------------------------------------------------------
+##  MyPy Configurations
+#   https://mypy.readthedocs.io/en/stable/config_file.html#example-pyproject-toml
+# -----------------------------------------------------------------------------
+
+[tool.mypy]
+exclude = '''(?x)(
+    assets/
+    | build/
+    | build_tools
+    | setup.py$
+    | test/
+)'''
+
+cache_dir = "build/.mypy_cache"
+
+# Using no incremental to avoid known issue with mypy and some packages:
+# https://github.com/python/mypy/issues/9852
+incremental = false
+
+# Import Discovery
+# https://mypy.readthedocs.io/en/stable/config_file.html#import-discovery
+ignore_missing_imports = true
+follow_imports = "silent"
+no_site_packages = true
+
+# Untyped definitions and calls
+# https://mypy.readthedocs.io/en/stable/config_file.html#untyped-definitions-and-calls
+# TODO: enable and fix errors
+check_untyped_defs = false
+
+# Miscellaneous strictness flags
+# https://mypy.readthedocs.io/en/latest/config_file.html#miscellaneous-strictness-flags
+allow_redefinition = true
+
+
+# Configuring Error Messages
+# https://mypy.readthedocs.io/en/stable/config_file.html#configuring-error-messages
+show_error_codes = true
+color_output = true
+pretty = true
+show_absolute_path = false
+
+# Reporting Generation
+# https://mypy.readthedocs.io/en/stable/config_file.html#report-generation
+# [DEBUG] UNCOMMENT FOR DEBUG PURPOSES ONLY! Type coverage impacts test 
+# html_report = build/documentation/mypy/
+# xml_report = build/documentation/mypy/
+# cobertura_xml_report = build/documentation/mypy/
+
+# None and Optional handling
+# https://mypy.readthedocs.io/en/latest/config_file.html#none-and-optional-handling
+strict_optional = false
+
+# Miscellaneous
+# https://mypy.readthedocs.io/en/latest/config_file.html#miscellaneous
+# [DEBUG] If you need to better understand type error, increase verbosity
+verbosity = 0
+
+[[tool.mypy.overrides]]
+module = [
+    "src.*",
+    "test.*"
+]
+ignore_errors = false
+
+
+# -----------------------------------------------------------------------------
+##  Black Configurations
+#   https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html#what-on-earth-is-a-pyproject-toml-file
+# -----------------------------------------------------------------------------
+
+
+[tool.black]
+line-length = 99
+include = '\.pyi?$'
+
+# -----------------------------------------------------------------------------
+##  isort Configurations
+#   https://pycqa.github.io/isort/docs/configuration/config_files.html
+# -----------------------------------------------------------------------------
+
+[tool.isort]
+# required for compatibility with black:
+line_length = 99
+profile = "black"
+src_paths = ["src", "test"]
+
+# -----------------------------------------------------------------------------
+##  bumpversion Configurations
+#   https://callowayproject.github.io/bump-my-version/
+# -----------------------------------------------------------------------------
+
+[tool.bumpversion]
+allow_dirty = false
+commit = true
+commit_args = ""
+current_version = "0.0.2"
+ignore_missing_version = false
+message = "Bump version: {current_version} → {new_version}"
+parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
+regex = false
+replace = "{new_version}"
+search = "{current_version}"
+serialize = ["{major}.{minor}.{patch}"]
+sign_tags = false
+tag = true
+tag_message = "Bump version: {current_version} → {new_version}"
+tag_name = "v{new_version}"
+
+[[tool.bumpversion.files]]
+filename = "src/aibs_informatics_aws_utils/_version.py"
+search = "__version__ = \"{current_version}\""
+replace = "__version__ = \"{new_version}\""

aibs_informatics_aws_utils-0.0.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

aibs_informatics_aws_utils-0.0.2/src/aibs_informatics_aws_utils/__init__.py

@@ -0,0 +1 @@
+from aibs_informatics_aws_utils.core import *  # type: ignore

aibs_informatics_aws_utils-0.0.2/src/aibs_informatics_aws_utils/_version.py

@@ -0,0 +1 @@
+__version__ = "0.0.2"

aibs_informatics_aws_utils-0.0.2/src/aibs_informatics_aws_utils/apigateway.py

@@ -0,0 +1,35 @@
+from typing import TYPE_CHECKING, List
+
+from aibs_informatics_aws_utils.core import AWSService, get_region
+from aibs_informatics_aws_utils.exceptions import ResourceNotFoundError
+
+if TYPE_CHECKING:  # pragma: no cover
+    from mypy_boto3_apigateway.type_defs import RestApiTypeDef
+else:
+    RestApiTypeDef = dict
+
+get_apigateway_client = AWSService.API_GATEWAY.get_client
+
+
+def get_rest_api(api_name: str, region: str = None) -> RestApiTypeDef:
+    apigw = get_apigateway_client(region=region)
+
+    paginator = apigw.get_paginator("get_rest_apis")
+    rest_apis: List[RestApiTypeDef] = paginator.paginate(
+        PaginationConfig={"MaxItems": 100}
+    ).build_full_result()["items"]
+
+    for rest_api in rest_apis:
+        # In theory, only one api should be associated with env-base
+        if rest_api.get("name") == api_name:
+            return rest_api
+    else:
+        raise ResourceNotFoundError(f"Could not resolve REST Api with {api_name}")
+
+
+def get_rest_api_endpoint(
+    rest_api: RestApiTypeDef, stage: str = "prod", region: str = None
+) -> str:
+    api_id = rest_api["id"]  # type: ignore  # mypy_boto3 TypeDict makes optional, but actually is required
+    region = get_region(region)
+    return f"https://{api_id}.execute-api.{region}.amazonaws.com/{stage}"

aibs_informatics_aws_utils-0.0.2/src/aibs_informatics_aws_utils/athena.py

@@ -0,0 +1,84 @@
+import logging
+import re
+import time
+from string import Template
+from typing import TYPE_CHECKING, List, Literal, Optional, Tuple
+
+from aibs_informatics_core.env import EnvBase
+from botocore.exceptions import ClientError
+
+from aibs_informatics_aws_utils.core import AWSService
+from aibs_informatics_aws_utils.exceptions import AWSError
+
+if TYPE_CHECKING:  # pragma: no cover
+    from mypy_boto3_athena.type_defs import (
+        GetQueryExecutionInputRequestTypeDef,
+        GetQueryExecutionOutputTypeDef,
+        QueryExecutionStatusTypeDef,
+        QueryExecutionTypeDef,
+        StartQueryExecutionInputRequestTypeDef,
+        StartQueryExecutionOutputTypeDef,
+    )
+else:
+    GetQueryExecutionInputRequestTypeDef = dict
+    GetQueryExecutionOutputTypeDef = dict
+    QueryExecutionStatusTypeDef = dict
+    QueryExecutionTypeDef = dict
+    StartQueryExecutionInputRequestTypeDef = dict
+    StartQueryExecutionOutputTypeDef = dict
+
+
+ATHENA_QUERY_WAITER_STATUS = Literal["SUCCEEDED", "FAILED", "CANCELLED", "TIMEOUT"]
+
+logger = logging.getLogger(__name__)
+
+get_athena_client = AWSService.ATHENA.get_client
+
+
+def start_query_execution(
+    query_string: str,
+    work_group: Optional[str] = None,
+    execution_parameters: Optional[List[str]] = None,
+    **kwargs,
+) -> StartQueryExecutionOutputTypeDef:
+    athena = get_athena_client()
+
+    request = StartQueryExecutionInputRequestTypeDef(QueryString=query_string)
+    if work_group:
+        request["WorkGroup"] = work_group
+    if execution_parameters:
+        request["ExecutionParameters"] = execution_parameters
+    request.update(kwargs)
+    try:
+        metadata = athena.start_query_execution(**request)
+        return metadata
+    except ClientError as e:
+        logger.error(f"Error executing : {request} {e}", exc_info=True)
+        raise AWSError(f"Error starting query execution: {request} {e}") from e
+
+
+def get_query_execution(query_execution_id: str) -> GetQueryExecutionOutputTypeDef:
+    athena = get_athena_client()
+    try:
+        return athena.get_query_execution(QueryExecutionId=query_execution_id)
+    except Exception as e:
+        logger.error(f"Error executing : {query_execution_id} {e}", exc_info=True)
+        raise AWSError(f"Error starting query execution: {query_execution_id} {e}") from e
+
+
+def query_waiter(
+    query_execution_id: str, timeout: int = 60
+) -> Tuple[ATHENA_QUERY_WAITER_STATUS, QueryExecutionStatusTypeDef]:
+    start = time.time()
+    logger.info(f"Polling for status of query execution: {query_execution_id}")
+    while True:
+        stats = get_query_execution(query_execution_id=query_execution_id)
+        logger.info(f"Query Execution Status: {stats}")
+        status = stats["QueryExecution"].get("Status", {})
+        state = status.get("State")
+        if state and state in ["SUCCEEDED", "FAILED", "CANCELLED"]:
+            return state, status
+        time.sleep(0.2)  # 200ms
+        # Exit if the time waiting exceed the timeout seconds
+        if time.time() > start + timeout:
+            return "TIMEOUT", status

aibs_informatics_aws_utils-0.0.2/src/aibs_informatics_aws_utils/auth.py

@@ -0,0 +1,59 @@
+from typing import Optional
+
+import requests
+from botocore.auth import SigV4Auth
+from botocore.awsrequest import AWSRequest
+from botocore.compat import parse_qsl, urlparse
+from botocore.session import Session
+from requests.auth import AuthBase
+
+from aibs_informatics_aws_utils.core import get_session
+
+
+class IamAWSRequestsAuth(AuthBase):
+    """
+    IAM authorizer.
+
+    :param boto3.Session session: Optional boto3 Session object
+    :param str service_name: Optional AWS service name
+
+    :Example:
+
+    >>> IAMAuth()
+    >>> IAMAuth(boto3.Session(), 'execute-api')
+    """
+
+    def __init__(self, session: Optional[Session] = None, service_name: str = "execute-api"):
+        self.boto3_session = get_session(session)
+        credentials = self.boto3_session.get_credentials()
+        if not credentials:
+            raise ValueError("No AWS credentials found")
+
+        self.sigv4 = SigV4Auth(
+            credentials=credentials.get_frozen_credentials(),
+            service_name=service_name,
+            region_name=self.boto3_session.region_name,
+        )
+
+    def __call__(self, request):
+        # Parse request URL
+        url = urlparse(request.url)
+
+        # Prepare AWS request
+        awsrequest = AWSRequest(
+            method=request.method,
+            url=f"{url.scheme}://{url.netloc}{url.path}",
+            data=request.body if hasattr(request, "body") else (request.json or request.data),
+            params=dict(parse_qsl(url.query)),
+        )
+
+        # Sign request
+        self.sigv4.add_auth(awsrequest)
+
+        # Re-add original headers
+        for key, val in request.headers.items():
+            if key not in awsrequest.headers:
+                awsrequest.headers[key] = val
+
+        # Return prepared request
+        return awsrequest.prepare()