sift-stack-py 0.3.2__py3-none-any.whl

sift_py/grpc/_retry.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import json
+from typing import Dict, List, TypedDict
+
+from grpc import StatusCode
+from typing_extensions import Self
+
+
+class RetryPolicy:
+    """
+    Retry policy meant to be used for `sift_py.grpc.transport.SiftChannel`. Users may have the ability to configure their own
+    custom retry policy in the future, but for now this is primarily intended for internal use.
+
+    - [Retry policy schema](https://github.com/grpc/grpc-proto/blob/ec30f589e2519d595688b9a42f88a91bdd6b733f/grpc/service_config/service_config.proto#L136)
+    - [Enable gRPC retry option](https://github.com/grpc/grpc/blob/9a5fdfc3d3a7fc575a394360be4532ee09a85620/include/grpc/impl/channel_arg_names.h#L311)
+    - [Service config option](https://github.com/grpc/grpc/blob/9a5fdfc3d3a7fc575a394360be4532ee09a85620/include/grpc/impl/channel_arg_names.h#L207)
+    """
+
+    config: RetryConfig
+
+    DEFAULT_POLICY: RetryConfig = {
+        "methodConfig": [
+            {
+                # We can configure this on a per-service and RPC basis but for now we'll
+                # apply this across all services and RPCs.
+                "name": [{}],
+                "retryPolicy": {
+                    # gRPC does not allow more than 5 attempts
+                    "maxAttempts": 5,
+                    "initialBackoff": "0.05s",
+                    "maxBackoff": "5s",
+                    "backoffMultiplier": 4,
+                    "retryableStatusCodes": [
+                        StatusCode.UNKNOWN.name,
+                        StatusCode.UNAVAILABLE.name,
+                        StatusCode.ABORTED.name,
+                        StatusCode.DEADLINE_EXCEEDED.name,
+                    ],
+                },
+            }
+        ]
+    }
+
+    def __init__(self, config: RetryConfig):
+        self.config = config
+
+    def as_json(self) -> str:
+        return json.dumps(self.config)
+
+    @classmethod
+    def default(cls) -> Self:
+        return cls(config=cls.DEFAULT_POLICY)
+
+
+class RetryConfig(TypedDict):
+    methodConfig: List[MethodConfigDict]
+
+
+class MethodConfigDict(TypedDict):
+    name: List[Dict[str, str]]
+    retryPolicy: RetryConfigDict
+
+
+class RetryConfigDict(TypedDict):
+    maxAttempts: int
+    initialBackoff: str
+    maxBackoff: str
+    backoffMultiplier: int
+    retryableStatusCodes: List[str]

sift_py/grpc/keepalive.py

@@ -0,0 +1,34 @@
+from typing import TypedDict
+
+DEFAULT_KEEPALIVE_TIME_MS = 20_000
+"""Interval with which to send keepalive pings"""
+
+DEFAULT_KEEPALIVE_TIMEOUT_MS = 20_000
+"""Timeout while waiting for server to acknowledge keepalive ping"""
+
+DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS = 1
+"""Allows connection without any active RPCs"""
+
+DEFAULT_MAX_PINGS_WITHOUT_DATA = 0
+"""Disabled"""
+
+
+# https://github.com/grpc/grpc/blob/master/doc/keepalive.md
+class KeepaliveConfig(TypedDict):
+    """
+    Make make this public in the future to allow folks to configure their own keepalive settings
+    if there is demand for it.
+    """
+
+    keepalive_time_ms: int
+    keepalive_timeout_ms: int
+    keepalive_permit_without_calls: int
+    max_pings_without_data: int
+
+
+DEFAULT_KEEPALIVE_CONFIG: KeepaliveConfig = {
+    "keepalive_time_ms": DEFAULT_KEEPALIVE_TIME_MS,
+    "keepalive_timeout_ms": DEFAULT_KEEPALIVE_TIMEOUT_MS,
+    "keepalive_permit_without_calls": DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS,
+    "max_pings_without_data": DEFAULT_MAX_PINGS_WITHOUT_DATA,
+}

sift_py/grpc/transport.py

@@ -0,0 +1,250 @@
+"""
+This module is concerned with creating a gRPC transport channel specifically for
+interacting with Sift's gRPC API. the `use_sift_channel` method creates said channel
+and should generally be used within a with-block for correct resource management.
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union, cast
+from urllib.parse import ParseResult, urlparse
+
+import grpc
+import grpc.aio as grpc_aio
+from typing_extensions import NotRequired, TypeAlias
+
+from sift_py.grpc._async_interceptors.base import ClientAsyncInterceptor
+from sift_py.grpc._async_interceptors.metadata import MetadataAsyncInterceptor
+from sift_py.grpc._interceptors.base import ClientInterceptor
+from sift_py.grpc._interceptors.metadata import Metadata, MetadataInterceptor
+from sift_py.grpc._retry import RetryPolicy
+from sift_py.grpc.keepalive import DEFAULT_KEEPALIVE_CONFIG, KeepaliveConfig
+
+SiftChannel: TypeAlias = grpc.Channel
+SiftAsyncChannel: TypeAlias = grpc_aio.Channel
+
+
+def get_ssl_credentials(cert_via_openssl: bool) -> grpc.ChannelCredentials:
+    """
+    Returns SSL credentials for use with gRPC.
+    Workaround for this issue: https://github.com/grpc/grpc/issues/29682
+    """
+    if not cert_via_openssl:
+        return grpc.ssl_channel_credentials()
+
+    try:
+        import ssl
+
+        from OpenSSL import crypto
+
+        ssl_context = ssl.create_default_context()
+        certs_der = ssl_context.get_ca_certs(binary_form=True)
+        certs_x509 = [crypto.load_certificate(crypto.FILETYPE_ASN1, x) for x in certs_der]
+        certs_pem = [crypto.dump_certificate(crypto.FILETYPE_PEM, x) for x in certs_x509]
+        certs_bytes = b"".join(certs_pem)
+
+        return grpc.ssl_channel_credentials(certs_bytes)
+    except ImportError as e:
+        raise Exception(
+            "Missing required dependencies for cert_via_openssl. Run `pip install sift-stack-py[openssl]` to install the required dependencies."
+        ) from e
+
+
+def use_sift_channel(
+    config: SiftChannelConfig, metadata: Optional[Dict[str, Any]] = None
+) -> SiftChannel:
+    """
+    Returns an intercepted channel that is meant to be used across all services that
+    make RPCs to Sift's API. It is highly encouraged to use this within a with-block
+    for correct resource clean-up.
+
+    Should an RPC fail for a reason that isn't explicitly controlled by Sift, `SiftChannel`
+    will automatically leverage gRPC's retry mechanism to try and recover until the max-attempts
+    are exceeded, after which the underlying exception will be raised.
+    """
+    use_ssl = config.get("use_ssl", True)
+    cert_via_openssl = config.get("cert_via_openssl", False)
+
+    if not use_ssl:
+        return _use_insecure_sift_channel(config, metadata)
+
+    credentials = get_ssl_credentials(cert_via_openssl)
+    options = _compute_channel_options(config)
+    api_uri = _clean_uri(config["uri"], use_ssl)
+    channel = grpc.secure_channel(api_uri, credentials, options)
+    interceptors = _compute_sift_interceptors(config, metadata)
+    return grpc.intercept_channel(channel, *interceptors)
+
+
+def use_sift_async_channel(
+    config: SiftChannelConfig, metadata: Optional[Dict[str, Any]] = None
+) -> SiftAsyncChannel:
+    """
+    Like `use_sift_channel` but returns a channel meant to be used within the context
+    of an async runtime when asynchonous I/O is required.
+    """
+    use_ssl = config.get("use_ssl", True)
+    cert_via_openssl = config.get("cert_via_openssl", False)
+
+    if not use_ssl:
+        return _use_insecure_sift_async_channel(config, metadata)
+
+    return grpc_aio.secure_channel(
+        target=_clean_uri(config["uri"], use_ssl),
+        credentials=get_ssl_credentials(cert_via_openssl),
+        options=_compute_channel_options(config),
+        interceptors=_compute_sift_async_interceptors(config, metadata),
+    )
+
+
+def _use_insecure_sift_channel(
+    config: SiftChannelConfig, metadata: Optional[Dict[str, Any]] = None
+) -> SiftChannel:
+    """
+    FOR DEVELOPMENT PURPOSES ONLY
+    """
+    options = _compute_channel_options(config)
+    api_uri = _clean_uri(config["uri"], False)
+    channel = grpc.insecure_channel(api_uri, options)
+    interceptors = _compute_sift_interceptors(config, metadata)
+    return grpc.intercept_channel(channel, *interceptors)
+
+
+def _use_insecure_sift_async_channel(
+    config: SiftChannelConfig, metadata: Optional[Dict[str, Any]] = None
+) -> SiftAsyncChannel:
+    """
+    FOR DEVELOPMENT PURPOSES ONLY
+    """
+    return grpc_aio.insecure_channel(
+        target=config["uri"],
+        options=_compute_channel_options(config),
+        interceptors=_compute_sift_async_interceptors(config, metadata),
+    )
+
+
+def _compute_sift_interceptors(
+    config: SiftChannelConfig, metadata: Optional[Dict[str, Any]] = None
+) -> List[ClientInterceptor]:
+    """
+    Initialized all interceptors here.
+    """
+    return [
+        _metadata_interceptor(config, metadata),
+    ]
+
+
+def _compute_sift_async_interceptors(
+    config: SiftChannelConfig, metadata: Optional[Dict[str, Any]] = None
+) -> List[grpc_aio.ClientInterceptor]:
+    return [
+        _metadata_async_interceptor(config, metadata),
+    ]
+
+
+def _compute_channel_options(opts: SiftChannelConfig) -> List[Tuple[str, Any]]:
+    """
+    Initialize all [channel options](https://github.com/grpc/grpc/blob/v1.64.x/include/grpc/impl/channel_arg_names.h) here.
+    """
+
+    options = [
+        ("grpc.enable_retries", 1),
+        ("grpc.service_config", RetryPolicy.default().as_json()),
+        # Primary cannot be overriden:
+        #  https://github.com/grpc/grpc/blob/0498194240f55d7f4b12633ad01339fb690621bf/src/core/ext/filters/http/client/http_client_filter.cc#L97
+        ("grpc.secondary_user_agent", _compute_user_agent()),
+    ]
+
+    enable_keepalive = opts.get("enable_keepalive", True)
+    if isinstance(enable_keepalive, dict):
+        config = cast(KeepaliveConfig, enable_keepalive)
+        options.extend(_compute_keep_alive_channel_opts(config))
+    elif enable_keepalive:
+        options.extend(_compute_keep_alive_channel_opts(DEFAULT_KEEPALIVE_CONFIG))
+
+    return options
+
+
+def _metadata_interceptor(
+    config: SiftChannelConfig, metadata: Optional[Dict[str, Any]] = None
+) -> ClientInterceptor:
+    """
+    Any new metadata goes here.
+    """
+    apikey = config["apikey"]
+    md: Metadata = [("authorization", f"Bearer {apikey}")]
+
+    if metadata:
+        for key, val in metadata.items():
+            md.append((key, val))
+
+    return MetadataInterceptor(md)
+
+
+def _metadata_async_interceptor(
+    config: SiftChannelConfig, metadata: Optional[Dict[str, Any]] = None
+) -> ClientAsyncInterceptor:
+    """
+    Any new metadata goes here for unary-unary calls.
+    """
+    apikey = config["apikey"]
+    md: Metadata = [("authorization", f"Bearer {apikey}")]
+
+    if metadata:
+        for key, val in metadata.items():
+            md.append((key, val))
+
+    return MetadataAsyncInterceptor(md)
+
+
+def _clean_uri(uri: str, use_ssl: bool) -> str:
+    """
+    This will automatically transform the URI to an acceptable form regardless of whether or not
+    users included the scheme in the URL or included trailing slashes.
+    """
+
+    if "http://" in uri or "https://" in uri:
+        parsed: ParseResult = urlparse(uri)
+        return parsed.netloc
+
+    full_uri = f"https://{uri}" if use_ssl else f"http://{uri}"
+    parsed_res: ParseResult = urlparse(full_uri)
+    return parsed_res.netloc
+
+
+def _compute_user_agent() -> str:
+    try:
+        return f"sift_stack_py/{version('sift_stack_py')}"
+    except PackageNotFoundError:
+        return "sift-stack-py"
+
+
+def _compute_keep_alive_channel_opts(config: KeepaliveConfig) -> List[Tuple[str, int]]:
+    return [
+        ("grpc.keepalive_time_ms", config["keepalive_time_ms"]),
+        ("grpc.keepalive_timeout_ms", config["keepalive_timeout_ms"]),
+        ("grpc.http2.max_pings_without_data", config["max_pings_without_data"]),
+        ("grpc.keepalive_permit_without_calls", config["keepalive_permit_without_calls"]),
+    ]
+
+
+class SiftChannelConfig(TypedDict):
+    """
+    Config class used to instantiate a `SiftChannel` via `use_sift_channel`.
+    - `uri`: The URI of Sift's gRPC API. The scheme portion of the URI i.e. `https://` should be ommitted.
+    - `apikey`: User-generated API key generated via the Sift application.
+    - `enable_keepalive`: Enabled by default, but can be disabled by passing in `False`. HTTP/2 keep-alive prevents connections from
+    being terminated during idle periods. A custom `sift_py.grpc.keepalive.KeepaliveConfig` may also be provided.
+    - `use_ssl`: INTERNAL USE. Meant to be used for local development.
+    - `cert_via_openssl`: Enable this if you want to use OpenSSL to load the certificates.
+    Run `pip install sift-stack-py[openssl]` to install the dependencies required to use this option.
+    This works around this issue with grpc loading SSL certificates: https://github.com/grpc/grpc/issues/29682.
+    Default is False.
+    """
+
+    uri: str
+    apikey: str
+    enable_keepalive: NotRequired[Union[bool, KeepaliveConfig]]
+    use_ssl: NotRequired[bool]
+    cert_via_openssl: NotRequired[bool]

sift_py/grpc/transport_test.py

@@ -0,0 +1,170 @@
+# ruff: noqa: N802
+
+import re
+from concurrent import futures
+from contextlib import contextmanager
+from typing import Any, Callable, Iterator, cast
+
+import grpc
+import pytest
+from pytest_mock import MockFixture, MockType
+from sift.data.v1.data_pb2 import GetDataRequest, GetDataResponse
+from sift.data.v1.data_pb2_grpc import (
+    DataServiceServicer,
+    DataServiceStub,
+    add_DataServiceServicer_to_server,
+)
+
+from sift_py._internal.test_util.server_interceptor import ServerInterceptor
+from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel
+
+
+class DataService(DataServiceServicer):
+    def GetData(self, request: GetDataRequest, context: grpc.ServicerContext):
+        return GetDataResponse(next_page_token="next-page-token")
+
+
+class AuthInterceptor(ServerInterceptor):
+    AUTH_REGEX = re.compile(r"^Bearer (.+)$")
+
+    def intercept(
+        self,
+        method: Callable,
+        request_or_iterator: Any,
+        context: grpc.ServicerContext,
+        method_name: str,
+    ) -> Any:
+        authenticated = False
+        for metadata in context.invocation_metadata():
+            if metadata.key == "authorization":
+                auth = self.__class__.AUTH_REGEX.match(metadata.value)
+
+                if auth is not None and len(auth.group(1)) > 0:
+                    authenticated = True
+
+                break
+
+        if authenticated:
+            return method(request_or_iterator, context)
+        else:
+            context.set_code(grpc.StatusCode.UNAUTHENTICATED)
+            context.set_details("Invalid or missing API key")
+            raise
+
+
+class ForceFailInterceptor(ServerInterceptor):
+    """
+    Force RPC to fail a few times before letting it pass.
+
+    `failed_attempts`: Count of how many times failed
+    `expected_num_fails`: How many times you want call to fail
+    """
+
+    failed_attempts: int
+    expected_num_fails: int
+
+    def __init__(self, expected_num_fails: int):
+        self.expected_num_fails = expected_num_fails
+        self.failed_attempts = 0
+        super().__init__()
+
+    def intercept(
+        self,
+        method: Callable,
+        request_or_iterator: Any,
+        context: grpc.ServicerContext,
+        method_name: str,
+    ) -> Any:
+        if self.failed_attempts < self.expected_num_fails:
+            self.failed_attempts += 1
+            context.set_code(grpc.StatusCode.UNKNOWN)
+            context.set_details("something unknown happened")
+            raise
+
+        return method(request_or_iterator, context)
+
+
+def test_sift_channel(mocker: MockFixture):
+    @contextmanager
+    def test_server_spy(*interceptors: ServerInterceptor) -> Iterator[MockType]:
+        server = grpc.server(
+            thread_pool=futures.ThreadPoolExecutor(max_workers=1), interceptors=list(interceptors)
+        )
+
+        data_service = DataService()
+        spy = mocker.spy(data_service, "GetData")
+
+        add_DataServiceServicer_to_server(data_service, server)
+        server.add_insecure_port("[::]:50052")
+        server.start()
+        try:
+            yield spy
+        finally:
+            server.stop(None)
+            server.wait_for_termination()
+
+    with test_server_spy(AuthInterceptor()) as get_data_spy:
+        sift_channel_config_a: SiftChannelConfig = {
+            "uri": "localhost:50052",
+            "apikey": "",
+            "use_ssl": False,
+        }
+
+        with use_sift_channel(sift_channel_config_a) as channel:
+            with pytest.raises(grpc.RpcError, match="UNAUTHENTICATED"):
+                stub = DataServiceStub(channel)
+                _ = cast(GetDataResponse, stub.GetData(GetDataRequest()))
+
+            get_data_spy.assert_not_called()
+
+        sift_channel_config_b: SiftChannelConfig = {
+            "uri": "localhost:50052",
+            "apikey": "some-token",
+            "use_ssl": False,
+        }
+
+        with use_sift_channel(sift_channel_config_b) as channel:
+            stub = DataServiceStub(channel)
+            res = cast(GetDataResponse, stub.GetData(GetDataRequest()))
+            assert res.next_page_token == "next-page-token"
+            get_data_spy.assert_called_once()
+
+    force_fail_interceptor = ForceFailInterceptor(4)
+    with test_server_spy(AuthInterceptor(), force_fail_interceptor) as get_data_spy:
+        sift_channel_config_c: SiftChannelConfig = {
+            "uri": "localhost:50052",
+            "apikey": "some-token",
+            "use_ssl": False,
+        }
+
+        with use_sift_channel(sift_channel_config_c) as channel:
+            stub = DataServiceStub(channel)
+            # This will attempt 5 times: fail 4 times, succeed on 5th
+            res = cast(GetDataResponse, stub.GetData(GetDataRequest()))
+            assert res.next_page_token == "next-page-token"
+            get_data_spy.assert_called_once()
+
+    # fail 4 times, pass the 5th attempt
+    assert force_fail_interceptor.failed_attempts == 4
+
+    # Now we're going to fail beyond the max retry attempts
+
+    force_fail_interceptor_max = ForceFailInterceptor(7)
+    with test_server_spy(AuthInterceptor(), force_fail_interceptor_max) as get_data_spy:
+        sift_channel_config_d: SiftChannelConfig = {
+            "uri": "localhost:50052",
+            "apikey": "some-token",
+            "use_ssl": False,
+        }
+
+        with use_sift_channel(sift_channel_config_d) as channel:
+            stub = DataServiceStub(channel)
+
+            # This will go beyond the max number of attempts
+            with pytest.raises(Exception):
+                res = cast(GetDataResponse, stub.GetData(GetDataRequest()))
+
+            get_data_spy.assert_not_called()
+
+    # All attempts failed
+    assert force_fail_interceptor_max.failed_attempts == 5

sift_py/ingestion/__init__.py

@@ -0,0 +1,6 @@
+"""
+Module containing components necessary to leverage Sift's data ingestion and rule evaluation API.
+The fundamental components of this module are the following:
+- `sift_py.ingestion.config.telemetry.TelemetryConfig`
+- `sift_py.ingestion.service.IngestionService`
+"""

sift_py/ingestion/_internal/__init__.py

@@ -0,0 +1,6 @@
+"""
+INTERNAL MODULE
+
+This module contains implementation details that aren't meant to be used directly.
+APIs in this module are garaunteed to not be stable so proceed at your own risk.
+"""

sift_py/ingestion/_internal/channel.py

@@ -0,0 +1,12 @@
+from sift.rules.v1.rules_pb2 import ChannelReference
+
+
+def channel_reference_from_fqn(fqn: str) -> ChannelReference:
+    parts = fqn.split(".")
+
+    if len(parts) == 1:
+        return ChannelReference(name=parts[0])
+
+    component_parts = parts[: len(parts) - 1]
+
+    return ChannelReference(name=parts[-1], component=".".join(component_parts))

sift_py/ingestion/_internal/error.py

@@ -0,0 +1,10 @@
+class IngestionValidationError(Exception):
+    """
+    Errors that can occur while initializing the ingestion service
+    or when creating ingestion requests.
+    """
+
+    message: str
+
+    def __init__(self, message: str):
+        super().__init__(message)