diaspora-event-sdk 0.4.0__py3-none-any.whl → 0.4.3__py3-none-any.whl

diaspora_event_sdk/__init__.py

@@ -9,7 +9,10 @@ from diaspora_event_sdk.sdk.client import Client
 from diaspora_event_sdk.sdk.kafka_client import (
     KafkaProducer,
     KafkaConsumer,
-    block_until_ready,
 )
 
-__all__ = ("Client", "KafkaProducer", "KafkaConsumer", "block_until_ready")
+__all__ = (
+    "Client",
+    "KafkaProducer",
+    "KafkaConsumer",
+)

diaspora_event_sdk/sdk/_environments.py

@@ -13,9 +13,7 @@ def get_web_service_url(envname: Union[str, None] = None) -> str:
     env = envname or _get_envname()
     urls = {
         "production": "https://diaspora-web-service.qpp943wkvr7b2.us-east-1.cs.amazonlightsail.com/",
-        "dev": "https://diaspora-web-service.qpp943wkvr7b2.us-east-1.cs.amazonlightsail.com/",
         "local": "http://localhost:8000",
-        "legacy": "https://diaspora-web-service-dev.ml22sevubfnks.us-east-1.cs.amazonlightsail.com",
     }
 
     return urls.get(env, urls["production"])

diaspora_event_sdk/sdk/client.py

@@ -1,4 +1,3 @@
-import json
 from typing import Optional
 
 from diaspora_event_sdk.sdk.login_manager import (
@@ -7,7 +6,7 @@ from diaspora_event_sdk.sdk.login_manager import (
     requires_login,
 )
 
-from ._environments import DIASPORA_RESOURCE_SERVER, get_web_service_url
+from ._environments import get_web_service_url
 
 
 class Client:
@@ -32,213 +31,83 @@ class Client:
         )
         self.auth_client = self.login_manager.get_auth_client()
         self.subject_openid = self.auth_client.userinfo()["sub"]
+        self.namespace = f"ns-{self.subject_openid.replace('-', '')[-12:]}"
 
     def logout(self):
         """Remove credentials from your local system"""
         self.login_manager.logout()
 
     @requires_login
-    def create_key(self):
-        """
-        Revokes existing keys, generates a new key, and updates the token storage with the newly created key and the Diaspora endpoint.
-        """
-        resp = self.web_client.create_key(self.subject_openid)
-        if resp["status"] == "error":
-            raise Exception("should not happen")
-        tokens = self.login_manager._token_storage.get_token_data(
-            DIASPORA_RESOURCE_SERVER
-        )
-        tokens["access_key"] = resp["access_key"]
-        tokens["secret_key"] = resp["secret_key"]
-        tokens["endpoint"] = resp["endpoint"]
-        with self.login_manager._access_lock:
-            self.login_manager._token_storage._connection.executemany(
-                "REPLACE INTO token_storage(namespace, resource_server, token_data_json) "
-                "VALUES(?, ?, ?)",
-                [
-                    (
-                        self.login_manager._token_storage.namespace,
-                        DIASPORA_RESOURCE_SERVER,
-                        json.dumps(tokens),
-                    )
-                ],
-            )
-            self.login_manager._token_storage._connection.commit()
-        return {
-            "access_key": tokens["access_key"],
-            "secret_key": tokens["secret_key"],
-            "endpoint": tokens["endpoint"],
-        }
-
-    @requires_login
-    def retrieve_key(self):
-        """
-        Retrieves the key from local token storage, and calls create_key() if the local key is not found.
-        """
-        tokens = self.login_manager._token_storage.get_token_data(
-            DIASPORA_RESOURCE_SERVER
-        )
-        if (
-            tokens is None
-            or "endpoint" not in tokens
-            or "access_key" not in tokens
-            or "secret_key" not in tokens
-        ):
-            return self.create_key()
-        else:
-            return {
-                "access_key": tokens["access_key"],
-                "secret_key": tokens["secret_key"],
-                "endpoint": tokens["endpoint"],
-            }
-
-    @requires_login
-    def put_secret_key(self, access_key, secret_key, endpoint):
-        tokens = self.login_manager._token_storage.get_token_data(
-            DIASPORA_RESOURCE_SERVER
-        )
-        tokens["access_key"] = access_key
-        tokens["secret_key"] = secret_key
-        tokens["endpoint"] = endpoint
-        with self.login_manager._access_lock:
-            self.login_manager._token_storage._connection.executemany(
-                "REPLACE INTO token_storage(namespace, resource_server, token_data_json) "
-                "VALUES(?, ?, ?)",
-                [
-                    (
-                        self.login_manager._token_storage.namespace,
-                        DIASPORA_RESOURCE_SERVER,
-                        json.dumps(tokens),
-                    )
-                ],
-            )
-            self.login_manager._token_storage._connection.commit()
-
-    @requires_login
-    def list_topics(self):
-        """
-        Returns a list of topics currently registered under the user's account.
-        """
-        return self.web_client.list_topics(self.subject_openid)
-
-    @requires_login
-    def register_topic(self, topic):
-        """
-        Registers a new topic the user's account with permissions to read, write, and describe the topic.
-        """
-        return self.web_client.register_topic(self.subject_openid, topic, "register")
-
-    @requires_login
-    def unregister_topic(self, topic):
-        """
-        Unregisters a topic from a user's account, but all existing events within the topic are unaffected.
-        """
-        return self.web_client.register_topic(self.subject_openid, topic, "unregister")
-
-    @requires_login
-    def get_topic_configs(self, topic):
+    def create_user(self):
         """
-        Retrieves the current configurations for a registered topic.
+        Create an IAM user with policy and namespace for the current user (POST /api/v3/user).
+        Returns status, message, subject, and namespace.
         """
-        return self.web_client.get_topic_configs(self.subject_openid, topic)
+        resp = self.web_client.create_user(self.subject_openid)
+        return resp.data if hasattr(resp, "data") else resp
 
     @requires_login
-    def update_topic_configs(self, topic, configs):
+    def delete_user(self):
         """
-        Updates the configurations for a registered topic.
+        Delete the IAM user and all associated resources for the current user (DELETE /api/v3/user).
+        Returns status and message.
         """
-        return self.web_client.update_topic_configs(self.subject_openid, topic, configs)
+        resp = self.web_client.delete_user(self.subject_openid)
+        return resp.data if hasattr(resp, "data") else resp
 
     @requires_login
-    def update_topic_partitions(self, topic, new_partitions):
-        """
-        Increases the number of partitions for a given topic to the specified new partition count.
-        """
-        return self.web_client.update_topic_partitions(
-            self.subject_openid, topic, new_partitions
-        )
-
-    @requires_login
-    def reset_topic(self, topic):
-        """
-        Deletes and recreates the topic, removing all messages and restoring the topic to the default configurations while user access is not affected.
-        """
-        return self.web_client.reset_topic(self.subject_openid, topic)
-
-    @requires_login
-    def grant_user_access(self, topic, user):
-        """
-        Authorizes another user to access a registered topic under the invoker's account.
-        """
-        return self.web_client.grant_user_access(
-            self.subject_openid, topic, user, "grant"
-        )
-
-    @requires_login
-    def revoke_user_access(self, topic, user):
+    def create_key(self):
         """
-        Removes access permissions for another user from a registered topic under the invoker's account.
+        Create a new access key for the current user (POST /api/v3/key).
+        This will replace any existing access key (force refresh).
+        Returns status, message, access_key, secret_key, and create_date.
         """
-        return self.web_client.grant_user_access(
-            self.subject_openid, topic, user, "revoke"
-        )
+        resp = self.web_client.create_key(self.subject_openid)
+        return resp.data if hasattr(resp, "data") else resp
 
     @requires_login
-    def list_topic_users(self, topic):
+    def delete_key(self):
         """
-        Returns a list of users that have access to the topic.
+        Delete access keys from IAM and DynamoDB for the current user (DELETE /api/v3/key).
+        Returns status and message.
         """
-        return self.web_client.list_topic_users(self.subject_openid, topic)
+        resp = self.web_client.delete_key(self.subject_openid)
+        return resp.data if hasattr(resp, "data") else resp
 
     @requires_login
-    def list_triggers(self):
+    def list_namespaces(self):
         """
-        Retrieves a list of triggers associated created under the user's account, showing each trigger's configurations and UUID.
+        List all namespaces owned by the current user and their topics (GET /api/v3/namespace).
+        Returns status, message, and namespaces dict (namespace -> list of topics).
         """
-        return self.web_client.list_triggers(self.subject_openid)
+        resp = self.web_client.list_namespaces(self.subject_openid)
+        return resp.data if hasattr(resp, "data") else resp
 
     @requires_login
-    def create_trigger(self, topic, function, function_configs, trigger_configs):
+    def create_topic(self, topic: str):
         """
-        Creates a new trigger under the user's account with specific function and invocation configurations.
+        Create a topic under the user's default namespace (POST /api/v3/{namespace}/{topic}).
+        Returns status, message, and topics list.
         """
-        return self.web_client.create_trigger(
-            self.subject_openid,
-            topic,
-            function,
-            "create",
-            function_configs,
-            trigger_configs,
-        )
+        resp = self.web_client.create_topic(self.subject_openid, self.namespace, topic)
+        return resp.data if hasattr(resp, "data") else resp
 
     @requires_login
-    def delete_trigger(self, topic, function):
+    def delete_topic(self, topic: str):
         """
-        Deletes a trigger and related AWS resources, while the associated topic remains unaffected.
+        Delete a topic from the user's default namespace (DELETE /api/v3/{namespace}/{topic}).
+        Returns status, message, and topics list.
         """
-        return self.web_client.create_trigger(
-            self.subject_openid, topic, function, "delete", {}, {}
-        )
+        resp = self.web_client.delete_topic(self.subject_openid, self.namespace, topic)
+        return resp.data if hasattr(resp, "data") else resp
 
     @requires_login
-    def update_trigger(self, trigger_uuid, trigger_configs):
+    def recreate_topic(self, topic: str):
         """
-        Updates invocation configurations of an existing trigger, identified by its unique trigger UUID.
+        Recreate a topic in the user's default namespace by deleting and recreating it (PUT /api/v3/{namespace}/{topic}/recreate).
+        Returns status and message.
         """
-        return self.web_client.update_trigger(
-            self.subject_openid, trigger_uuid, trigger_configs
+        resp = self.web_client.recreate_topic(
+            self.subject_openid, self.namespace, topic
         )
-
-    @requires_login
-    def list_log_streams(self, trigger):
-        """
-        List log streams of a trigger under the user's account
-        """
-        return self.web_client.list_log_streams(self.subject_openid, trigger)
-
-    @requires_login
-    def get_log_events(self, trigger, stream):
-        """
-        Get events in a particular log stream of a trigger under the user's account
-        """
-        return self.web_client.get_log_events(self.subject_openid, trigger, stream)
+        return resp.data if hasattr(resp, "data") else resp

diaspora_event_sdk/sdk/kafka_client.py

@@ -1,24 +1,37 @@
 import json
-from typing import Dict, Any
-import warnings
+import logging
 import time
+import uuid
+import warnings
+from typing import Any, Dict
 
-from .client import Client
 from .aws_iam_msk import generate_auth_token
+from .client import Client
+
+# File-level logger
+logger = logging.getLogger(__name__)
 
 # If kafka-python is not installed, Kafka functionality is not available through diaspora-event-sdk.
 kafka_available = True
 try:
-    from kafka import KafkaProducer as KProd  # type: ignore[import,import-not-found]
-    from kafka import KafkaConsumer as KCons  # type: ignore[import,import-not-found]
     import os
 
-    class MSKTokenProvider:
+    from kafka import KafkaConsumer as KCons  # type: ignore[import,import-not-found]
+    from kafka import KafkaProducer as KProd  # type: ignore[import,import-not-found]
+    from kafka.errors import KafkaTimeoutError, TopicAuthorizationFailedError  # type: ignore[import,import-not-found]
+    from kafka.sasl.oauth import (
+        AbstractTokenProvider,  # type: ignore[import,import-not-found]
+    )
+
+    class MSKTokenProvider(AbstractTokenProvider):
         def token(self):
             token, _ = generate_auth_token("us-east-1")
             return token
 except Exception:
     kafka_available = False
+    # Fallback if kafka-python is not available
+    TopicAuthorizationFailedError = Exception
+    KafkaTimeoutError = Exception
 
 
 def get_diaspora_config(extra_configs: Dict[str, Any] = {}) -> Dict[str, Any]:
@@ -28,15 +41,11 @@ def get_diaspora_config(extra_configs: Dict[str, Any] = {}) -> Dict[str, Any]:
     """
 
     try:
-        if (
-            "OCTOPUS_AWS_ACCESS_KEY_ID" not in os.environ
-            and "OCTOPUS_AWS_SECRET_ACCESS_KEY" not in os.environ
-            and "OCTOPUS_BOOTSTRAP_SERVERS" not in os.environ
-        ):
-            keys = Client().retrieve_key()
-            os.environ["OCTOPUS_AWS_ACCESS_KEY_ID"] = keys["access_key"]
-            os.environ["OCTOPUS_AWS_SECRET_ACCESS_KEY"] = keys["secret_key"]
-            os.environ["OCTOPUS_BOOTSTRAP_SERVERS"] = keys["endpoint"]
+        client = Client()
+        keys = client.create_key()  # create or retrieve key
+        os.environ["OCTOPUS_AWS_ACCESS_KEY_ID"] = keys["access_key"]
+        os.environ["OCTOPUS_AWS_SECRET_ACCESS_KEY"] = keys["secret_key"]
+        os.environ["OCTOPUS_BOOTSTRAP_SERVERS"] = keys["endpoint"]
 
     except Exception as e:
         raise RuntimeError("Failed to retrieve Kafka keys") from e
@@ -45,7 +54,7 @@ def get_diaspora_config(extra_configs: Dict[str, Any] = {}) -> Dict[str, Any]:
         "bootstrap_servers": os.environ["OCTOPUS_BOOTSTRAP_SERVERS"],
         "security_protocol": "SASL_SSL",
         "sasl_mechanism": "OAUTHBEARER",
-        "api_version": (3, 5, 1),
+        "api_version": (3, 8, 1),
         "sasl_oauth_token_provider": MSKTokenProvider(),
     }
     conf.update(extra_configs)
@@ -55,15 +64,35 @@ def get_diaspora_config(extra_configs: Dict[str, Any] = {}) -> Dict[str, Any]:
 if kafka_available:
 
     class KafkaProducer(KProd):
-        def __init__(self, **configs):
+        """
+        Wrapper around KProd that:
+        - Requires at least one topic
+        - Sets a default JSON serializer
+        - Does NOT block until topics have partition metadata
+        """
+
+        def __init__(self, *topics, **configs):
+            if not topics:
+                raise ValueError("KafkaProducer requires at least one topic")
+            self.topics = topics
+
             configs.setdefault(
-                "value_serializer", lambda v: json.dumps(v).encode("utf-8")
+                "value_serializer",
+                lambda v: json.dumps(v).encode("utf-8"),
             )
+
             super().__init__(**get_diaspora_config(configs))
+            # Note: We do NOT block on metadata here
 
     class KafkaConsumer(KCons):
         def __init__(self, *topics, **configs):
+            if not topics:
+                raise ValueError("KafkaConsumer requires at least one topic")
+            self.topics = topics
+
             super().__init__(*topics, **get_diaspora_config(configs))
+            # Note: We do NOT block on metadata here
+
 
 else:
     # Create dummy classes that issue a warning when instantiated
@@ -82,56 +111,72 @@ else:
             )
 
 
-# TODO: mypy diaspora_event_sdk/sdk/kafka_client.py --disallow-untyped-defs
-def block_until_ready(max_minutes=5):
-    """
-    Test Kafka producer and consumer connections.
-    By default, this method blocks for five minutes before giving up.
-    It returns a boolean that indicates whether the connections can be successfully established.
+def reliable_client_creation() -> str:
     """
+    Reliably create a client and test topic operations with retry logic.
 
-    def producer_connection_test(result):
-        try:
-            producer = KafkaProducer(max_block_ms=10 * 1000)
-            future = producer.send(
-                topic="__connection_test",
-                value={"message": "Synchronous message from Diaspora SDK"},
-            )
-            result["producer_connection_test"] = future.get(timeout=10)
-        except Exception as e:
-            raise e
-            print(e)
-
-    def consumer_connection_test(result):
+    Returns:
+        str: The full topic name in format "namespace.topic-name"
+    """
+    attempt = 0
+    while True:
+        attempt += 1
+        if attempt > 1:
+            time.sleep(5)
+
+        topic_name = None
+        kafka_topic = None
+        client = None
         try:
-            consumer = KafkaConsumer(
-                "__connection_test",
-                consumer_timeout_ms=10 * 1000,
-                auto_offset_reset="earliest",
-            )
-            for msg in consumer:
-                result["consumer_connection_test"] = msg
-                break
+            client = Client()
+            key_result = client.create_key()
+
+            # If status is not success, throw exception
+            if key_result.get("status") != "success":
+                raise RuntimeError(
+                    f"Failed to create key: {key_result.get('message', 'Unknown error')}"
+                )
+
+            # If key is fresh (just created), wait for IAM policy to propagate
+            if key_result.get("fresh", False):
+                time.sleep(8)
+
+            topic_name = f"topic-{str(uuid.uuid4())[:5]}"
+            kafka_topic = f"{client.namespace}.{topic_name}"
+
+            # If status is not success, throw exception
+            topic_result = client.create_topic(topic_name)
+            if topic_result.get("status") != "success":
+                raise RuntimeError(
+                    f"Failed to create topic: {topic_result.get('message', 'Unknown error')}"
+                )
+            time.sleep(3)  # Wait after topic creation before produce
+
+            producer = KafkaProducer(kafka_topic)
+            for i in range(3):
+                future = producer.send(
+                    kafka_topic, {"message_id": i + 1, "content": f"Message {i + 1}"}
+                )
+                future.get(timeout=30)
+            producer.close()
+            time.sleep(2)  # Wait for the produced messages to be consumed
+
+            consumer = KafkaConsumer(kafka_topic, auto_offset_reset="earliest")
+            consumer.poll(timeout_ms=10000)
+            consumer.close()
+
+            client.delete_topic(topic_name)
+            return kafka_topic
         except Exception as e:
-            raise e
-            print(e)
-
-    result, retry_count = {}, 0
-    start_time = time.time()
-    while len(result) < 2:  # two tests
-        if retry_count > 0:
-            print(
-                f"Block until connected or timed out ({max_minutes} minutes)... retry count:",
-                retry_count,
-                ", time passed:",
-                int(time.time() - start_time),
-                "seconds",
-            )
-        producer_connection_test(result)
-        consumer_connection_test(result)
-        retry_count += 1
-        elapsed_time = time.time() - start_time
-        if elapsed_time >= max_minutes * 60:
-            print("Time limit exceeded. Exiting loop.")
-            return False
-    return True
+            logger.info(f"Error in attempt {attempt}: {type(e).__name__}: {str(e)}")
+            if client:
+                try:
+                    if topic_name:
+                        client.delete_topic(topic_name)
+                except Exception:
+                    pass
+                try:
+                    client.delete_user()
+                except Exception:
+                    pass
+            continue

diaspora_event_sdk/sdk/login_manager/manager.py

@@ -19,6 +19,61 @@ from .login_flow import do_link_auth_flow
 log = logging.getLogger(__name__)
 
 
+class FilteredClientCredentialsAuthorizer(globus_sdk.ClientCredentialsAuthorizer):
+    """
+    A custom ClientCredentialsAuthorizer that filters token responses to only
+    include tokens for a specific resource server.
+
+    This is needed when client credentials return tokens for multiple resource
+    servers, but ClientCredentialsAuthorizer expects exactly one token.
+    """
+
+    def __init__(
+        self,
+        confidential_client: globus_sdk.ConfidentialAppAuthClient,
+        scopes: list[str],
+        *,
+        resource_server: str,
+        access_token: str | None = None,
+        expires_at: int | None = None,
+        on_refresh: t.Callable[[globus_sdk.OAuthTokenResponse], None] | None = None,
+    ) -> None:
+        self._target_resource_server = resource_server
+        # Store the original on_refresh callback
+        self._original_on_refresh = on_refresh
+        super().__init__(
+            confidential_client=confidential_client,
+            scopes=scopes,
+            access_token=access_token,
+            expires_at=expires_at,
+            on_refresh=self._filtered_on_refresh,
+        )
+
+    def _extract_token_data(
+        self, res: globus_sdk.OAuthClientCredentialsResponse
+    ) -> dict[str, t.Any]:
+        """
+        Extract token data, filtering to only the target resource server.
+        """
+        token_data = res.by_resource_server
+        if self._target_resource_server in token_data:
+            # Return only the token for the target resource server
+            return token_data[self._target_resource_server]
+        else:
+            raise ValueError(
+                f"Token response does not contain token for {self._target_resource_server}"
+            )
+
+    def _filtered_on_refresh(
+        self, token_response: globus_sdk.OAuthTokenResponse
+    ) -> None:
+        """
+        Call the original on_refresh callback with the filtered token response.
+        """
+        if self._original_on_refresh:
+            self._original_on_refresh(token_response)
+
+
 def _get_diaspora_all_scope() -> str:
     return os.getenv(
         "DIASPORA_SCOPE",
@@ -153,9 +208,13 @@ class LoginManager:
                 expires_at = tokens["expires_at_seconds"]
 
             with self._access_lock:
-                return globus_sdk.ClientCredentialsAuthorizer(
+                # Use a custom authorizer that filters token responses to only
+                # the requested resource server, handling cases where client
+                # credentials return tokens for multiple resource servers
+                return FilteredClientCredentialsAuthorizer(
                     confidential_client=get_client_login(),
                     scopes=scopes,
+                    resource_server=resource_server,
                     access_token=access_token,
                     expires_at=expires_at,
                     on_refresh=self._token_storage.on_refresh,