diaspora-event-sdk 0.0.16__py3-none-any.whl → 0.0.18__py3-none-any.whl

diaspora_event_sdk/__init__.py

@@ -7,5 +7,10 @@ __author__ = "The Diaspora Event Team"
 __version__ = _version
 
 from diaspora_event_sdk.sdk.client import Client
-from diaspora_event_sdk.sdk.kafka_client import KafkaProducer, KafkaConsumer
-__all__ = ("Client", "KafkaProducer", "KafkaConsumer")
+from diaspora_event_sdk.sdk.kafka_client import (
+    KafkaProducer,
+    KafkaConsumer,
+    block_until_ready,
+)
+
+__all__ = ("Client", "KafkaProducer", "KafkaConsumer", "block_until_ready")

diaspora_event_sdk/sdk/kafka_client.py

@@ -1,6 +1,7 @@
 import json
 from typing import Dict, Any
 import warnings
+import time
 
 from ._environments import MSK_SCRAM_ENDPOINT
 from .client import Client
@@ -8,8 +9,8 @@ from .client import Client
 # If kafka-python is not installed, Kafka functionality is not available through diaspora-event-sdk.
 kafka_available = True
 try:
-    from kafka import KafkaProducer as KProd  # type: ignore[import-not-found]
-    from kafka import KafkaConsumer as KCons  # type: ignore[import-not-found]
+    from kafka import KafkaProducer as KProd  # type: ignore[import,import-not-found]
+    from kafka import KafkaConsumer as KCons  # type: ignore[import,import-not-found]
 
 except ImportError:
     kafka_available = False
@@ -38,6 +39,7 @@ def get_diaspora_config(extra_configs: Dict[str, Any] = {}) -> Dict[str, Any]:
 
 
 if kafka_available:
+
     class KafkaProducer(KProd):
         def __init__(self, **configs):
             configs.setdefault(
@@ -53,9 +55,60 @@ else:
     class KafkaProducer:  # type: ignore[no-redef]
         def __init__(self, *args, **kwargs):
             warnings.warn(
-                "KafkaProducer is not available. Initialization is a no-op.", RuntimeWarning)
+                "KafkaProducer is not available. Initialization is a no-op.",
+                RuntimeWarning,
+            )
 
     class KafkaConsumer:  # type: ignore[no-redef]
         def __init__(self, *args, **kwargs):
             warnings.warn(
-                "KafkaConsumer is not available. Initialization is a no-op.", RuntimeWarning)
+                "KafkaConsumer is not available. Initialization is a no-op.",
+                RuntimeWarning,
+            )
+
+
+# TODO: mypy diaspora_event_sdk/sdk/kafka_client.py --disallow-untyped-defs
+def block_until_ready(max_minutes=5):
+    """
+    Test Kafka producer and consumer connections.
+    By default, this method blocks for five minutes before giving up.
+    It returns a boolean that indicates whether the connections can be successfully established.
+    """
+
+    def producer_connection_test(result):
+        try:
+            producer = KafkaProducer(max_block_ms=10 * 1000)
+            future = producer.send(
+                topic="__connection_test",
+                value={"message": "Synchronous message from Diaspora SDK"},
+            )
+            result["producer_connection_test"] = future.get(timeout=10)
+        except Exception as e:
+            pass
+
+    def consumer_connection_test(result):
+        try:
+            consumer = KafkaConsumer(
+                "__connection_test",
+                consumer_timeout_ms=10 * 1000,
+                auto_offset_reset="earliest",
+            )
+            for msg in consumer:
+                result["consumer_connection_test"] = msg
+                break
+        except Exception as e:
+            pass
+
+    result, retry_count = {}, 0
+    start_time = time.time()
+    while len(result) < 2:  # two tests
+        if retry_count > 0:
+            print(f"Block until connected or timed out ({max_minutes} minutes)... retry count:", retry_count, ", time passed:", int(time.time() - start_time), "seconds")
+        producer_connection_test(result)
+        consumer_connection_test(result)
+        retry_count += 1
+        elapsed_time = time.time() - start_time
+        if elapsed_time >= max_minutes * 60:
+            print("Time limit exceeded. Exiting loop.")
+            return False
+    return True

diaspora_event_sdk/sdk/login_manager/tokenstore.py

@@ -18,29 +18,10 @@ def _home() -> pathlib.Path:
     return pathlib.Path.home()
 
 
-def invalidate_old_config() -> None:
-    token_file = _home() / ".funcx" / "credentials" / "funcx_sdk_tokens.json"
-
-    if token_file.exists():
-        try:
-            auth_client = internal_auth_client()
-            with open(token_file) as fp:
-                data = json.load(fp)
-            for token_data in data.values():
-                if "access_token" in token_data:
-                    auth_client.oauth2_revoke_token(token_data["access_token"])
-                if "refresh_token" in token_data:
-                    auth_client.oauth2_revoke_token(
-                        token_data["refresh_token"])
-        finally:
-            os.remove(token_file)
-
-
-def ensure_compute_dir() -> pathlib.Path:
-    legacy_dirname = _home() / ".funcx"
-    dirname = _home() / ".globus_compute"
-
-    user_dir = os.getenv("GLOBUS_COMPUTE_USER_DIR")
+def ensure_diaspora_dir() -> pathlib.Path:
+    dirname = _home() / ".diaspora"
+
+    user_dir = os.getenv("DIASPORA_USER_DIR")
     if user_dir:
         dirname = pathlib.Path(user_dir)
 
@@ -51,9 +32,6 @@ def ensure_compute_dir() -> pathlib.Path:
             f"Error creating directory {dirname}, "
             "please remove or rename the conflicting file"
         )
-    elif legacy_dirname.is_dir() and not user_dir:
-        legacy_dirname.replace(dirname)
-        legacy_dirname.symlink_to(dirname, target_is_directory=True)
     else:
         dirname.mkdir(mode=0o700, parents=True, exist_ok=True)
 
@@ -61,7 +39,7 @@ def ensure_compute_dir() -> pathlib.Path:
 
 
 def _get_storage_filename():
-    datadir = ensure_compute_dir()
+    datadir = ensure_diaspora_dir()
     return os.path.join(datadir, "storage.db")
 
 
@@ -90,8 +68,6 @@ def get_token_storage_adapter(*, environment: str | None = None) -> SQLiteAdapte
     # when initializing the token storage adapter, check if the storage file exists
     # if it does not, then use this as a flag to clean the old config
     fname = _get_storage_filename()
-    if not os.path.exists(fname):
-        invalidate_old_config()
     # namespace is equal to the current environment
     return SQLiteAdapter(
         fname,

diaspora_event_sdk/version.py

@@ -1 +1 @@
-__version__ = "0.0.16"
+__version__ = "0.0.18"

{diaspora_event_sdk-0.0.16.dist-info → diaspora_event_sdk-0.0.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: diaspora-event-sdk
-Version: 0.0.16
+Version: 0.0.18
 Summary: SDK of Diaspora Event Fabric: Resilience-enabling services for science from HPC to edge
 Home-page: https://github.com/globus-labs/diaspora-event-sdk
 License: LICENSE
@@ -9,26 +9,36 @@ License-File: LICENSE
 Requires-Dist: globus-sdk <4,>=3.20.1
 Provides-Extra: kafka-python
 Requires-Dist: kafka-python ; extra == 'kafka-python'
+Provides-Extra: test
+Requires-Dist: pytest ; extra == 'test'
+Requires-Dist: pytest-cov ; extra == 'test'
+Requires-Dist: coverage ; extra == 'test'
+Requires-Dist: mypy ; extra == 'test'
+Requires-Dist: tox ; extra == 'test'
+Requires-Dist: check-manifest ; extra == 'test'
 
 <h1>Diaspora Event Fabric: Resilience-enabling services for science from HPC to edge</h1>
 
 - [Installation](#installation)
-   * [Recommended Installation with Kafka Client Library](#recommended-installation-with-kafka-client-library)
-   * [Installation Without Kafka Client Library](#installation-without-kafka-client-library)
+  * [Recommended Installation with Kafka Client Library](#recommended-installation-with-kafka-client-library)
+  * [Installation Without Kafka Client Library](#installation-without-kafka-client-library)
 - [Use Diaspora Event SDK](#use-diaspora-event-sdk)
-   * [Use the SDK to communicate with Kafka (kafka-python Required)](#use-the-sdk-to-communicate-with-kafka-kafka-python-required)
-      + [Register Topic (create topic ACLs)](#register-topic-create-topic-acls)
-      + [Start Producer](#start-producer)
-      + [Start Consumer](#start-consumer)
-      + [Unregister Topic (remove topic ACLs)](#unregister-topic-remove-topic-acls)
-   * [Use Your Preferred Kafka Client Library](#use-your-preferred-kafka-client-library)
-      + [Register and Unregister Topic](#register-and-unregister-topic)
-      + [Cluster Connection Details](#cluster-connection-details)
-   * [Advanced Usage](#advanced-usage)
-      + [Password Refresh](#password-refresh)
+  * [Use the SDK to communicate with Kafka (kafka-python Required)](#use-the-sdk-to-communicate-with-kafka--kafka-python-required-)
+    + [Register Topic (create topic ACLs)](#register-topic--create-topic-acls-)
+    + [Block Until Ready](#block-until-ready)
+    + [Start Producer](#start-producer)
+    + [Start Consumer](#start-consumer)
+    + [Unregister Topic (remove topic ACLs)](#unregister-topic--remove-topic-acls-)
+  * [Use Your Preferred Kafka Client Library](#use-your-preferred-kafka-client-library)
+    + [Register and Unregister Topic](#register-and-unregister-topic)
+    + [Cluster Connection Details](#cluster-connection-details)
+  * [Advanced Usage](#advanced-usage)
+    + [Password Refresh](#password-refresh)
 - [Common Issues](#common-issues)
-   * [ImportError: cannot import name 'KafkaProducer' from 'diaspora_event_sdk'](#importerror-cannot-import-name-kafkaproducer-from-diaspora_event_sdk)
-   * [kafka.errors.NoBrokersAvailable and kafka.errors.NodeNotReadyError](#kafkaerrorsnobrokersavailable-and-kafkaerrorsnodenotreadyerror)
+  * [ImportError: cannot import name 'KafkaProducer' from 'diaspora_event_sdk'](#importerror--cannot-import-name--kafkaproducer--from--diaspora-event-sdk-)
+  * [kafka.errors.NoBrokersAvailable and kafka.errors.NodeNotReadyError](#kafkaerrorsnobrokersavailable-and-kafkaerrorsnodenotreadyerror)
+  * [kafka.errors.KafkaTimeoutError: KafkaTimeoutError: Failed to update metadata after 60.0 secs.](#kafkaerrorskafkatimeouterror--kafkatimeouterror--failed-to-update-metadata-after-600-secs)
+  * [ssl.SSLCertVerificationError](#sslsslcertverificationerror)
 
 
 ## Installation
@@ -65,6 +75,14 @@ print(c.list_topics())
 ```
 Register a topic also creates it, if the topic previously does not exist.
 
+#### Block Until Ready
+`KafkaProducer` and `KafkaConsumer` would internally call `create_key` if the the connection credential is not found locally (e.g., when you first authenticated with Globus). Behind the sence, the middle service contacts AWS to initialize the asynchronous process of creating and associating the secret. The method below blocks until the credential is ready to be used by producer and consumer. When the method finishes, it returns True and the producer and consumer code below should work without further waiting. By default, the method retries in loop for five minutes before giving up and return False. Use parameter `max_minutes` to change the number of minutes of max waiting.
+
+```python 
+from diaspora_event_sdk import block_until_ready
+assert block_until_ready()
+```
+
 #### Start Producer
 
 Once the topic is created we can publish to it. The KafkaProducer wraps the [Python KafkaProducer](https://kafka-python.readthedocs.io/en/master/apidoc/KafkaProducer.html) Event publication can be either synchronous or asynchronous. Below demonstrates the synchronous approach. 

{diaspora_event_sdk-0.0.16.dist-info → diaspora_event_sdk-0.0.18.dist-info}/RECORD

@@ -1,10 +1,10 @@
-diaspora_event_sdk/__init__.py,sha256=ZMeMHPNphLB61jBsOO6kCRUtCWYBtvvNvl1QhwHvrqI,399
-diaspora_event_sdk/version.py,sha256=8ss7zPyQ3YfaQJw9IIGX35NOL7ASZ7-LErAxKnQDN7c,23
+diaspora_event_sdk/__init__.py,sha256=v8IN3-WFpliakQKru8TAcmQ4IRdvRe_m9-abSDnGIFM,457
+diaspora_event_sdk/version.py,sha256=qgKF-lRlzaBqf95e1sodHCZkSUCbz7ECKSeYHwXfAvI,23
 diaspora_event_sdk/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 diaspora_event_sdk/sdk/_environments.py,sha256=UwzEeBVRuP7ZyqTJagNVA24EiCLuOYiy395q608AwQ0,358
 diaspora_event_sdk/sdk/client.py,sha256=2KMUdnOEtSe1uV3rpRZHd5tLOFgC_fztzny1XxL4rh8,3496
 diaspora_event_sdk/sdk/decorators.py,sha256=Gel8AyhIjbf4-FNintTNcOqvC9hHH_YwbOH257Nfmf0,884
-diaspora_event_sdk/sdk/kafka_client.py,sha256=uibvx41bpHP4dv-A2trmJKb2oovzH4FQ6FUBLC9DLlg,2122
+diaspora_event_sdk/sdk/kafka_client.py,sha256=e_c9M1L-HAV346iwR7tsH7ZZUON1tgSDCGlOkvNKdWQ,3959
 diaspora_event_sdk/sdk/web_client.py,sha256=RCXZhMtPBe53c4-MmyBiFIdqsmcTOnV3ashaaQHKMew,1325
 diaspora_event_sdk/sdk/login_manager/__init__.py,sha256=yeqVgjeHLMX0WZJu2feJmq-fbeXvSxWghVV81ygfY-w,239
 diaspora_event_sdk/sdk/login_manager/client_login.py,sha256=gvR4PkIqQpIywNieJQ_u11PHUmdLxQ0Ho-QgPSfu8bw,1798
@@ -13,13 +13,13 @@ diaspora_event_sdk/sdk/login_manager/globus_auth.py,sha256=9Hymp0tv91OI5dBMUgh4r
 diaspora_event_sdk/sdk/login_manager/login_flow.py,sha256=2TodgsvlEYPoZPQPkp6FHOC9IkSM07pS7MIVGS4MZNE,954
 diaspora_event_sdk/sdk/login_manager/manager.py,sha256=uJHosI7ipqGt5KiL-iv2IGB4bNyZ6jQZQi7g4ka0pts,7052
 diaspora_event_sdk/sdk/login_manager/protocol.py,sha256=RCuo2jy_XkpZvbxnKlDfTKs-L6b9_8_JR-Kq9wHwhoM,710
-diaspora_event_sdk/sdk/login_manager/tokenstore.py,sha256=7jRm01rzsbvniaCfYtDDWE3det_1_b6oQkS-YQ2Qjg4,3037
+diaspora_event_sdk/sdk/login_manager/tokenstore.py,sha256=Kq0IZGf9G9dE44yoyUZInod5xL_8caN9OugeTK6GBGg,2094
 diaspora_event_sdk/sdk/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 diaspora_event_sdk/sdk/utils/uuid_like.py,sha256=xbxf0YXpDhdii16lwPLWRN21qFekHrNrqODSToMPtCg,470
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/unit/test_client.py,sha256=KjqXW7Mm3fm17lqhYdixL8s50z83F6g7MGKcOXJIyFc,2635
-diaspora_event_sdk-0.0.16.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-diaspora_event_sdk-0.0.16.dist-info/METADATA,sha256=Hf_rvVt37hJn-iunRpkYO0pxP_gtJl_GbWdmXvTposs,8181
-diaspora_event_sdk-0.0.16.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-diaspora_event_sdk-0.0.16.dist-info/top_level.txt,sha256=OVun-67t3fkLFEIwvJuNINgFFvAc--bClYhXjLhMmvs,25
-diaspora_event_sdk-0.0.16.dist-info/RECORD,,
+diaspora_event_sdk-0.0.18.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+diaspora_event_sdk-0.0.18.dist-info/METADATA,sha256=4YlQ90MxHz4T1ILh2wrj_H97ouosqI_OX1OcRm7SQsI,9542
+diaspora_event_sdk-0.0.18.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+diaspora_event_sdk-0.0.18.dist-info/top_level.txt,sha256=OVun-67t3fkLFEIwvJuNINgFFvAc--bClYhXjLhMmvs,25
+diaspora_event_sdk-0.0.18.dist-info/RECORD,,