asbflow 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- asbflow/__init__.py +75 -0
- asbflow/auth/__init__.py +13 -0
- asbflow/auth/base.py +50 -0
- asbflow/auth/factory.py +42 -0
- asbflow/auth/providers.py +126 -0
- asbflow/config/__init__.py +40 -0
- asbflow/config/connection.py +96 -0
- asbflow/config/consumer.py +115 -0
- asbflow/config/defaults.py +119 -0
- asbflow/config/entity.py +28 -0
- asbflow/config/publisher.py +91 -0
- asbflow/consumer/__init__.py +29 -0
- asbflow/consumer/base.py +160 -0
- asbflow/consumer/factory.py +135 -0
- asbflow/consumer/failure_handler.py +154 -0
- asbflow/consumer/result.py +115 -0
- asbflow/consumer/service.py +426 -0
- asbflow/consumer/strategies/__init__.py +9 -0
- asbflow/consumer/strategies/async_strategy.py +180 -0
- asbflow/consumer/strategies/sequential.py +110 -0
- asbflow/consumer/strategies/thread_pool.py +129 -0
- asbflow/dlq/__init__.py +23 -0
- asbflow/dlq/factory.py +101 -0
- asbflow/dlq/protocols.py +44 -0
- asbflow/dlq/result.py +132 -0
- asbflow/dlq/service.py +314 -0
- asbflow/entity/__init__.py +10 -0
- asbflow/entity/base.py +38 -0
- asbflow/entity/factory.py +32 -0
- asbflow/entity/providers.py +54 -0
- asbflow/exceptions.py +44 -0
- asbflow/publisher/__init__.py +29 -0
- asbflow/publisher/base.py +161 -0
- asbflow/publisher/factory.py +146 -0
- asbflow/publisher/service.py +309 -0
- asbflow/publisher/strategies/__init__.py +9 -0
- asbflow/publisher/strategies/async_strategy.py +116 -0
- asbflow/publisher/strategies/sequential.py +57 -0
- asbflow/publisher/strategies/thread_pool.py +73 -0
- asbflow/py.typed +0 -0
- asbflow/shared/__init__.py +45 -0
- asbflow/shared/asb_ops.py +112 -0
- asbflow/shared/message.py +17 -0
- asbflow/shared/parsing.py +103 -0
- asbflow/shared/payloads.py +80 -0
- asbflow/shared/sdk.py +168 -0
- asbflow-1.0.0.dist-info/METADATA +164 -0
- asbflow-1.0.0.dist-info/RECORD +51 -0
- asbflow-1.0.0.dist-info/WHEEL +5 -0
- asbflow-1.0.0.dist-info/licenses/LICENSE +21 -0
- asbflow-1.0.0.dist-info/top_level.txt +1 -0
asbflow/__init__.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
2
|
+
|
|
3
|
+
from asbflow.config import (
|
|
4
|
+
ASBAuthMethod,
|
|
5
|
+
ASBConnectionConfig,
|
|
6
|
+
ASBConsumerConfig,
|
|
7
|
+
ASBMessageConfig,
|
|
8
|
+
ASBMessagingEntity,
|
|
9
|
+
ASBPublisherConfig,
|
|
10
|
+
ParseFailurePolicy,
|
|
11
|
+
)
|
|
12
|
+
from asbflow.consumer import (
|
|
13
|
+
ASBConsumer,
|
|
14
|
+
ConsumedPayloadFailure,
|
|
15
|
+
ConsumeExecutionMode,
|
|
16
|
+
ConsumeResult,
|
|
17
|
+
ParsedConsumeResult,
|
|
18
|
+
PydanticModelParser,
|
|
19
|
+
RawConsumeResult,
|
|
20
|
+
create_consumer,
|
|
21
|
+
)
|
|
22
|
+
from asbflow.dlq import (
|
|
23
|
+
ASBDLQManager,
|
|
24
|
+
DLQParsedReadResult,
|
|
25
|
+
DLQPurgeResult,
|
|
26
|
+
DLQRawReadResult,
|
|
27
|
+
DLQReadResult,
|
|
28
|
+
DLQRedriveResult,
|
|
29
|
+
create_dlq_manager,
|
|
30
|
+
)
|
|
31
|
+
from asbflow.exceptions import (
|
|
32
|
+
ConsumeError,
|
|
33
|
+
DLQError,
|
|
34
|
+
DLQPublisherNotConfiguredError,
|
|
35
|
+
PublishError,
|
|
36
|
+
)
|
|
37
|
+
from asbflow.publisher import ASBPublisher, PublishExecutionMode, create_publisher
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
__version__ = version("asbflow")
|
|
41
|
+
except PackageNotFoundError:
|
|
42
|
+
__version__ = "0.0.0"
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
"__version__",
|
|
46
|
+
"ASBConnectionConfig",
|
|
47
|
+
"ASBPublisherConfig",
|
|
48
|
+
"ASBConsumerConfig",
|
|
49
|
+
"ASBMessagingEntity",
|
|
50
|
+
"ParseFailurePolicy",
|
|
51
|
+
"ASBAuthMethod",
|
|
52
|
+
"ASBMessageConfig",
|
|
53
|
+
"ASBPublisher",
|
|
54
|
+
"create_publisher",
|
|
55
|
+
"PublishExecutionMode",
|
|
56
|
+
"ASBConsumer",
|
|
57
|
+
"create_consumer",
|
|
58
|
+
"ASBDLQManager",
|
|
59
|
+
"create_dlq_manager",
|
|
60
|
+
"ConsumeExecutionMode",
|
|
61
|
+
"RawConsumeResult",
|
|
62
|
+
"ParsedConsumeResult",
|
|
63
|
+
"ConsumeResult",
|
|
64
|
+
"ConsumedPayloadFailure",
|
|
65
|
+
"DLQRawReadResult",
|
|
66
|
+
"DLQParsedReadResult",
|
|
67
|
+
"DLQReadResult",
|
|
68
|
+
"DLQRedriveResult",
|
|
69
|
+
"DLQPurgeResult",
|
|
70
|
+
"ConsumeError",
|
|
71
|
+
"PublishError",
|
|
72
|
+
"DLQError",
|
|
73
|
+
"DLQPublisherNotConfiguredError",
|
|
74
|
+
"PydanticModelParser",
|
|
75
|
+
]
|
asbflow/auth/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from asbflow.auth.base import ASBClientProvider
|
|
2
|
+
from asbflow.auth.factory import ASBClientProviderFactory
|
|
3
|
+
from asbflow.auth.providers import (
|
|
4
|
+
ConnectionStringClientProvider,
|
|
5
|
+
ManagedIdentityClientProvider,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"ASBClientProvider",
|
|
10
|
+
"ASBClientProviderFactory",
|
|
11
|
+
"ConnectionStringClientProvider",
|
|
12
|
+
"ManagedIdentityClientProvider",
|
|
13
|
+
]
|
asbflow/auth/base.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
|
|
5
|
+
from asbflow.config.connection import ASBConnectionConfig
|
|
6
|
+
from asbflow.shared.sdk import ASBAsyncClient, ASBSyncClient
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ASBClientProvider(ABC):
|
|
10
|
+
"""Base abstraction for Azure Service Bus client providers."""
|
|
11
|
+
|
|
12
|
+
def __init__(self, config: ASBConnectionConfig) -> None:
|
|
13
|
+
"""Initialize the provider.
|
|
14
|
+
|
|
15
|
+
Parameters
|
|
16
|
+
----------
|
|
17
|
+
config : ASBConnectionConfig
|
|
18
|
+
Connection/auth configuration used to create clients.
|
|
19
|
+
"""
|
|
20
|
+
self._config: ASBConnectionConfig = config
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def config(self) -> ASBConnectionConfig:
|
|
24
|
+
"""Return the connection configuration used by this provider."""
|
|
25
|
+
return self._config
|
|
26
|
+
|
|
27
|
+
@abstractmethod
|
|
28
|
+
def create_sync_client(self) -> ASBSyncClient:
|
|
29
|
+
"""Create a synchronous Azure Service Bus client.
|
|
30
|
+
|
|
31
|
+
Returns
|
|
32
|
+
-------
|
|
33
|
+
ASBSyncClient
|
|
34
|
+
Sync client implementing context manager semantics.
|
|
35
|
+
"""
|
|
36
|
+
raise NotImplementedError
|
|
37
|
+
|
|
38
|
+
@abstractmethod
|
|
39
|
+
async def create_async_client(self) -> ASBAsyncClient:
|
|
40
|
+
"""Create an asynchronous Azure Service Bus client.
|
|
41
|
+
|
|
42
|
+
Returns
|
|
43
|
+
-------
|
|
44
|
+
ASBAsyncClient
|
|
45
|
+
Async client implementing async context manager semantics.
|
|
46
|
+
"""
|
|
47
|
+
raise NotImplementedError
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
__all__ = ["ASBClientProvider"]
|
asbflow/auth/factory.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from asbflow.auth.base import ASBClientProvider
|
|
4
|
+
from asbflow.auth.providers import (
|
|
5
|
+
ConnectionStringClientProvider,
|
|
6
|
+
ManagedIdentityClientProvider,
|
|
7
|
+
)
|
|
8
|
+
from asbflow.config.connection import ASBAuthMethod, ASBConnectionConfig
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ASBClientProviderFactory:
|
|
12
|
+
"""Build the client provider matching the configured auth method."""
|
|
13
|
+
|
|
14
|
+
@staticmethod
|
|
15
|
+
def create(config: ASBConnectionConfig) -> ASBClientProvider:
|
|
16
|
+
"""Create an auth provider.
|
|
17
|
+
|
|
18
|
+
Parameters
|
|
19
|
+
----------
|
|
20
|
+
config : ASBConnectionConfig
|
|
21
|
+
Connection/auth configuration.
|
|
22
|
+
|
|
23
|
+
Returns
|
|
24
|
+
-------
|
|
25
|
+
ASBClientProvider
|
|
26
|
+
Provider implementation compatible with the selected auth method.
|
|
27
|
+
|
|
28
|
+
Raises
|
|
29
|
+
------
|
|
30
|
+
ValueError
|
|
31
|
+
If the auth method is unsupported.
|
|
32
|
+
"""
|
|
33
|
+
match config.auth:
|
|
34
|
+
case ASBAuthMethod.CONNECTION_STRING:
|
|
35
|
+
return ConnectionStringClientProvider(config)
|
|
36
|
+
case ASBAuthMethod.MANAGED_IDENTITY:
|
|
37
|
+
return ManagedIdentityClientProvider(config)
|
|
38
|
+
|
|
39
|
+
raise ValueError(f"Unsupported Service Bus auth method: {config.auth}")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
__all__ = ["ASBClientProviderFactory"]
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from importlib import import_module
|
|
4
|
+
from types import ModuleType
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from asbflow.auth.base import ASBClientProvider
|
|
8
|
+
from asbflow.config.connection import ASBConnectionConfig
|
|
9
|
+
from asbflow.config.defaults import get_asbflow_logger
|
|
10
|
+
from asbflow.shared.sdk import (
|
|
11
|
+
ASBAsyncClient,
|
|
12
|
+
ASBAsyncClientFactory,
|
|
13
|
+
ASBSyncClient,
|
|
14
|
+
ASBSyncClientFactory,
|
|
15
|
+
load_asb_async_client,
|
|
16
|
+
load_asb_client_factory,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
LOGGER = get_asbflow_logger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ConnectionStringClientProvider(ASBClientProvider):
|
|
23
|
+
"""Create clients using Service Bus connection-string authentication."""
|
|
24
|
+
|
|
25
|
+
def create_sync_client(self) -> ASBSyncClient:
|
|
26
|
+
"""Create a synchronous client from connection-string auth.
|
|
27
|
+
|
|
28
|
+
Returns
|
|
29
|
+
-------
|
|
30
|
+
ASBSyncClient
|
|
31
|
+
Synchronous Service Bus client.
|
|
32
|
+
"""
|
|
33
|
+
LOGGER.info("Creating synchronous ASB client with connection-string auth")
|
|
34
|
+
servicebus_client: ASBSyncClientFactory = load_asb_client_factory()
|
|
35
|
+
LOGGER.debug(
|
|
36
|
+
"Loaded sync ASB factory class: %s",
|
|
37
|
+
type(servicebus_client).__name__,
|
|
38
|
+
)
|
|
39
|
+
return servicebus_client.from_connection_string(**self.config.to_connection_string_client_kwargs())
|
|
40
|
+
|
|
41
|
+
async def create_async_client(self) -> ASBAsyncClient:
|
|
42
|
+
"""Create an asynchronous client from connection-string auth.
|
|
43
|
+
|
|
44
|
+
Returns
|
|
45
|
+
-------
|
|
46
|
+
ASBAsyncClient
|
|
47
|
+
Asynchronous Service Bus client.
|
|
48
|
+
"""
|
|
49
|
+
LOGGER.info("Creating asynchronous ASB client with connection-string auth")
|
|
50
|
+
servicebus_client: ASBAsyncClientFactory = load_asb_async_client()
|
|
51
|
+
LOGGER.debug(
|
|
52
|
+
"Loaded async ASB factory class: %s",
|
|
53
|
+
type(servicebus_client).__name__,
|
|
54
|
+
)
|
|
55
|
+
return servicebus_client.from_connection_string(**self.config.to_connection_string_client_kwargs())
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ManagedIdentityClientProvider(ASBClientProvider):
|
|
59
|
+
"""Create clients using Azure Managed Identity authentication."""
|
|
60
|
+
|
|
61
|
+
def _build_credential(self) -> Any:
|
|
62
|
+
config: ASBConnectionConfig = self.config
|
|
63
|
+
if config.credential is not None:
|
|
64
|
+
LOGGER.debug("Using explicit credential instance from configuration")
|
|
65
|
+
return self.config.credential
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
identity_module: ModuleType = import_module("azure.identity")
|
|
69
|
+
except Exception:
|
|
70
|
+
LOGGER.exception("Unable to import azure.identity while building managed-identity credential")
|
|
71
|
+
raise
|
|
72
|
+
|
|
73
|
+
credential_class: Any = getattr(identity_module, "ManagedIdentityCredential")
|
|
74
|
+
if config.managed_identity_client_id is not None:
|
|
75
|
+
LOGGER.debug("Building ManagedIdentityCredential with client_id")
|
|
76
|
+
return credential_class(client_id=config.managed_identity_client_id)
|
|
77
|
+
|
|
78
|
+
LOGGER.debug("Building default ManagedIdentityCredential")
|
|
79
|
+
return credential_class()
|
|
80
|
+
|
|
81
|
+
def _build_kwargs(self) -> dict[str, Any]:
|
|
82
|
+
credential: Any = self._build_credential()
|
|
83
|
+
LOGGER.debug(
|
|
84
|
+
"Managed-identity client kwargs prepared (namespace=%s, logging_enable=%s)",
|
|
85
|
+
self.config.fully_qualified_namespace,
|
|
86
|
+
self.config.logging_enable,
|
|
87
|
+
)
|
|
88
|
+
return self.config.to_managed_identity_client_kwargs(credential=credential)
|
|
89
|
+
|
|
90
|
+
def create_sync_client(self) -> ASBSyncClient:
|
|
91
|
+
"""Create a synchronous client from managed-identity auth.
|
|
92
|
+
|
|
93
|
+
Returns
|
|
94
|
+
-------
|
|
95
|
+
ASBSyncClient
|
|
96
|
+
Synchronous Service Bus client.
|
|
97
|
+
"""
|
|
98
|
+
LOGGER.info("Creating synchronous ASB client with managed-identity auth")
|
|
99
|
+
servicebus_client: ASBSyncClientFactory = load_asb_client_factory()
|
|
100
|
+
LOGGER.debug(
|
|
101
|
+
"Loaded sync ASB factory class: %s",
|
|
102
|
+
type(servicebus_client).__name__,
|
|
103
|
+
)
|
|
104
|
+
return servicebus_client(**self._build_kwargs())
|
|
105
|
+
|
|
106
|
+
async def create_async_client(self) -> ASBAsyncClient:
|
|
107
|
+
"""Create an asynchronous client from managed-identity auth.
|
|
108
|
+
|
|
109
|
+
Returns
|
|
110
|
+
-------
|
|
111
|
+
ASBAsyncClient
|
|
112
|
+
Asynchronous Service Bus client.
|
|
113
|
+
"""
|
|
114
|
+
LOGGER.info("Creating asynchronous ASB client with managed-identity auth")
|
|
115
|
+
servicebus_client: ASBAsyncClientFactory = load_asb_async_client()
|
|
116
|
+
LOGGER.debug(
|
|
117
|
+
"Loaded async ASB factory class: %s",
|
|
118
|
+
type(servicebus_client).__name__,
|
|
119
|
+
)
|
|
120
|
+
return servicebus_client(**self._build_kwargs())
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
__all__ = [
|
|
124
|
+
"ConnectionStringClientProvider",
|
|
125
|
+
"ManagedIdentityClientProvider",
|
|
126
|
+
]
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from asbflow.config.connection import ASBAuthMethod, ASBConnectionConfig
|
|
2
|
+
from asbflow.config.consumer import ASBConsumerConfig, ParseFailurePolicy
|
|
3
|
+
from asbflow.config.defaults import (
|
|
4
|
+
ASBFLOW_LOGGER_NAME,
|
|
5
|
+
DEFAULT_ASB_SDK_IMPORT_MODE,
|
|
6
|
+
DEFAULT_ASBFLOW_LOG_LEVEL,
|
|
7
|
+
DEFAULT_CHUNK_SIZE,
|
|
8
|
+
DEFAULT_CONSUME_THREAD_POOL_MAX_WORKERS,
|
|
9
|
+
DEFAULT_DLQ_PURGE_MAX_MESSAGE_COUNT,
|
|
10
|
+
DEFAULT_DLQ_READ_MAX_MESSAGE_COUNT,
|
|
11
|
+
DEFAULT_MAX_MESSAGE_COUNT,
|
|
12
|
+
DEFAULT_PUBLISH_THREAD_POOL_MAX_WORKERS,
|
|
13
|
+
ASBSDKImportMode,
|
|
14
|
+
configure_asbflow_logging,
|
|
15
|
+
get_asbflow_logger,
|
|
16
|
+
)
|
|
17
|
+
from asbflow.config.entity import ASBMessagingEntity
|
|
18
|
+
from asbflow.config.publisher import ASBMessageConfig, ASBPublisherConfig
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"ASBConnectionConfig",
|
|
22
|
+
"ASBAuthMethod",
|
|
23
|
+
"ASBPublisherConfig",
|
|
24
|
+
"ASBConsumerConfig",
|
|
25
|
+
"ASBMessagingEntity",
|
|
26
|
+
"ParseFailurePolicy",
|
|
27
|
+
"ASBMessageConfig",
|
|
28
|
+
"DEFAULT_MAX_MESSAGE_COUNT",
|
|
29
|
+
"DEFAULT_DLQ_READ_MAX_MESSAGE_COUNT",
|
|
30
|
+
"DEFAULT_DLQ_PURGE_MAX_MESSAGE_COUNT",
|
|
31
|
+
"DEFAULT_CHUNK_SIZE",
|
|
32
|
+
"DEFAULT_PUBLISH_THREAD_POOL_MAX_WORKERS",
|
|
33
|
+
"DEFAULT_CONSUME_THREAD_POOL_MAX_WORKERS",
|
|
34
|
+
"ASBSDKImportMode",
|
|
35
|
+
"DEFAULT_ASB_SDK_IMPORT_MODE",
|
|
36
|
+
"ASBFLOW_LOGGER_NAME",
|
|
37
|
+
"DEFAULT_ASBFLOW_LOG_LEVEL",
|
|
38
|
+
"get_asbflow_logger",
|
|
39
|
+
"configure_asbflow_logging",
|
|
40
|
+
]
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from enum import Enum
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ASBAuthMethod(str, Enum):
|
|
9
|
+
CONNECTION_STRING = "connection_string"
|
|
10
|
+
MANAGED_IDENTITY = "managed_identity"
|
|
11
|
+
|
|
12
|
+
@classmethod
|
|
13
|
+
def parse(cls, value: "ASBAuthMethod | str") -> "ASBAuthMethod":
|
|
14
|
+
if isinstance(value, cls):
|
|
15
|
+
return value
|
|
16
|
+
|
|
17
|
+
normalized = value.strip().lower().replace("-", "_").replace(" ", "_")
|
|
18
|
+
aliases = {
|
|
19
|
+
"connection_string": cls.CONNECTION_STRING,
|
|
20
|
+
"conn_str": cls.CONNECTION_STRING,
|
|
21
|
+
"managed_identity": cls.MANAGED_IDENTITY,
|
|
22
|
+
"mi": cls.MANAGED_IDENTITY,
|
|
23
|
+
}
|
|
24
|
+
try:
|
|
25
|
+
return aliases[normalized]
|
|
26
|
+
except KeyError as exc:
|
|
27
|
+
raise ValueError(
|
|
28
|
+
f"Unknown auth method: {value}. " "Supported values are connection_string and managed_identity."
|
|
29
|
+
) from exc
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True, slots=True)
|
|
33
|
+
class ASBConnectionConfig:
|
|
34
|
+
auth_method: ASBAuthMethod | str = ASBAuthMethod.CONNECTION_STRING
|
|
35
|
+
connection_string: str | None = None
|
|
36
|
+
fully_qualified_namespace: str | None = None
|
|
37
|
+
managed_identity_client_id: str | None = None
|
|
38
|
+
credential: Any | None = None
|
|
39
|
+
logging_enable: bool = False
|
|
40
|
+
transport_type: Any | None = None
|
|
41
|
+
http_proxy: dict[str, Any] | None = None
|
|
42
|
+
user_agent: str | None = None
|
|
43
|
+
custom_endpoint_address: str | None = None
|
|
44
|
+
client_kwargs: dict[str, Any] = field(default_factory=dict)
|
|
45
|
+
|
|
46
|
+
def __post_init__(self) -> None:
|
|
47
|
+
resolved_auth_method: ASBAuthMethod = ASBAuthMethod.parse(self.auth_method)
|
|
48
|
+
object.__setattr__(self, "auth_method", resolved_auth_method)
|
|
49
|
+
object.__setattr__(self, "client_kwargs", dict(self.client_kwargs))
|
|
50
|
+
|
|
51
|
+
if resolved_auth_method is ASBAuthMethod.CONNECTION_STRING and not self.connection_string:
|
|
52
|
+
raise ValueError("connection_string is required when auth_method='connection_string'")
|
|
53
|
+
|
|
54
|
+
if resolved_auth_method is ASBAuthMethod.MANAGED_IDENTITY and not self.fully_qualified_namespace:
|
|
55
|
+
raise ValueError("fully_qualified_namespace is required when auth_method='managed_identity'")
|
|
56
|
+
|
|
57
|
+
def _base_client_kwargs(self) -> dict[str, Any]:
|
|
58
|
+
kwargs: dict[str, Any] = {
|
|
59
|
+
"logging_enable": self.logging_enable,
|
|
60
|
+
}
|
|
61
|
+
optional_map: dict[str, Any] = {
|
|
62
|
+
"transport_type": self.transport_type,
|
|
63
|
+
"http_proxy": self.http_proxy,
|
|
64
|
+
"user_agent": self.user_agent,
|
|
65
|
+
"custom_endpoint_address": self.custom_endpoint_address,
|
|
66
|
+
}
|
|
67
|
+
kwargs.update({k: v for k, v in optional_map.items() if v is not None})
|
|
68
|
+
kwargs.update(self.client_kwargs)
|
|
69
|
+
return kwargs
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def auth(self) -> ASBAuthMethod:
|
|
73
|
+
value = self.auth_method
|
|
74
|
+
if isinstance(value, ASBAuthMethod):
|
|
75
|
+
return value
|
|
76
|
+
return ASBAuthMethod.parse(value)
|
|
77
|
+
|
|
78
|
+
def to_connection_string_client_kwargs(self) -> dict[str, Any]:
|
|
79
|
+
if self.auth_method is not ASBAuthMethod.CONNECTION_STRING:
|
|
80
|
+
raise ValueError("to_connection_string_client_kwargs can be used only with connection_string auth")
|
|
81
|
+
|
|
82
|
+
kwargs = self._base_client_kwargs()
|
|
83
|
+
kwargs["conn_str"] = self.connection_string
|
|
84
|
+
return kwargs
|
|
85
|
+
|
|
86
|
+
def to_managed_identity_client_kwargs(self, *, credential: Any) -> dict[str, Any]:
|
|
87
|
+
if self.auth_method is not ASBAuthMethod.MANAGED_IDENTITY:
|
|
88
|
+
raise ValueError("to_managed_identity_client_kwargs can be used only with managed_identity auth")
|
|
89
|
+
|
|
90
|
+
kwargs = self._base_client_kwargs()
|
|
91
|
+
kwargs["fully_qualified_namespace"] = self.fully_qualified_namespace
|
|
92
|
+
kwargs["credential"] = credential
|
|
93
|
+
return kwargs
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
__all__ = ["ASBConnectionConfig", "ASBAuthMethod"]
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import asdict, dataclass, field
|
|
4
|
+
from enum import Enum
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from asbflow.config.entity import ASBMessagingEntity
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ParseFailurePolicy(str, Enum):
|
|
11
|
+
# Move invalid payloads to dead-letter queue (recommended default).
|
|
12
|
+
DEAD_LETTER = "dead_letter"
|
|
13
|
+
# Mark invalid payloads as consumed and remove them from the queue.
|
|
14
|
+
COMPLETE = "complete"
|
|
15
|
+
# Release invalid payloads back to the queue for redelivery.
|
|
16
|
+
ABANDON = "abandon"
|
|
17
|
+
# Leave invalid payloads unsettled (can cause re-delivery loops).
|
|
18
|
+
LEAVE_UNSETTLED = "leave_unsettled"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True, slots=True)
|
|
22
|
+
class ASBConsumerConfig:
|
|
23
|
+
entity_type: ASBMessagingEntity | str = ASBMessagingEntity.TOPIC
|
|
24
|
+
topic_name: str | None = None
|
|
25
|
+
subscription_name: str | None = None
|
|
26
|
+
queue_name: str | None = None
|
|
27
|
+
max_wait_time: float | None = None
|
|
28
|
+
receive_mode: Any | None = None
|
|
29
|
+
sub_queue: Any | None = None
|
|
30
|
+
prefetch_count: int | None = None
|
|
31
|
+
client_identifier: str | None = None
|
|
32
|
+
socket_timeout: float | None = None
|
|
33
|
+
parse_failure_policy: ParseFailurePolicy | str = ParseFailurePolicy.DEAD_LETTER
|
|
34
|
+
receiver_kwargs: dict[str, Any] = field(default_factory=dict)
|
|
35
|
+
|
|
36
|
+
def __post_init__(self) -> None:
|
|
37
|
+
resolved_entity: ASBMessagingEntity = ASBMessagingEntity.parse(self.entity_type)
|
|
38
|
+
object.__setattr__(self, "entity_type", resolved_entity)
|
|
39
|
+
|
|
40
|
+
if resolved_entity is ASBMessagingEntity.TOPIC:
|
|
41
|
+
if not self.topic_name:
|
|
42
|
+
raise ValueError("topic_name is required when entity_type='topic'")
|
|
43
|
+
if not self.subscription_name:
|
|
44
|
+
raise ValueError("subscription_name is required when entity_type='topic'")
|
|
45
|
+
return
|
|
46
|
+
|
|
47
|
+
if not self.queue_name:
|
|
48
|
+
raise ValueError("queue_name is required when entity_type='queue'")
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def entity(self) -> ASBMessagingEntity:
|
|
52
|
+
value = self.entity_type
|
|
53
|
+
if isinstance(value, ASBMessagingEntity):
|
|
54
|
+
return value
|
|
55
|
+
return ASBMessagingEntity.parse(value)
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def entity_name(self) -> str:
|
|
59
|
+
if self.entity is ASBMessagingEntity.TOPIC:
|
|
60
|
+
if not self.topic_name:
|
|
61
|
+
raise ValueError("topic_name is required when entity_type='topic'")
|
|
62
|
+
return self.topic_name
|
|
63
|
+
|
|
64
|
+
if not self.queue_name:
|
|
65
|
+
raise ValueError("queue_name is required when entity_type='queue'")
|
|
66
|
+
return self.queue_name
|
|
67
|
+
|
|
68
|
+
def to_receiver_kwargs(self) -> dict[str, Any]:
|
|
69
|
+
source: dict[str, Any] = asdict(self)
|
|
70
|
+
|
|
71
|
+
extra_kwargs: dict[str, Any] = source.pop("receiver_kwargs")
|
|
72
|
+
source.pop("parse_failure_policy")
|
|
73
|
+
source.pop("entity_type")
|
|
74
|
+
|
|
75
|
+
kwargs: dict[str, Any] = {k: v for k, v in source.items() if v is not None}
|
|
76
|
+
|
|
77
|
+
if self.entity is ASBMessagingEntity.TOPIC:
|
|
78
|
+
kwargs.pop("queue_name", None)
|
|
79
|
+
kwargs["topic_name"] = self.entity_name
|
|
80
|
+
if not self.subscription_name:
|
|
81
|
+
raise ValueError("subscription_name is required when entity_type='topic'")
|
|
82
|
+
kwargs["subscription_name"] = self.subscription_name
|
|
83
|
+
else:
|
|
84
|
+
kwargs.pop("topic_name", None)
|
|
85
|
+
kwargs.pop("subscription_name", None)
|
|
86
|
+
kwargs["queue_name"] = self.entity_name
|
|
87
|
+
|
|
88
|
+
kwargs.update(extra_kwargs)
|
|
89
|
+
return kwargs
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def resolved_parse_failure_policy(self) -> ParseFailurePolicy:
|
|
93
|
+
value = self.parse_failure_policy
|
|
94
|
+
if isinstance(value, ParseFailurePolicy):
|
|
95
|
+
return value
|
|
96
|
+
|
|
97
|
+
normalized: str = value.strip().lower().replace("-", "_").replace(" ", "_")
|
|
98
|
+
aliases = {
|
|
99
|
+
"dead_letter": ParseFailurePolicy.DEAD_LETTER,
|
|
100
|
+
"deadletter": ParseFailurePolicy.DEAD_LETTER,
|
|
101
|
+
"complete": ParseFailurePolicy.COMPLETE,
|
|
102
|
+
"abandon": ParseFailurePolicy.ABANDON,
|
|
103
|
+
"leave_unsettled": ParseFailurePolicy.LEAVE_UNSETTLED,
|
|
104
|
+
"leave": ParseFailurePolicy.LEAVE_UNSETTLED,
|
|
105
|
+
}
|
|
106
|
+
try:
|
|
107
|
+
return aliases[normalized]
|
|
108
|
+
except KeyError as exc:
|
|
109
|
+
raise ValueError(
|
|
110
|
+
f"Unknown parse failure policy: {value}. "
|
|
111
|
+
"Supported values are dead_letter, complete, abandon, leave_unsettled."
|
|
112
|
+
) from exc
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
__all__ = ["ASBConsumerConfig", "ParseFailurePolicy"]
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import time
|
|
5
|
+
from enum import Enum
|
|
6
|
+
from typing import Final
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ASBSDKImportMode(str, Enum):
|
|
10
|
+
# Import Azure SDK modules on first usage.
|
|
11
|
+
LAZY = "lazy"
|
|
12
|
+
# Import Azure SDK modules at asbflow SDK initialization time.
|
|
13
|
+
EAGER = "eager"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# ----------------------------------------------------
|
|
17
|
+
# Constants
|
|
18
|
+
# ----------------------------------------------------
|
|
19
|
+
|
|
20
|
+
# Default number of messages consumed per batch in generic consumers.
|
|
21
|
+
DEFAULT_MAX_MESSAGE_COUNT: Final[int] = 10
|
|
22
|
+
# Default number of DLQ messages read in a single management operation.
|
|
23
|
+
DEFAULT_DLQ_READ_MAX_MESSAGE_COUNT: Final[int] = 50
|
|
24
|
+
# Default upper bound of DLQ messages purged in one call.
|
|
25
|
+
DEFAULT_DLQ_PURGE_MAX_MESSAGE_COUNT: Final[int] = 200
|
|
26
|
+
# Default publisher chunk size. `None` means SDK-driven batching only.
|
|
27
|
+
DEFAULT_CHUNK_SIZE: Final[int | None] = None
|
|
28
|
+
# Max worker threads for publisher thread-pool strategy.
|
|
29
|
+
DEFAULT_PUBLISH_THREAD_POOL_MAX_WORKERS: Final[int] = 8
|
|
30
|
+
# Max worker threads for consumer thread-pool strategy.
|
|
31
|
+
DEFAULT_CONSUME_THREAD_POOL_MAX_WORKERS: Final[int] = 32
|
|
32
|
+
# Default Azure SDK import mode used by shared SDK loaders.
|
|
33
|
+
DEFAULT_ASB_SDK_IMPORT_MODE: Final[ASBSDKImportMode] = ASBSDKImportMode.LAZY
|
|
34
|
+
|
|
35
|
+
# ----------------------------------------------------
|
|
36
|
+
# Logger
|
|
37
|
+
# ----------------------------------------------------
|
|
38
|
+
|
|
39
|
+
# Base logger name for all asbflow logs.
|
|
40
|
+
ASBFLOW_LOGGER_NAME: Final[str] = "asbflow"
|
|
41
|
+
# Default logger level used by helper configuration.
|
|
42
|
+
DEFAULT_ASBFLOW_LOG_LEVEL: Final[int] = logging.INFO
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def get_asbflow_logger(module_name: str | None = None) -> logging.Logger:
|
|
46
|
+
"""Return a logger under the library namespace.
|
|
47
|
+
|
|
48
|
+
Parameters
|
|
49
|
+
----------
|
|
50
|
+
module_name : str | None
|
|
51
|
+
Optional module name to create a child logger.
|
|
52
|
+
|
|
53
|
+
Returns
|
|
54
|
+
-------
|
|
55
|
+
logging.Logger
|
|
56
|
+
Library logger instance.
|
|
57
|
+
"""
|
|
58
|
+
base: str = ASBFLOW_LOGGER_NAME
|
|
59
|
+
if not module_name:
|
|
60
|
+
return logging.getLogger(base)
|
|
61
|
+
|
|
62
|
+
normalized: str = module_name
|
|
63
|
+
if normalized.startswith("asbflow."):
|
|
64
|
+
normalized = normalized[len("asbflow.") :]
|
|
65
|
+
|
|
66
|
+
if not normalized:
|
|
67
|
+
return logging.getLogger(base)
|
|
68
|
+
return logging.getLogger(f"{base}.{normalized}")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def configure_asbflow_logging(
|
|
72
|
+
level: int = DEFAULT_ASBFLOW_LOG_LEVEL,
|
|
73
|
+
*,
|
|
74
|
+
add_stream_handler: bool = False,
|
|
75
|
+
) -> logging.Logger:
|
|
76
|
+
"""Configure the base asbflow logger.
|
|
77
|
+
|
|
78
|
+
Parameters
|
|
79
|
+
----------
|
|
80
|
+
level : int
|
|
81
|
+
Logging level for the base asbflow logger.
|
|
82
|
+
add_stream_handler : bool
|
|
83
|
+
If ``True``, attach a basic ``StreamHandler`` when no handlers exist.
|
|
84
|
+
|
|
85
|
+
Returns
|
|
86
|
+
-------
|
|
87
|
+
logging.Logger
|
|
88
|
+
Configured base logger.
|
|
89
|
+
"""
|
|
90
|
+
logger = get_asbflow_logger()
|
|
91
|
+
logger.setLevel(level)
|
|
92
|
+
|
|
93
|
+
if add_stream_handler and not logger.handlers:
|
|
94
|
+
handler = logging.StreamHandler()
|
|
95
|
+
formatter = logging.Formatter(
|
|
96
|
+
"%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
|
97
|
+
datefmt="%Y-%m-%dT%H:%M:%SZ",
|
|
98
|
+
)
|
|
99
|
+
formatter.converter = time.gmtime # UTC timezone
|
|
100
|
+
handler.setFormatter(formatter)
|
|
101
|
+
logger.addHandler(handler)
|
|
102
|
+
|
|
103
|
+
return logger
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
__all__ = [
|
|
107
|
+
"DEFAULT_MAX_MESSAGE_COUNT",
|
|
108
|
+
"DEFAULT_DLQ_READ_MAX_MESSAGE_COUNT",
|
|
109
|
+
"DEFAULT_DLQ_PURGE_MAX_MESSAGE_COUNT",
|
|
110
|
+
"DEFAULT_CHUNK_SIZE",
|
|
111
|
+
"DEFAULT_PUBLISH_THREAD_POOL_MAX_WORKERS",
|
|
112
|
+
"DEFAULT_CONSUME_THREAD_POOL_MAX_WORKERS",
|
|
113
|
+
"ASBSDKImportMode",
|
|
114
|
+
"DEFAULT_ASB_SDK_IMPORT_MODE",
|
|
115
|
+
"ASBFLOW_LOGGER_NAME",
|
|
116
|
+
"DEFAULT_ASBFLOW_LOG_LEVEL",
|
|
117
|
+
"get_asbflow_logger",
|
|
118
|
+
"configure_asbflow_logging",
|
|
119
|
+
]
|