modmex-lambda 0.6.1__py3-none-any.whl → 0.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modmex_lambda/stream/events/s3.py +52 -0
- modmex_lambda/stream/sources/__init__.py +18 -1
- modmex_lambda/stream/sources/s3.py +25 -1
- {modmex_lambda-0.6.1.dist-info → modmex_lambda-0.6.3.dist-info}/METADATA +1 -1
- {modmex_lambda-0.6.1.dist-info → modmex_lambda-0.6.3.dist-info}/RECORD +7 -7
- {modmex_lambda-0.6.1.dist-info → modmex_lambda-0.6.3.dist-info}/WHEEL +0 -0
- {modmex_lambda-0.6.1.dist-info → modmex_lambda-0.6.3.dist-info}/licenses/LICENSE +0 -0
|
@@ -2,6 +2,15 @@ import json
|
|
|
2
2
|
import pydash
|
|
3
3
|
from pydash import get
|
|
4
4
|
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
'from_s3',
|
|
8
|
+
'from_sqs_eventbridge_s3',
|
|
9
|
+
'from_sqs_sns_s3',
|
|
10
|
+
'to_s3_records',
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
|
|
5
14
|
def from_s3(event):
|
|
6
15
|
return pydash.map_(
|
|
7
16
|
event['Records'],
|
|
@@ -27,6 +36,41 @@ def from_sqs_sns_s3(event):
|
|
|
27
36
|
).flatten().value()
|
|
28
37
|
|
|
29
38
|
|
|
39
|
+
def from_sqs_eventbridge_s3(event):
|
|
40
|
+
"""Normalize an S3 EventBridge notification delivered through SQS."""
|
|
41
|
+
return pydash._(event['Records']).map(
|
|
42
|
+
lambda sqs_record: _from_sqs_eventbridge_record(sqs_record)
|
|
43
|
+
).flatten().value()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _from_sqs_eventbridge_record(sqs_record):
|
|
47
|
+
eventbridge_record = json.loads(sqs_record['body'])
|
|
48
|
+
detail = eventbridge_record['detail']
|
|
49
|
+
s3_record = {
|
|
50
|
+
**eventbridge_record,
|
|
51
|
+
'eventName': _eventbridge_s3_event_name(eventbridge_record),
|
|
52
|
+
's3': detail,
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return pydash.map_(from_s3({'Records': [s3_record]}), lambda uow: {
|
|
56
|
+
**uow,
|
|
57
|
+
'record': {
|
|
58
|
+
'sqs': sqs_record,
|
|
59
|
+
'eventbridge': eventbridge_record,
|
|
60
|
+
's3': uow['record'],
|
|
61
|
+
},
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _eventbridge_s3_event_name(eventbridge_record):
|
|
66
|
+
detail_type = eventbridge_record.get('detail-type', '')
|
|
67
|
+
if detail_type.startswith('Object Created'):
|
|
68
|
+
return 'ObjectCreated:Put'
|
|
69
|
+
if detail_type.startswith('Object Deleted'):
|
|
70
|
+
return 'ObjectCreated:Delete'
|
|
71
|
+
return detail_type
|
|
72
|
+
|
|
73
|
+
|
|
30
74
|
def _from_sqs_record(sqs_record):
|
|
31
75
|
sns_record = json.loads(sqs_record['body'])
|
|
32
76
|
s3_event = json.loads(sns_record['Message'])
|
|
@@ -92,3 +136,11 @@ def to_s3_records(notifications):
|
|
|
92
136
|
for i,n in enumerate(notifications)
|
|
93
137
|
]
|
|
94
138
|
}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
__all__ = [
|
|
142
|
+
'from_s3',
|
|
143
|
+
'from_sqs_sns_s3',
|
|
144
|
+
'from_sqs_eventbridge_s3',
|
|
145
|
+
'to_s3_records',
|
|
146
|
+
]
|
|
@@ -6,7 +6,14 @@ from typing import TYPE_CHECKING, Any
|
|
|
6
6
|
if TYPE_CHECKING:
|
|
7
7
|
from modmex_lambda.stream.sources.dynamodb import DynamoDBSource, dynamodb_source
|
|
8
8
|
from modmex_lambda.stream.sources.kinesis import KinesisSource, kinesis_source
|
|
9
|
-
from modmex_lambda.stream.sources.s3 import
|
|
9
|
+
from modmex_lambda.stream.sources.s3 import (
|
|
10
|
+
S3Source,
|
|
11
|
+
SqsEventBridgeS3Source,
|
|
12
|
+
SqsSnsS3Source,
|
|
13
|
+
s3_source,
|
|
14
|
+
sqs_eventbridge_s3_source,
|
|
15
|
+
sqs_sns_s3_source,
|
|
16
|
+
)
|
|
10
17
|
from modmex_lambda.stream.sources.sns import SnsSource, sns_source
|
|
11
18
|
from modmex_lambda.stream.sources.sqs import SqsSource, sqs_source
|
|
12
19
|
|
|
@@ -17,6 +24,12 @@ _EXPORTS = {
|
|
|
17
24
|
"kinesis_source": ("modmex_lambda.stream.sources.kinesis", "kinesis_source"),
|
|
18
25
|
"S3Source": ("modmex_lambda.stream.sources.s3", "S3Source"),
|
|
19
26
|
"s3_source": ("modmex_lambda.stream.sources.s3", "s3_source"),
|
|
27
|
+
"SqsSnsS3Source": ("modmex_lambda.stream.sources.s3", "SqsSnsS3Source"),
|
|
28
|
+
"sqs_sns_s3_source": ("modmex_lambda.stream.sources.s3", "sqs_sns_s3_source"),
|
|
29
|
+
"SqsEventBridgeS3Source": ("modmex_lambda.stream.sources.s3", "SqsEventBridgeS3Source"),
|
|
30
|
+
"sqs_eventbridge_s3_source": (
|
|
31
|
+
"modmex_lambda.stream.sources.s3", "sqs_eventbridge_s3_source"
|
|
32
|
+
),
|
|
20
33
|
"SnsSource": ("modmex_lambda.stream.sources.sns", "SnsSource"),
|
|
21
34
|
"sns_source": ("modmex_lambda.stream.sources.sns", "sns_source"),
|
|
22
35
|
"SqsSource": ("modmex_lambda.stream.sources.sqs", "SqsSource"),
|
|
@@ -33,6 +46,10 @@ __all__ = [
|
|
|
33
46
|
"dynamodb_source",
|
|
34
47
|
"kinesis_source",
|
|
35
48
|
"s3_source",
|
|
49
|
+
"SqsSnsS3Source",
|
|
50
|
+
"sqs_sns_s3_source",
|
|
51
|
+
"SqsEventBridgeS3Source",
|
|
52
|
+
"sqs_eventbridge_s3_source",
|
|
36
53
|
"sns_source",
|
|
37
54
|
"sqs_source",
|
|
38
55
|
]
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
from typing import Callable, Optional
|
|
2
2
|
|
|
3
3
|
from modmex_lambda.dependencies import DependencyResolver
|
|
4
|
-
from modmex_lambda.stream.events.s3 import
|
|
4
|
+
from modmex_lambda.stream.events.s3 import (
|
|
5
|
+
from_s3,
|
|
6
|
+
from_sqs_eventbridge_s3,
|
|
7
|
+
from_sqs_sns_s3,
|
|
8
|
+
)
|
|
5
9
|
from modmex_lambda.stream.irules_registry import IRulesRegistry
|
|
6
10
|
from modmex_lambda.stream.sources.base import SourceHandler
|
|
7
11
|
|
|
@@ -32,6 +36,18 @@ class S3Source(SourceHandler):
|
|
|
32
36
|
)
|
|
33
37
|
|
|
34
38
|
|
|
39
|
+
class SqsSnsS3Source(S3Source):
|
|
40
|
+
def __init__(self, registry: IRulesRegistry, **kwargs) -> None:
|
|
41
|
+
super().__init__(registry, **kwargs)
|
|
42
|
+
self.parser = from_sqs_sns_s3
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class SqsEventBridgeS3Source(S3Source):
|
|
46
|
+
def __init__(self, registry: IRulesRegistry, **kwargs) -> None:
|
|
47
|
+
super().__init__(registry, **kwargs)
|
|
48
|
+
self.parser = from_sqs_eventbridge_s3
|
|
49
|
+
|
|
50
|
+
|
|
35
51
|
def s3_source(
|
|
36
52
|
registry: IRulesRegistry,
|
|
37
53
|
*,
|
|
@@ -53,3 +69,11 @@ def s3_source(
|
|
|
53
69
|
logger=logger,
|
|
54
70
|
dependency_resolver=dependency_resolver,
|
|
55
71
|
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def sqs_sns_s3_source(registry: IRulesRegistry, **kwargs):
|
|
75
|
+
return SqsSnsS3Source(registry, **kwargs)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def sqs_eventbridge_s3_source(registry: IRulesRegistry, **kwargs):
|
|
79
|
+
return SqsEventBridgeS3Source(registry, **kwargs)
|
|
@@ -77,7 +77,7 @@ modmex_lambda/stream/schedulers.py,sha256=A_wFD1e7CGFGO6--QHln8Eho08fdH5W-OVWfho
|
|
|
77
77
|
modmex_lambda/stream/events/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
78
78
|
modmex_lambda/stream/events/dynamodb.py,sha256=T1g9j4QpWC1gBYaFm11YdcQ9Ma0konHIVF48mOAD_PE,4895
|
|
79
79
|
modmex_lambda/stream/events/kinesis.py,sha256=BDSO7mwAV6utXROBxbyv36fbNspwOkHpvnysmFNDz0Q,1154
|
|
80
|
-
modmex_lambda/stream/events/s3.py,sha256=
|
|
80
|
+
modmex_lambda/stream/events/s3.py,sha256=618h_xKpkA_6OfXzjVii3QwJWJUjqMn_FeMG2hoqit0,4417
|
|
81
81
|
modmex_lambda/stream/events/sns.py,sha256=MHtLaKKYvmhPOQjqCMgxNFXpBuYfW4wGaC1BxwWTaIw,2073
|
|
82
82
|
modmex_lambda/stream/events/sqs.py,sha256=AnaIdTJA5emzPKMXG-8OQTmhmB6PrGu4zXDepcHqxNE,1470
|
|
83
83
|
modmex_lambda/stream/filters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -110,11 +110,11 @@ modmex_lambda/stream/operators/s3.py,sha256=hGptuorkRRFHiGtBdtYvJofmDwfQ9St0jyR1
|
|
|
110
110
|
modmex_lambda/stream/operators/scheduler.py,sha256=KlVXQswqmD2XrpYNyT-nKV3lfu05TQv3Edo40MZAX1k,1109
|
|
111
111
|
modmex_lambda/stream/operators/sns.py,sha256=9LoaQp_JsmP0xPzLF9XMcd5qbK7lg7daViaU4vkozG0,1740
|
|
112
112
|
modmex_lambda/stream/operators/sqs.py,sha256=4y2wrfVCl-TygrieFpfTEPjGUYcFV9gpYUT77cfyYm4,2464
|
|
113
|
-
modmex_lambda/stream/sources/__init__.py,sha256=
|
|
113
|
+
modmex_lambda/stream/sources/__init__.py,sha256=mWJYsGBy-eZkR-B31JPicH-q00CdO-YUIvG5giACuKE,2469
|
|
114
114
|
modmex_lambda/stream/sources/base.py,sha256=S1yrVTGl8KMIaUVG3iq0yLY_u8K_QBN8CY0JxzzEgng,2086
|
|
115
115
|
modmex_lambda/stream/sources/dynamodb.py,sha256=JRMQlneXbli9CTBfqidQtXIPcOxMEofsQRwKR38P0Ns,2545
|
|
116
116
|
modmex_lambda/stream/sources/kinesis.py,sha256=pWDXjfcjTspDk36sLrZQ28iPeB28zRrLpzxO3wiOXRE,1694
|
|
117
|
-
modmex_lambda/stream/sources/s3.py,sha256=
|
|
117
|
+
modmex_lambda/stream/sources/s3.py,sha256=kBLRP6KSDW2E7xA6z07c_FAnc85BnwT6FJGMqcAwqVU,2339
|
|
118
118
|
modmex_lambda/stream/sources/sns.py,sha256=LqWVSpuCFunUEKnxC3IIkgyp7AHuog8GPqYJPF9kZb0,1670
|
|
119
119
|
modmex_lambda/stream/sources/sqs.py,sha256=rfweOWQg-55jkmVZpws4bvyCWnKgVHdB8FW8Qml5src,1670
|
|
120
120
|
modmex_lambda/stream/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -144,7 +144,7 @@ modmex_lambda/stream/utils/time.py,sha256=dnsL2xeWpa1FXm2p8y9rWXtICMBlCBkgLaJ_en
|
|
|
144
144
|
modmex_lambda/stream/utils/uow.py,sha256=8WSPVE7AiHInkFDfBEeKOvdirY6GAgBoS7qkF1f1JVM,3207
|
|
145
145
|
modmex_lambda/stream/utils/data_classes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
146
146
|
modmex_lambda/stream/utils/data_classes/dynamodb.py,sha256=h_7MtUyRkvvV1r2QECrRDu7RR4Hh94SwSjc_8K2_XGM,363
|
|
147
|
-
modmex_lambda-0.6.
|
|
148
|
-
modmex_lambda-0.6.
|
|
149
|
-
modmex_lambda-0.6.
|
|
150
|
-
modmex_lambda-0.6.
|
|
147
|
+
modmex_lambda-0.6.3.dist-info/METADATA,sha256=qTzeKW4ces7f_H2gc4FqGohOILhO3Qm8G0ZhzSfpuxM,35443
|
|
148
|
+
modmex_lambda-0.6.3.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
149
|
+
modmex_lambda-0.6.3.dist-info/licenses/LICENSE,sha256=_C2TDTOsYeJvE4vn9VB51laKvleBKbdNnn96wJVtXhQ,1063
|
|
150
|
+
modmex_lambda-0.6.3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|