kloship-utils 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kloship-utils might be problematic. Click here for more details.
- kloship_utils/__init__.py +26 -0
- kloship_utils/_version.py +24 -0
- kloship_utils/aws/__init__.py +36 -0
- kloship_utils/aws/dynamodb.py +376 -0
- kloship_utils/aws/s3.py +216 -0
- kloship_utils/cella.py +255 -0
- kloship_utils/encoding.py +41 -0
- kloship_utils/exceptions.py +45 -0
- kloship_utils/handle_units.py +136 -0
- kloship_utils/helpers.py +51 -0
- kloship_utils/py.typed +0 -0
- kloship_utils/templates.py +73 -0
- kloship_utils/text.py +57 -0
- kloship_utils/transfers/__init__.py +11 -0
- kloship_utils/transfers/config.py +177 -0
- kloship_utils/transfers/ftp.py +97 -0
- kloship_utils/transfers/sftp.py +148 -0
- kloship_utils/units.py +255 -0
- kloship_utils/xml.py +67 -0
- kloship_utils-0.1.0.dist-info/METADATA +252 -0
- kloship_utils-0.1.0.dist-info/RECORD +23 -0
- kloship_utils-0.1.0.dist-info/WHEEL +4 -0
- kloship_utils-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Shared utilities for KloShip carrier integrations.
|
|
2
|
+
|
|
3
|
+
The package is split into small, independent modules so that a carrier Lambda only imports
|
|
4
|
+
(and therefore only needs the dependencies of) what it uses:
|
|
5
|
+
|
|
6
|
+
* :mod:`kloship_utils.encoding` - base64/gzip/JSON payload helpers used between Step Functions and Lambdas
|
|
7
|
+
* :mod:`kloship_utils.helpers` - tiny generic helpers (``to_list``, ``chunks``...)
|
|
8
|
+
* :mod:`kloship_utils.units` - unit (length, volume, weight, liquid) and date conversions
|
|
9
|
+
* :mod:`kloship_utils.text` - diacritics removal and special characters replacement
|
|
10
|
+
* :mod:`kloship_utils.templates` - Jinja2 rendering of ZPL / XML / EDI templates
|
|
11
|
+
* :mod:`kloship_utils.xml` - dict to XML conversion
|
|
12
|
+
* :mod:`kloship_utils.handle_units` - HandleUnit helpers (group ids, grouping, statuses)
|
|
13
|
+
* :mod:`kloship_utils.aws.s3` - archiving of labels, documents and EDI files on S3
|
|
14
|
+
* :mod:`kloship_utils.aws.dynamodb` - HandleUnit lookups, counters and conversions in DynamoDB
|
|
15
|
+
* :mod:`kloship_utils.cella` - CELLA (KloShip front) GraphQL client: rules execution, PDF to ZPL
|
|
16
|
+
* :mod:`kloship_utils.transfers` - FTP and SFTP uploads (optionally through an SSH jump host)
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
__version__ = version("kloship-utils")
|
|
23
|
+
except PackageNotFoundError: # pragma: no cover - only when running from a raw source tree
|
|
24
|
+
__version__ = "0.0.0"
|
|
25
|
+
|
|
26
|
+
__all__ = ["__version__"]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.1.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""AWS helpers (S3, DynamoDB).
|
|
2
|
+
|
|
3
|
+
``boto3`` is not a mandatory dependency of kloship-utils because the AWS Lambda Python runtime already provides
|
|
4
|
+
it. Outside Lambda, install the ``aws`` extra: ``pip install "kloship-utils[aws]"``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from kloship_utils.exceptions import ConfigurationError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def require_env(name: str, value: str | None = None) -> str:
|
|
14
|
+
"""Return ``value`` when given, else the ``name`` environment variable.
|
|
15
|
+
|
|
16
|
+
Raises:
|
|
17
|
+
ConfigurationError: neither ``value`` nor the environment variable is set.
|
|
18
|
+
"""
|
|
19
|
+
if value:
|
|
20
|
+
return value
|
|
21
|
+
try:
|
|
22
|
+
return os.environ[name]
|
|
23
|
+
except KeyError:
|
|
24
|
+
raise ConfigurationError(f"Environment variable {name} is not set") from None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def boto3_module() -> Any:
|
|
28
|
+
"""Import ``boto3`` lazily, with a helpful message when it is missing."""
|
|
29
|
+
try:
|
|
30
|
+
import boto3 # noqa: PLC0415 - lazy import: boto3 is optional (provided by the Lambda runtime)
|
|
31
|
+
except ImportError as error: # pragma: no cover - boto3 is installed in the test environment
|
|
32
|
+
raise ImportError(
|
|
33
|
+
"boto3 is required by kloship_utils.aws: it is provided by the AWS Lambda runtime, "
|
|
34
|
+
'install "kloship-utils[aws]" elsewhere.'
|
|
35
|
+
) from error
|
|
36
|
+
return boto3
|
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
"""DynamoDB helpers for the KloShip tables.
|
|
2
|
+
|
|
3
|
+
Table names are read from the standard environment variables of the carrier Lambdas (they can always be given
|
|
4
|
+
explicitly instead):
|
|
5
|
+
|
|
6
|
+
* ``HANDLE_UNIT_TABLE_NAME`` - HandleUnits (indexes ``GlobalGroupId-id-index`` and ``GlobalGroupId2-id-index``)
|
|
7
|
+
* ``CARRIER_ACCOUNT_CONFIG_TABLE_NAME`` - CarrierAccountConfigs (key ``@senderCodeTDM`` / ``@serviceTagTDM``)
|
|
8
|
+
* ``PARAMETERS_TABLE_NAME`` - global parameters and counters (key ``id``, counter attribute ``valueInt``)
|
|
9
|
+
* ``CONVERSIONS_TABLE_NAME`` - conversions (``conversion_type``, ``packaging_in`` -> ``packaging_out``...)
|
|
10
|
+
|
|
11
|
+
All PartiQL statements are parameterized (``?`` placeholders): values are never interpolated in the statement.
|
|
12
|
+
Like the legacy carrier code, the helpers use the DynamoDB *resource* client, so items are returned as plain Python
|
|
13
|
+
dicts (numbers as :class:`~decimal.Decimal`) and parameters are given as plain Python values.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import logging
|
|
17
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from kloship_utils.aws import boto3_module, require_env
|
|
21
|
+
from kloship_utils.exceptions import ConversionNotFound, HandleUnitNotFound
|
|
22
|
+
from kloship_utils.handle_units import Status, global_group_id, global_group_id2, next_index_in_order
|
|
23
|
+
from kloship_utils.helpers import chunks
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
HANDLE_UNIT_TABLE_ENV = "HANDLE_UNIT_TABLE_NAME"
|
|
28
|
+
CARRIER_ACCOUNT_CONFIG_TABLE_ENV = "CARRIER_ACCOUNT_CONFIG_TABLE_NAME"
|
|
29
|
+
PARAMETERS_TABLE_ENV = "PARAMETERS_TABLE_NAME"
|
|
30
|
+
CONVERSIONS_TABLE_ENV = "CONVERSIONS_TABLE_NAME"
|
|
31
|
+
|
|
32
|
+
GLOBAL_GROUP_ID_INDEX = "GlobalGroupId-id-index"
|
|
33
|
+
GLOBAL_GROUP_ID2_INDEX = "GlobalGroupId2-id-index"
|
|
34
|
+
|
|
35
|
+
#: ``conversion_type`` of the packaging conversions (``packaging_in`` -> ``packaging_out``).
|
|
36
|
+
PACKAGING_CONVERSION_TYPE = 50000
|
|
37
|
+
#: Maximum number of statements accepted by ``BatchExecuteStatement``.
|
|
38
|
+
BATCH_SIZE = 25
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def get_resource() -> Any:
|
|
42
|
+
"""Return a new boto3 DynamoDB resource."""
|
|
43
|
+
return boto3_module().resource("dynamodb")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def get_client() -> Any:
|
|
47
|
+
"""Return the client of a new DynamoDB resource (high-level (de)serialization of the attribute values)."""
|
|
48
|
+
return get_resource().meta.client
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _identifier(name: str) -> str:
|
|
52
|
+
"""Quote a PartiQL identifier (table, index or attribute name)."""
|
|
53
|
+
return '"' + name.replace('"', '""') + '"'
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def execute_statement_all(
|
|
57
|
+
statement: str,
|
|
58
|
+
parameters: Sequence[Any] | None = None,
|
|
59
|
+
*,
|
|
60
|
+
consistent_read: bool = False,
|
|
61
|
+
client: Any = None,
|
|
62
|
+
) -> list[dict[str, Any]]:
|
|
63
|
+
"""Execute a PartiQL statement and return all the items, following ``NextToken`` pagination.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
statement: PartiQL statement with ``?`` placeholders.
|
|
67
|
+
parameters: values of the placeholders (plain Python values).
|
|
68
|
+
consistent_read: strongly consistent read (not available on indexes).
|
|
69
|
+
client: optional DynamoDB client (see :func:`get_client`).
|
|
70
|
+
"""
|
|
71
|
+
client = client or get_client()
|
|
72
|
+
params: dict[str, Any] = {"Statement": statement}
|
|
73
|
+
if parameters:
|
|
74
|
+
params["Parameters"] = list(parameters)
|
|
75
|
+
if consistent_read:
|
|
76
|
+
params["ConsistentRead"] = True
|
|
77
|
+
|
|
78
|
+
response = client.execute_statement(**params)
|
|
79
|
+
items: list[dict[str, Any]] = list(response.get("Items", []))
|
|
80
|
+
while "NextToken" in response:
|
|
81
|
+
response = client.execute_statement(**params, NextToken=response["NextToken"])
|
|
82
|
+
items.extend(response.get("Items", []))
|
|
83
|
+
return items
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def batch_execute(statements: Sequence[Mapping[str, Any]], *, client: Any = None) -> list[dict[str, Any]]:
|
|
87
|
+
"""Run ``BatchExecuteStatement`` by batches of 25 statements (DynamoDB limit).
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
statements: ``[{"Statement": "...", "Parameters": [...]}, ...]``.
|
|
91
|
+
client: optional DynamoDB client.
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
The concatenated ``Responses``.
|
|
95
|
+
"""
|
|
96
|
+
client = client or get_client()
|
|
97
|
+
responses: list[dict[str, Any]] = []
|
|
98
|
+
for batch in chunks(list(statements), BATCH_SIZE):
|
|
99
|
+
response = client.batch_execute_statement(Statements=list(batch))
|
|
100
|
+
responses.extend(response.get("Responses", []))
|
|
101
|
+
return responses
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# --- HandleUnits ---------------------------------------------------------------------------------------------------
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def get_handle_units_by_global_group_id(
|
|
108
|
+
account_id: str,
|
|
109
|
+
config: Mapping[str, Any],
|
|
110
|
+
*,
|
|
111
|
+
statuses: Iterable[str] = (Status.PRE_SHIPPED, Status.SHIPPED),
|
|
112
|
+
table_name: str | None = None,
|
|
113
|
+
raise_if_empty: bool = True,
|
|
114
|
+
client: Any = None,
|
|
115
|
+
) -> list[dict[str, Any]]:
|
|
116
|
+
"""Return the HandleUnits of a shipment (``GlobalGroupId``).
|
|
117
|
+
|
|
118
|
+
See :func:`kloship_utils.handle_units.global_group_id`.
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
account_id: KloShip account id.
|
|
122
|
+
config: the CarrierAccountConfig (``@senderCodeTDM``, ``@serviceTagTDM``, ``shippingCarrierNumber``).
|
|
123
|
+
statuses: statuses to keep (default: ``PRE-SHIPPED`` and ``SHIPPED``); empty to keep all.
|
|
124
|
+
table_name: HandleUnit table (defaults to ``HANDLE_UNIT_TABLE_NAME``).
|
|
125
|
+
raise_if_empty: raise :class:`HandleUnitNotFound` when nothing is found.
|
|
126
|
+
client: optional DynamoDB client.
|
|
127
|
+
"""
|
|
128
|
+
table = require_env(HANDLE_UNIT_TABLE_ENV, table_name)
|
|
129
|
+
group_id = global_group_id(account_id, dict(config))
|
|
130
|
+
status_list = [str(status) for status in statuses]
|
|
131
|
+
logger.info("Search HandleUnits with GlobalGroupId %s", group_id)
|
|
132
|
+
|
|
133
|
+
statement = (
|
|
134
|
+
f"SELECT * FROM {_identifier(table)}.{_identifier(GLOBAL_GROUP_ID_INDEX)} "
|
|
135
|
+
"WHERE GlobalGroupId = ? AND AccountId = ?"
|
|
136
|
+
)
|
|
137
|
+
if status_list:
|
|
138
|
+
statement += " AND (" + " OR ".join("status = ?" for _ in status_list) + ")"
|
|
139
|
+
items = execute_statement_all(statement, [group_id, account_id, *status_list], client=client)
|
|
140
|
+
|
|
141
|
+
if not items and raise_if_empty:
|
|
142
|
+
raise HandleUnitNotFound(
|
|
143
|
+
f"No handle unit for shipment number ({config.get('shippingCarrierNumber', 0)}) in table {table}"
|
|
144
|
+
)
|
|
145
|
+
return items
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def get_handle_unit_from_shipment_number(
|
|
149
|
+
config: Mapping[str, Any], table_name: str, account_id: str
|
|
150
|
+
) -> list[dict[str, Any]]:
|
|
151
|
+
"""Legacy-compatible alias of :func:`get_handle_units_by_global_group_id` (raises when nothing is found)."""
|
|
152
|
+
return get_handle_units_by_global_group_id(account_id, config, table_name=table_name)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def get_handle_units_by_global_group_id2(
|
|
156
|
+
global_group_id2_value: str,
|
|
157
|
+
*,
|
|
158
|
+
projection: Sequence[str] | None = ("@id", "@indexInOrder"),
|
|
159
|
+
exclude_status: str | None = Status.CANCELLED,
|
|
160
|
+
table_name: str | None = None,
|
|
161
|
+
raise_if_empty: bool = False,
|
|
162
|
+
client: Any = None,
|
|
163
|
+
) -> list[dict[str, Any]]:
|
|
164
|
+
"""Return the HandleUnits of a delivery (``GlobalGroupId2``).
|
|
165
|
+
|
|
166
|
+
See :func:`kloship_utils.handle_units.global_group_id2`.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
global_group_id2_value: the ``GlobalGroupId2``.
|
|
170
|
+
projection: attributes to return (``None`` for all).
|
|
171
|
+
exclude_status: status to exclude (default ``CANCELLED``, ``None`` to keep all).
|
|
172
|
+
table_name: HandleUnit table (defaults to ``HANDLE_UNIT_TABLE_NAME``).
|
|
173
|
+
raise_if_empty: raise :class:`HandleUnitNotFound` when nothing is found.
|
|
174
|
+
client: optional DynamoDB client.
|
|
175
|
+
"""
|
|
176
|
+
table = require_env(HANDLE_UNIT_TABLE_ENV, table_name)
|
|
177
|
+
columns = ", ".join(_identifier(column) for column in projection) if projection else "*"
|
|
178
|
+
statement = (
|
|
179
|
+
f"SELECT {columns} FROM {_identifier(table)}.{_identifier(GLOBAL_GROUP_ID2_INDEX)} WHERE GlobalGroupId2 = ?"
|
|
180
|
+
)
|
|
181
|
+
parameters: list[Any] = [global_group_id2_value]
|
|
182
|
+
if exclude_status is not None:
|
|
183
|
+
statement += " AND status != ?"
|
|
184
|
+
parameters.append(str(exclude_status))
|
|
185
|
+
logger.info("Search HandleUnits with GlobalGroupId2 %s", global_group_id2_value)
|
|
186
|
+
items = execute_statement_all(statement, parameters, client=client)
|
|
187
|
+
|
|
188
|
+
if not items and raise_if_empty:
|
|
189
|
+
raise HandleUnitNotFound(f"No handle unit for GlobalGroupId2 ({global_group_id2_value}) in table {table}")
|
|
190
|
+
return items
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def get_handle_unit_from_GlobalGroupId2( # noqa: N802 - legacy name
|
|
194
|
+
config: Mapping[str, Any],
|
|
195
|
+
table_name: str,
|
|
196
|
+
GlobalGroupId2: str, # noqa: N803 - legacy name
|
|
197
|
+
) -> list[dict[str, Any]]:
|
|
198
|
+
"""Legacy-compatible alias of :func:`get_handle_units_by_global_group_id2` (``[]`` when nothing is found)."""
|
|
199
|
+
return get_handle_units_by_global_group_id2(GlobalGroupId2, table_name=table_name)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def search_parcel_from_same_delivery(
|
|
203
|
+
handle_unit: Mapping[str, Any],
|
|
204
|
+
config: Mapping[str, Any],
|
|
205
|
+
account_id: str,
|
|
206
|
+
hu_existing_position: Iterable[int] = (),
|
|
207
|
+
*,
|
|
208
|
+
table_name: str | None = None,
|
|
209
|
+
client: Any = None,
|
|
210
|
+
) -> dict[str, Any]:
|
|
211
|
+
"""Compute ``{"nb_parcel", "@indexInOrder"}`` of a HandleUnit inside its delivery.
|
|
212
|
+
|
|
213
|
+
Looks up the other HandleUnits of the delivery in DynamoDB then applies
|
|
214
|
+
:func:`kloship_utils.handle_units.next_index_in_order`. ``config`` is unused and only kept for compatibility with
|
|
215
|
+
the legacy signature.
|
|
216
|
+
"""
|
|
217
|
+
existing = get_handle_units_by_global_group_id2(
|
|
218
|
+
global_group_id2(account_id, dict(handle_unit)), table_name=table_name, client=client
|
|
219
|
+
)
|
|
220
|
+
return next_index_in_order(handle_unit["@id"], existing, hu_existing_position)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
# --- Counters and parameters ---------------------------------------------------------------------------------------
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def increment_counter(
|
|
227
|
+
config: Mapping[str, Any] | None = None,
|
|
228
|
+
*,
|
|
229
|
+
parameter_name: str | None = None,
|
|
230
|
+
local_attribute: str = "HandleUnitNumber",
|
|
231
|
+
step: int = 1,
|
|
232
|
+
sender_code: str | None = None,
|
|
233
|
+
service_tag: str | None = None,
|
|
234
|
+
parameters_table: str | None = None,
|
|
235
|
+
config_table: str | None = None,
|
|
236
|
+
resource: Any = None,
|
|
237
|
+
) -> int:
|
|
238
|
+
"""Atomically increment a counter and return its new value.
|
|
239
|
+
|
|
240
|
+
* with ``parameter_name``: *global* counter, attribute ``valueInt`` of the item ``id = parameter_name`` of the
|
|
241
|
+
Parameters table (shared between several CarrierAccountConfigs);
|
|
242
|
+
* otherwise: *local* counter, attribute ``local_attribute`` of the CarrierAccountConfig item
|
|
243
|
+
(``@senderCodeTDM`` / ``@serviceTagTDM`` read from ``config`` or given explicitly).
|
|
244
|
+
|
|
245
|
+
The counter attribute must already exist (DynamoDB rejects the update otherwise), so a misconfigured counter
|
|
246
|
+
name fails instead of silently restarting a numbering.
|
|
247
|
+
|
|
248
|
+
Typical use::
|
|
249
|
+
|
|
250
|
+
number = increment_counter(config, parameter_name=config.get("ParameterNameHUCounter"))
|
|
251
|
+
"""
|
|
252
|
+
resource = resource or get_resource()
|
|
253
|
+
if parameter_name:
|
|
254
|
+
table = resource.Table(require_env(PARAMETERS_TABLE_ENV, parameters_table))
|
|
255
|
+
key: dict[str, Any] = {"id": parameter_name}
|
|
256
|
+
attribute = "valueInt"
|
|
257
|
+
else:
|
|
258
|
+
config = config or {}
|
|
259
|
+
table = resource.Table(require_env(CARRIER_ACCOUNT_CONFIG_TABLE_ENV, config_table))
|
|
260
|
+
key = {
|
|
261
|
+
"@senderCodeTDM": sender_code or config["@senderCodeTDM"],
|
|
262
|
+
"@serviceTagTDM": service_tag or config["@serviceTagTDM"],
|
|
263
|
+
}
|
|
264
|
+
attribute = local_attribute
|
|
265
|
+
|
|
266
|
+
response = table.update_item(
|
|
267
|
+
Key=key,
|
|
268
|
+
UpdateExpression="SET #counter = #counter + :step",
|
|
269
|
+
ExpressionAttributeNames={"#counter": attribute},
|
|
270
|
+
ExpressionAttributeValues={":step": step},
|
|
271
|
+
ReturnValues="UPDATED_NEW",
|
|
272
|
+
)
|
|
273
|
+
return int(response["Attributes"][attribute])
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def get_parameter(parameter_id: str, *, table_name: str | None = None, resource: Any = None) -> dict[str, Any] | None:
|
|
277
|
+
"""Return the item ``id = parameter_id`` of the Parameters table, or ``None``."""
|
|
278
|
+
resource = resource or get_resource()
|
|
279
|
+
table = resource.Table(require_env(PARAMETERS_TABLE_ENV, table_name))
|
|
280
|
+
item: dict[str, Any] | None = table.get_item(Key={"id": parameter_id}).get("Item")
|
|
281
|
+
return item
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def get_carrier_account_config(
|
|
285
|
+
sender_code: str, service_tag: str, *, table_name: str | None = None, resource: Any = None
|
|
286
|
+
) -> dict[str, Any] | None:
|
|
287
|
+
"""Return the CarrierAccountConfig ``(@senderCodeTDM, @serviceTagTDM)``, or ``None``."""
|
|
288
|
+
resource = resource or get_resource()
|
|
289
|
+
table = resource.Table(require_env(CARRIER_ACCOUNT_CONFIG_TABLE_ENV, table_name))
|
|
290
|
+
item: dict[str, Any] | None = table.get_item(
|
|
291
|
+
Key={"@senderCodeTDM": sender_code, "@serviceTagTDM": service_tag}
|
|
292
|
+
).get("Item")
|
|
293
|
+
return item
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
# --- Conversions ---------------------------------------------------------------------------------------------------
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def get_conversion(
|
|
300
|
+
config: Mapping[str, Any],
|
|
301
|
+
*,
|
|
302
|
+
conversion_type: int,
|
|
303
|
+
match: Mapping[str, Any],
|
|
304
|
+
fields: Sequence[str] = (),
|
|
305
|
+
table_name: str | None = None,
|
|
306
|
+
client: Any = None,
|
|
307
|
+
) -> dict[str, Any] | None:
|
|
308
|
+
"""Look up a conversion of the CarrierAccountConfig in the Conversions table.
|
|
309
|
+
|
|
310
|
+
Args:
|
|
311
|
+
config: the CarrierAccountConfig (``@senderCodeTDM`` and ``@serviceTagTDM`` are read).
|
|
312
|
+
conversion_type: type of conversion (e.g. :data:`PACKAGING_CONVERSION_TYPE`).
|
|
313
|
+
match: additional equality conditions, e.g. ``{"packaging_in": "PAL"}``.
|
|
314
|
+
fields: attributes to return (all when empty).
|
|
315
|
+
table_name: Conversions table (defaults to ``CONVERSIONS_TABLE_NAME``).
|
|
316
|
+
client: optional DynamoDB client.
|
|
317
|
+
|
|
318
|
+
Returns:
|
|
319
|
+
The first matching item, or ``None``.
|
|
320
|
+
"""
|
|
321
|
+
table = require_env(CONVERSIONS_TABLE_ENV, table_name)
|
|
322
|
+
columns = ", ".join(_identifier(field) for field in fields) if fields else "*"
|
|
323
|
+
conditions = ['"@serviceTagTDM" = ?', '"@senderCodeTDM" = ?', "conversion_type = ?"]
|
|
324
|
+
parameters: list[Any] = [config["@serviceTagTDM"], config["@senderCodeTDM"], conversion_type]
|
|
325
|
+
for name, value in match.items():
|
|
326
|
+
conditions.append(f"{_identifier(name)} = ?")
|
|
327
|
+
parameters.append(value)
|
|
328
|
+
|
|
329
|
+
statement = f"SELECT {columns} FROM {_identifier(table)} WHERE {' AND '.join(conditions)}"
|
|
330
|
+
items = execute_statement_all(statement, parameters, client=client)
|
|
331
|
+
return items[0] if items else None
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def get_packaging_conversion(
|
|
335
|
+
packaging_in: str,
|
|
336
|
+
config: Mapping[str, Any],
|
|
337
|
+
*,
|
|
338
|
+
fields: Sequence[str] = ("packaging_out",),
|
|
339
|
+
default: str | None = None,
|
|
340
|
+
raise_if_not_found: bool = True,
|
|
341
|
+
table_name: str | None = None,
|
|
342
|
+
client: Any = None,
|
|
343
|
+
) -> dict[str, Any]:
|
|
344
|
+
"""Convert a KloShip packaging code to the carrier packaging code (``conversion_type = 50000``).
|
|
345
|
+
|
|
346
|
+
Args:
|
|
347
|
+
packaging_in: KloShip packaging code (``handle_unit["PACKAGING"]``).
|
|
348
|
+
config: the CarrierAccountConfig.
|
|
349
|
+
fields: attributes to return (``packaging_out`` and any carrier specific column, e.g. ``DHL_loadingmetre``).
|
|
350
|
+
default: fallback ``packaging_out`` when no conversion exists (e.g. ``DefaultPackaging`` of the config).
|
|
351
|
+
raise_if_not_found: raise :class:`ConversionNotFound` when no conversion and no default exist; otherwise
|
|
352
|
+
every field is returned as ``""``.
|
|
353
|
+
table_name: Conversions table (defaults to ``CONVERSIONS_TABLE_NAME``).
|
|
354
|
+
client: optional DynamoDB client.
|
|
355
|
+
|
|
356
|
+
Returns:
|
|
357
|
+
``{"packaging_out": ..., <other fields>...}``.
|
|
358
|
+
"""
|
|
359
|
+
item = get_conversion(
|
|
360
|
+
config,
|
|
361
|
+
conversion_type=PACKAGING_CONVERSION_TYPE,
|
|
362
|
+
match={"packaging_in": packaging_in},
|
|
363
|
+
fields=fields,
|
|
364
|
+
table_name=table_name,
|
|
365
|
+
client=client,
|
|
366
|
+
)
|
|
367
|
+
if item is not None:
|
|
368
|
+
return item
|
|
369
|
+
if default:
|
|
370
|
+
return {"packaging_out": default}
|
|
371
|
+
if raise_if_not_found:
|
|
372
|
+
raise ConversionNotFound(
|
|
373
|
+
f"Packaging conversion not found for {packaging_in!r} "
|
|
374
|
+
f"({config.get('@senderCodeTDM')} / {config.get('@serviceTagTDM')})"
|
|
375
|
+
)
|
|
376
|
+
return dict.fromkeys(fields, "")
|
kloship_utils/aws/s3.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Archiving of labels, documents and EDI files on S3.
|
|
2
|
+
|
|
3
|
+
Every KloShip carrier stores its files with the same key layout::
|
|
4
|
+
|
|
5
|
+
{account_id}/{@senderCodeTDM}/{@serviceTagTDM}/{YYYY}/{MM}/{DD}/{file_name}
|
|
6
|
+
|
|
7
|
+
in the ``STANDARD_IA`` storage class (labels in ``LABELS_BUCKET``, EDI / shipment files in ``SHIPMENT_BUCKET``).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
from datetime import datetime
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from kloship_utils.aws import boto3_module, require_env
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
#: Storage class used for every archived file.
|
|
20
|
+
DEFAULT_STORAGE_CLASS = "STANDARD_IA"
|
|
21
|
+
#: Environment variable holding the labels bucket name.
|
|
22
|
+
LABELS_BUCKET_ENV = "LABELS_BUCKET"
|
|
23
|
+
#: Environment variable holding the shipment (EDI) bucket name.
|
|
24
|
+
SHIPMENT_BUCKET_ENV = "SHIPMENT_BUCKET"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_client() -> Any:
|
|
28
|
+
"""Return a new boto3 S3 client."""
|
|
29
|
+
return boto3_module().client("s3")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def build_key(account_id: str, config: Mapping[str, Any], file_name: str, *, now: datetime | None = None) -> str:
|
|
33
|
+
"""Build the S3 key ``{account}/{senderCodeTDM}/{serviceTagTDM}/{YYYY}/{MM}/{DD}/{file_name}``.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
account_id: KloShip account id.
|
|
37
|
+
config: the CarrierAccountConfig (``@senderCodeTDM`` and ``@serviceTagTDM`` are read).
|
|
38
|
+
file_name: name of the file.
|
|
39
|
+
now: date used for the folders (defaults to now).
|
|
40
|
+
"""
|
|
41
|
+
now = now or datetime.now()
|
|
42
|
+
return f"{account_id}/{config['@senderCodeTDM']}/{config['@serviceTagTDM']}/{now:%Y}/{now:%m}/{now:%d}/{file_name}"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def put_file(
|
|
46
|
+
bucket: str,
|
|
47
|
+
key: str,
|
|
48
|
+
body: str | bytes,
|
|
49
|
+
*,
|
|
50
|
+
content_type: str,
|
|
51
|
+
metadata: Mapping[str, Any] | None = None,
|
|
52
|
+
encoding: str = "utf-8",
|
|
53
|
+
content_encoding: str | None = None,
|
|
54
|
+
storage_class: str = DEFAULT_STORAGE_CLASS,
|
|
55
|
+
client: Any = None,
|
|
56
|
+
) -> str:
|
|
57
|
+
"""Upload a file to S3.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
bucket: bucket name.
|
|
61
|
+
key: object key (see :func:`build_key`).
|
|
62
|
+
body: file content; ``str`` bodies are encoded with ``encoding``.
|
|
63
|
+
content_type: MIME type (``"x-application/zpl"``, ``"application/pdf"``, ``"application/EDI-EDIFACT"``...).
|
|
64
|
+
metadata: user metadata (values are converted to ``str``).
|
|
65
|
+
encoding: encoding used for ``str`` bodies (e.g. ``"ISO-8859-1"`` for EDI files).
|
|
66
|
+
content_encoding: optional ``ContentEncoding`` header (the legacy carriers set it to the file charset).
|
|
67
|
+
storage_class: S3 storage class.
|
|
68
|
+
client: optional boto3 S3 client (a new one is created otherwise).
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
The object key.
|
|
72
|
+
"""
|
|
73
|
+
data = body.encode(encoding) if isinstance(body, str) else body
|
|
74
|
+
params: dict[str, Any] = {
|
|
75
|
+
"Bucket": bucket,
|
|
76
|
+
"Key": key,
|
|
77
|
+
"Body": data,
|
|
78
|
+
"StorageClass": storage_class,
|
|
79
|
+
"ContentType": content_type,
|
|
80
|
+
"Metadata": {str(k): str(v) for k, v in (metadata or {}).items()},
|
|
81
|
+
}
|
|
82
|
+
if content_encoding:
|
|
83
|
+
params["ContentEncoding"] = content_encoding
|
|
84
|
+
(client or get_client()).put_object(**params)
|
|
85
|
+
logger.info("File written at s3://%s/%s", bucket, key)
|
|
86
|
+
return key
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def get_file(bucket: str, key: str, *, client: Any = None) -> bytes:
|
|
90
|
+
"""Download a file from S3 and return its content."""
|
|
91
|
+
response = (client or get_client()).get_object(Bucket=bucket, Key=key)
|
|
92
|
+
content: bytes = response["Body"].read()
|
|
93
|
+
return content
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def label_file_name(
|
|
97
|
+
handle_unit_id: str, extension: str, *, suffix: str | None = "label", now: datetime | None = None
|
|
98
|
+
) -> str:
|
|
99
|
+
"""Build a label file name: ``{handle_unit_id}_{suffix}_{YYYYmmdd_HHMMSS}.{extension}``.
|
|
100
|
+
|
|
101
|
+
``suffix=None`` gives the template layout ``{handle_unit_id}_{YYYYmmdd_HHMMSS}.{extension}``.
|
|
102
|
+
"""
|
|
103
|
+
now = now or datetime.now()
|
|
104
|
+
parts = [handle_unit_id, suffix, f"{now:%Y%m%d_%H%M%S}"] if suffix else [handle_unit_id, f"{now:%Y%m%d_%H%M%S}"]
|
|
105
|
+
return f"{'_'.join(parts)}.{extension.lstrip('.')}"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def shipment_file_name(config: Mapping[str, Any], extension: str, *, now: datetime | None = None) -> str:
|
|
109
|
+
"""Build a shipment (EDI) file name: ``{YYYYmmdd_HHMMSS}_{shippingCarrierNumber}.{extension}``."""
|
|
110
|
+
now = now or datetime.now()
|
|
111
|
+
return f"{now:%Y%m%d_%H%M%S}_{config.get('shippingCarrierNumber')}.{extension.lstrip('.')}"
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def save_label(
|
|
115
|
+
handle_unit: Mapping[str, Any],
|
|
116
|
+
account_id: str,
|
|
117
|
+
config: Mapping[str, Any],
|
|
118
|
+
body: str | bytes,
|
|
119
|
+
*,
|
|
120
|
+
extension: str,
|
|
121
|
+
content_type: str,
|
|
122
|
+
carrier: str,
|
|
123
|
+
suffix: str | None = "label",
|
|
124
|
+
document: str = "label",
|
|
125
|
+
bucket: str | None = None,
|
|
126
|
+
encoding: str = "utf-8",
|
|
127
|
+
now: datetime | None = None,
|
|
128
|
+
client: Any = None,
|
|
129
|
+
) -> str:
|
|
130
|
+
"""Archive a label (or any HandleUnit document) in the labels bucket.
|
|
131
|
+
|
|
132
|
+
The key is ``{account}/{sender}/{service}/{YYYY}/{MM}/{DD}/{handle_unit_id}_{suffix}_{timestamp}.{extension}``
|
|
133
|
+
and the metadata ``carrier``, ``handle_unit_id``, ``document`` and ``document-type``.
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
handle_unit: the HandleUnit (``@id`` is read).
|
|
137
|
+
account_id: KloShip account id.
|
|
138
|
+
config: the CarrierAccountConfig.
|
|
139
|
+
body: label content (ZPL string, PDF bytes...).
|
|
140
|
+
extension: file extension (``"zpl"``, ``"pdf"``...), also stored as ``document-type`` metadata.
|
|
141
|
+
content_type: MIME type.
|
|
142
|
+
carrier: carrier name stored in the metadata.
|
|
143
|
+
suffix: file name suffix (``None`` to omit it), see :func:`label_file_name`.
|
|
144
|
+
document: ``document`` metadata value.
|
|
145
|
+
bucket: bucket name (defaults to the ``LABELS_BUCKET`` environment variable).
|
|
146
|
+
encoding: encoding used for ``str`` bodies.
|
|
147
|
+
now: date used for the key (defaults to now).
|
|
148
|
+
client: optional boto3 S3 client.
|
|
149
|
+
|
|
150
|
+
Returns:
|
|
151
|
+
The object key, to be stored in ``handle_unit["KLOSHIP_LABEL_FILENAME"]``.
|
|
152
|
+
"""
|
|
153
|
+
now = now or datetime.now()
|
|
154
|
+
key = build_key(account_id, config, label_file_name(handle_unit["@id"], extension, suffix=suffix, now=now), now=now)
|
|
155
|
+
return put_file(
|
|
156
|
+
require_env(LABELS_BUCKET_ENV, bucket),
|
|
157
|
+
key,
|
|
158
|
+
body,
|
|
159
|
+
content_type=content_type,
|
|
160
|
+
metadata={
|
|
161
|
+
"carrier": carrier,
|
|
162
|
+
"handle_unit_id": handle_unit["@id"],
|
|
163
|
+
"document": document,
|
|
164
|
+
"document-type": extension,
|
|
165
|
+
},
|
|
166
|
+
encoding=encoding,
|
|
167
|
+
client=client,
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def save_shipment_file(
|
|
172
|
+
body: str | bytes,
|
|
173
|
+
account_id: str,
|
|
174
|
+
config: Mapping[str, Any],
|
|
175
|
+
file_name: str,
|
|
176
|
+
*,
|
|
177
|
+
content_type: str,
|
|
178
|
+
carrier: str,
|
|
179
|
+
bucket: str | None = None,
|
|
180
|
+
encoding: str = "utf-8",
|
|
181
|
+
content_encoding: str | None = None,
|
|
182
|
+
metadata: Mapping[str, Any] | None = None,
|
|
183
|
+
now: datetime | None = None,
|
|
184
|
+
client: Any = None,
|
|
185
|
+
) -> str:
|
|
186
|
+
"""Archive a shipment file (EDI, manifest, XML...) in the shipment bucket.
|
|
187
|
+
|
|
188
|
+
Args:
|
|
189
|
+
body: file content.
|
|
190
|
+
account_id: KloShip account id.
|
|
191
|
+
config: the CarrierAccountConfig.
|
|
192
|
+
file_name: file name (see :func:`shipment_file_name`).
|
|
193
|
+
content_type: MIME type.
|
|
194
|
+
carrier: carrier name stored in the metadata.
|
|
195
|
+
bucket: bucket name (defaults to the ``SHIPMENT_BUCKET`` environment variable).
|
|
196
|
+
encoding: encoding used for ``str`` bodies.
|
|
197
|
+
content_encoding: optional ``ContentEncoding`` header.
|
|
198
|
+
metadata: extra metadata, merged with ``carrier`` and ``shippingCarrierNumber``.
|
|
199
|
+
now: date used for the key (defaults to now).
|
|
200
|
+
client: optional boto3 S3 client.
|
|
201
|
+
|
|
202
|
+
Returns:
|
|
203
|
+
The object key.
|
|
204
|
+
"""
|
|
205
|
+
all_metadata: dict[str, Any] = {"carrier": carrier, "shippingCarrierNumber": file_name}
|
|
206
|
+
all_metadata.update(metadata or {})
|
|
207
|
+
return put_file(
|
|
208
|
+
require_env(SHIPMENT_BUCKET_ENV, bucket),
|
|
209
|
+
build_key(account_id, config, file_name, now=now),
|
|
210
|
+
body,
|
|
211
|
+
content_type=content_type,
|
|
212
|
+
metadata=all_metadata,
|
|
213
|
+
encoding=encoding,
|
|
214
|
+
content_encoding=content_encoding,
|
|
215
|
+
client=client,
|
|
216
|
+
)
|