sdc-gdr2edifact 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdc_gdr2edifact/__init__.py +28 -0
- sdc_gdr2edifact/check.py +167 -0
- sdc_gdr2edifact/directory.py +299 -0
- sdc_gdr2edifact/mapper.py +130 -0
- sdc_gdr2edifact/mappings/credit_note-jkuyr38hydqaiwlq5vhp2rjr.yaml +529 -0
- sdc_gdr2edifact/mappings/despatch_advice-cacl4njrwanj15g3p3so8t6h.yaml +378 -0
- sdc_gdr2edifact/mappings/invoice-u19w614300a8ot4a7qnn2o1c.yaml +514 -0
- sdc_gdr2edifact/mappings/order-h8bttbt9afwzf9zs4jhae566.yaml +442 -0
- sdc_gdr2edifact/mappings/order_response-kuntv2wlkw54h7nxe0kqhvi2.yaml +598 -0
- sdc_gdr2edifact/mappings/receipt_advice-tog0v0p1zit1xwpxysxwpf3d.yaml +366 -0
- sdc_gdr2edifact/mappings/remittance_advice-fgtbf38cj0bwxppxn69etcci.yaml +246 -0
- sdc_gdr2edifact/mappings/torvale_order-cq1fjrvyn0fxl1kyk9y3kr1w.yaml +448 -0
- sdc_gdr2edifact/writer.py +383 -0
- sdc_gdr2edifact-4.0.0.dist-info/METADATA +153 -0
- sdc_gdr2edifact-4.0.0.dist-info/RECORD +20 -0
- sdc_gdr2edifact-4.0.0.dist-info/WHEEL +5 -0
- sdc_gdr2edifact-4.0.0.dist-info/entry_points.txt +2 -0
- sdc_gdr2edifact-4.0.0.dist-info/licenses/LICENSE +201 -0
- sdc_gdr2edifact-4.0.0.dist-info/licenses/NOTICE +14 -0
- sdc_gdr2edifact-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""sdc-gdr2edifact: Governed Data Records as UN/EDIFACT D.24A interchanges, through sdc-mapper.
|
|
2
|
+
|
|
3
|
+
from sdc_mapper import Engine, load_mapping
|
|
4
|
+
from sdc_gdr2edifact import EDIFACTMapper, mapping_path
|
|
5
|
+
mapper = EDIFACTMapper("/path/to/d24a.zip") # the directory as downloaded from UNECE; or $SDC_EDIFACT_DIRECTORY
|
|
6
|
+
result = Engine().map_record(xml, mapper, load_mapping(mapping_path("cacl4njrwanj15g3p3so8t6h")))
|
|
7
|
+
|
|
8
|
+
One mapping file per model, in `mappings/`. Registered with sdc-mapper as the target `edifact`.
|
|
9
|
+
"""
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
__version__ = "4.0.0"
|
|
13
|
+
|
|
14
|
+
MAPPINGS = Path(__file__).with_name("mappings")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def mapping_path(ct_id: str) -> Path:
|
|
18
|
+
"""The mapping file this package publishes for a model, by the model's identifier."""
|
|
19
|
+
found = sorted(MAPPINGS.glob(f"*-{ct_id}.yaml"))
|
|
20
|
+
if not found:
|
|
21
|
+
raise FileNotFoundError(f"sdc-gdr2edifact has no mapping for model {ct_id}")
|
|
22
|
+
return found[-1]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
from .directory import Directory, DirectoryNotFound # noqa: E402
|
|
26
|
+
from .mapper import EDIFACTMapper # noqa: E402
|
|
27
|
+
|
|
28
|
+
__all__ = ["EDIFACTMapper", "Directory", "DirectoryNotFound", "mapping_path", "MAPPINGS", "__version__"]
|
sdc_gdr2edifact/check.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""An interchange checked against the directory, from its text, by code that shares nothing with the writer.
|
|
2
|
+
|
|
3
|
+
errors = check(text, Directory.load()) # [] when the interchange conforms
|
|
4
|
+
|
|
5
|
+
The text is parsed afresh (the UNA service string, the release character, segments, data elements, components); the
|
|
6
|
+
envelope is checked (UNB first and UNZ last, one UNH to UNT message, UNT's segment count and UNZ's message count,
|
|
7
|
+
matching control references); the message's segment sequence is matched against the message specification's segment
|
|
8
|
+
table by a recursive descent over its positions (a segment position takes up to its repeat count of segments, a group
|
|
9
|
+
opens on its trigger segment and repeats up to its count, a mandatory position missing is an error, a segment nothing
|
|
10
|
+
takes is an error); and every segment against its definition (known tag, no more data elements or components than
|
|
11
|
+
defined, mandatory ones present, lengths and numeric representations within the directory's).
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
|
|
17
|
+
from .directory import Directory
|
|
18
|
+
|
|
19
|
+
REPR = re.compile(r"^(an|a|n)(\.\.)?(\d+)$")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def parse(text: str) -> list[tuple[str, list[list[str]]]]:
|
|
23
|
+
text = text.lstrip()
|
|
24
|
+
comp, elem, rel, term = ":", "+", "?", "'"
|
|
25
|
+
if text.startswith("UNA"):
|
|
26
|
+
comp, elem, rel, term = text[3], text[4], text[6], text[8]
|
|
27
|
+
text = text[9:]
|
|
28
|
+
segments, cur, comps, elements, tag = [], [], [], [], None
|
|
29
|
+
i = 0
|
|
30
|
+
while i < len(text):
|
|
31
|
+
ch = text[i]
|
|
32
|
+
if ch == rel and i + 1 < len(text):
|
|
33
|
+
cur.append(text[i + 1])
|
|
34
|
+
i += 2
|
|
35
|
+
continue
|
|
36
|
+
if ch in "\r\n" and not cur and tag is None:
|
|
37
|
+
i += 1
|
|
38
|
+
continue
|
|
39
|
+
if ch == elem or ch == term:
|
|
40
|
+
if tag is None:
|
|
41
|
+
tag = "".join(cur).strip()
|
|
42
|
+
else:
|
|
43
|
+
comps.append("".join(cur))
|
|
44
|
+
elements.append(comps)
|
|
45
|
+
cur, comps = [], []
|
|
46
|
+
if ch == term:
|
|
47
|
+
segments.append((tag, elements))
|
|
48
|
+
tag, elements = None, []
|
|
49
|
+
elif ch == comp and tag is not None:
|
|
50
|
+
comps.append("".join(cur))
|
|
51
|
+
cur = []
|
|
52
|
+
else:
|
|
53
|
+
cur.append(ch)
|
|
54
|
+
i += 1
|
|
55
|
+
if tag is not None or cur:
|
|
56
|
+
segments.append(("?", [["".join(cur)]]))
|
|
57
|
+
return segments
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _value(elements, i, j=0) -> str:
|
|
61
|
+
return elements[i][j] if i < len(elements) and j < len(elements[i]) else ""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def check(text: str, d: Directory) -> list[str]:
|
|
65
|
+
segs = parse(text)
|
|
66
|
+
errors: list[str] = []
|
|
67
|
+
tags = [t for t, _ in segs]
|
|
68
|
+
if not segs or tags[0] != "UNB" or tags[-1] != "UNZ":
|
|
69
|
+
return ["an interchange starts with UNB and ends with UNZ"]
|
|
70
|
+
if tags.count("UNH") != 1 or tags.count("UNT") != 1:
|
|
71
|
+
return [f"one message expected; {tags.count('UNH')} UNH and {tags.count('UNT')} UNT"]
|
|
72
|
+
h, t = tags.index("UNH"), tags.index("UNT")
|
|
73
|
+
msg = segs[h:t + 1]
|
|
74
|
+
unb, unh, unt, unz = segs[0][1], segs[h][1], segs[t][1], segs[-1][1]
|
|
75
|
+
if _value(unt, 0) != str(len(msg)):
|
|
76
|
+
errors.append(f"UNT counts {_value(unt, 0)} segments; the message has {len(msg)}")
|
|
77
|
+
if _value(unt, 1) != _value(unh, 0):
|
|
78
|
+
errors.append("UNT's message reference differs from UNH's")
|
|
79
|
+
if _value(unz, 0) != "1" or _value(unz, 1) != _value(unb, 4):
|
|
80
|
+
errors.append("UNZ's count or control reference does not match the interchange")
|
|
81
|
+
for k, (i, j) in enumerate(((0, 0), (0, 1), (1, 0), (2, 0), (3, 0), (3, 1), (4, 0))):
|
|
82
|
+
if not _value(unb, i, j):
|
|
83
|
+
errors.append(f"UNB is missing its mandatory data at element {i + 1}, component {j + 1}")
|
|
84
|
+
name = _value(unh, 1, 0)
|
|
85
|
+
spec = d.messages.get(name)
|
|
86
|
+
if spec is None:
|
|
87
|
+
return errors + [f"{name} is not a message of {d.version}"]
|
|
88
|
+
if (_value(unh, 1, 1), _value(unh, 1, 2), _value(unh, 1, 3)) != (spec["version"], spec["release"], spec["agency"]):
|
|
89
|
+
errors.append(f"UNH names {':'.join(unh[1][1:4])}, the directory {spec['version']}:{spec['release']}:{spec['agency']}")
|
|
90
|
+
errors += _table(spec["segment_table"], msg, name)
|
|
91
|
+
errors += _segments(msg, d)
|
|
92
|
+
return errors
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _table(table, segs, name) -> list[str]:
|
|
96
|
+
errors: list[str] = []
|
|
97
|
+
|
|
98
|
+
def match(items, i, where):
|
|
99
|
+
for it in items:
|
|
100
|
+
if "items" in it:
|
|
101
|
+
trigger, n = it["items"][0]["tag"], 0
|
|
102
|
+
while i < len(segs) and segs[i][0] == trigger and n < it["repeat"]:
|
|
103
|
+
i = match(it["items"], i, f"{where}/SG{it['group']}")
|
|
104
|
+
n += 1
|
|
105
|
+
if n == 0 and it["status"] == "M":
|
|
106
|
+
errors.append(f"{where}: mandatory segment group {it['group']} is missing")
|
|
107
|
+
else:
|
|
108
|
+
n = 0
|
|
109
|
+
while i < len(segs) and segs[i][0] == it["tag"] and n < it["repeat"]:
|
|
110
|
+
i += 1
|
|
111
|
+
n += 1
|
|
112
|
+
if n == 0 and it["status"] == "M":
|
|
113
|
+
errors.append(f"{where}: mandatory segment {it['tag']} is missing")
|
|
114
|
+
return i
|
|
115
|
+
|
|
116
|
+
i = match(table, 0, name)
|
|
117
|
+
if i < len(segs):
|
|
118
|
+
errors.append(f"{name}: segment {i + 1} ({segs[i][0]}) is not where the segment table allows it")
|
|
119
|
+
return errors
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _repr(value, repr_, where) -> list[str]:
|
|
123
|
+
m = REPR.match(repr_ or "")
|
|
124
|
+
if not m:
|
|
125
|
+
return []
|
|
126
|
+
kind, variable, size = m.group(1), m.group(2), int(m.group(3))
|
|
127
|
+
length = len(re.sub(r"[^0-9]", "", value)) if kind == "n" else len(value)
|
|
128
|
+
if length > size or (not variable and length != size):
|
|
129
|
+
return [f"{where}: {length} characters, representation {repr_}"]
|
|
130
|
+
if kind == "n" and not re.fullmatch(r"-?\d+(\.\d+)?", value):
|
|
131
|
+
return [f"{where}: '{value[:20]}' is not numeric ({repr_})"]
|
|
132
|
+
return []
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _segments(segs, d: Directory) -> list[str]:
|
|
136
|
+
errors: list[str] = []
|
|
137
|
+
for k, (tag, elements) in enumerate(segs, 1):
|
|
138
|
+
if tag in ("UNH", "UNT", "UNS"):
|
|
139
|
+
continue
|
|
140
|
+
spec = d.segments.get(tag)
|
|
141
|
+
if spec is None:
|
|
142
|
+
errors.append(f"segment {k}: {tag} is not a segment of {d.version}")
|
|
143
|
+
continue
|
|
144
|
+
defs = spec["elements"]
|
|
145
|
+
if len(elements) > len(defs):
|
|
146
|
+
errors.append(f"segment {k} ({tag}): {len(elements)} data elements; {len(defs)} defined")
|
|
147
|
+
for j, e in enumerate(defs):
|
|
148
|
+
given = elements[j] if j < len(elements) else []
|
|
149
|
+
present = any(c != "" for c in given)
|
|
150
|
+
if e["status"] == "M" and not present:
|
|
151
|
+
errors.append(f"segment {k} ({tag}): mandatory {e['tag']} is missing")
|
|
152
|
+
if not present:
|
|
153
|
+
continue
|
|
154
|
+
if e["components"]:
|
|
155
|
+
if len(given) > len(e["components"]):
|
|
156
|
+
errors.append(f"segment {k} ({tag}) {e['tag']}: {len(given)} components; {len(e['components'])} defined")
|
|
157
|
+
for c, cd in enumerate(e["components"]):
|
|
158
|
+
v = given[c] if c < len(given) else ""
|
|
159
|
+
if cd["status"] == "M" and v == "":
|
|
160
|
+
errors.append(f"segment {k} ({tag}) {e['tag']}: mandatory {cd['tag']} is missing")
|
|
161
|
+
elif v:
|
|
162
|
+
errors += _repr(v, cd["repr"], f"segment {k} ({tag}) {e['tag']}/{cd['tag']}")
|
|
163
|
+
else:
|
|
164
|
+
if len(given) > 1:
|
|
165
|
+
errors.append(f"segment {k} ({tag}) {e['tag']} is a simple data element; it has components")
|
|
166
|
+
errors += _repr(given[0], e["repr"], f"segment {k} ({tag}) {e['tag']}")
|
|
167
|
+
return errors
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
"""The UN/EDIFACT directory: read from the archive UNECE publishes, never shipped with this package.
|
|
2
|
+
|
|
3
|
+
UNECE licenses each directory (UNTDID: the data elements EDED, composites EDCD, segments EDSD, code lists UNCL and
|
|
4
|
+
message specifications EDMD) for use within the organization that downloads it, and lets the original file be passed
|
|
5
|
+
on with its license; a modified directory may not be distributed. So this package carries no part of a directory:
|
|
6
|
+
|
|
7
|
+
d = Directory.load() # $SDC_EDIFACT_DIRECTORY, or a path given to load()
|
|
8
|
+
d = Directory.load("/data/edifact/d24a.zip") # the zip downloaded from unece.org, unchanged
|
|
9
|
+
|
|
10
|
+
The archive is read as downloaded (its parts are zips inside the zip), extracted in memory, and the extraction kept as
|
|
11
|
+
a working copy in the user's cache directory, keyed by the archive's SHA-256, for use on that machine.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import hashlib
|
|
16
|
+
import io
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import zipfile
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import ClassVar
|
|
24
|
+
|
|
25
|
+
ENV = "SDC_EDIFACT_DIRECTORY"
|
|
26
|
+
EXTRACTION = 2 # the extraction's own version: a change to how the text files are read is a new cache entry
|
|
27
|
+
DASH = re.compile(r"^-{20,}\s*$")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class DirectoryNotFound(Exception):
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _cache_dir() -> Path:
|
|
35
|
+
base = os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache")
|
|
36
|
+
return Path(base) / "sdc-gdr2edifact"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# ---------------------------------------------------------------- reading the text files
|
|
40
|
+
def _text(raw: bytes) -> str:
|
|
41
|
+
try:
|
|
42
|
+
text = raw.decode("utf-8")
|
|
43
|
+
except UnicodeDecodeError:
|
|
44
|
+
text = raw.decode("latin-1", errors="replace")
|
|
45
|
+
return text.replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"')
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _blocks(text: str):
|
|
49
|
+
cur: list[str] = []
|
|
50
|
+
for line in text.splitlines():
|
|
51
|
+
if DASH.match(line):
|
|
52
|
+
if cur:
|
|
53
|
+
yield cur
|
|
54
|
+
cur = []
|
|
55
|
+
else:
|
|
56
|
+
cur.append(line)
|
|
57
|
+
if cur:
|
|
58
|
+
yield cur
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _joined(lines: list[str]) -> str:
|
|
62
|
+
return re.sub(r"\s+", " ", " ".join(x.strip() for x in lines)).strip()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _field(lines: list[str], name: str) -> str:
|
|
66
|
+
out, on = [], False
|
|
67
|
+
for x in lines:
|
|
68
|
+
m = re.match(rf"^\s*{name}:\s*(.*)$", x)
|
|
69
|
+
if m:
|
|
70
|
+
out.append(m.group(1))
|
|
71
|
+
on = True
|
|
72
|
+
continue
|
|
73
|
+
if on:
|
|
74
|
+
if x.strip() == "" or re.match(r"^\s*(Desc|Repr|Note|Function|Notes?):", x) or re.match(r"^\d{3}\s", x):
|
|
75
|
+
on = False
|
|
76
|
+
else:
|
|
77
|
+
out.append(x)
|
|
78
|
+
return _joined(out)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
ELEMENT_HEAD = re.compile(r"^[+*#|X\-]?\s{1,6}(\d{4})\s{2}(.+?)\s{2,}\[([BIC])\]\s*$")
|
|
82
|
+
COMPOSITE_HEAD = re.compile(r"^[+*#|X\-]?\s{1,8}(C\d{3})\s+(.+?)\s*$")
|
|
83
|
+
# A name can run into the status column with a single space ("...FUNCTION CODE QUALIFIER M 1 an..3"), so the
|
|
84
|
+
# status is found as a lone M or C before the repeat count or representation, not after two spaces.
|
|
85
|
+
COMPONENT = re.compile(r"^(\d{3})\s+(\d{4})\s+(.+?)\s+([MC])\s+(\S+)\s*$")
|
|
86
|
+
SEGMENT_HEAD = re.compile(r"^[+*#|X\-]?\s{1,8}([A-Z]{3})\s{2}(.+?)\s*$")
|
|
87
|
+
SEG_ELEMENT = re.compile(r"^(\d{3})\s+(C\d{3}|\d{4})\s+(.+?)\s+([MC])\s+(\d+)(?:\s+(\S+))?\s*$")
|
|
88
|
+
SEG_SUB = re.compile(r"^\s{4,}(\d{4})\s+(.+?)\s+([MC])\s+(\S+)\s*$")
|
|
89
|
+
CODE_LINE = re.compile(r"^[+*#|X\-]?\s{1,6}(\S{1,3})\s{2,}(.+?)\s*$")
|
|
90
|
+
GROUP = re.compile(r"^(\d{5})\s+-+\s+Segment group\s+(\d+)\s+-+\s+([MC])\s+(\d+)\s*-*\+\|*\s*$")
|
|
91
|
+
SEGLINE = re.compile(r"^(\d{5})\s+([A-Z]{3})\s+(.+?)\s{2,}([MC])\s+(\d+)\s*(-*)(\++)?\s*\|*\s*$")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _elements(text: str) -> dict:
|
|
95
|
+
out = {}
|
|
96
|
+
for b in _blocks(text):
|
|
97
|
+
head = next((x for x in b if ELEMENT_HEAD.match(x)), None)
|
|
98
|
+
if head:
|
|
99
|
+
m = ELEMENT_HEAD.match(head)
|
|
100
|
+
out[m.group(1)] = {"tag": m.group(1), "name": m.group(2).strip(), "repr": _field(b, "Repr")}
|
|
101
|
+
return out
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _composites(text: str) -> dict:
|
|
105
|
+
out = {}
|
|
106
|
+
for b in _blocks(text):
|
|
107
|
+
head = next((x for x in b if COMPOSITE_HEAD.match(x) and not x.strip().startswith("Desc")), None)
|
|
108
|
+
if not head:
|
|
109
|
+
continue
|
|
110
|
+
m = COMPOSITE_HEAD.match(head)
|
|
111
|
+
comps, pending = [], None
|
|
112
|
+
for x in b:
|
|
113
|
+
cm = COMPONENT.match(x)
|
|
114
|
+
if cm:
|
|
115
|
+
comps.append({"tag": cm.group(2), "name": cm.group(3).strip(), "status": cm.group(4), "repr": cm.group(5)})
|
|
116
|
+
pending = None
|
|
117
|
+
elif re.match(r"^(\d{3})\s+(\d{4})\s+(.+)$", x):
|
|
118
|
+
pending = re.match(r"^(\d{3})\s+(\d{4})\s+(.+)$", x)
|
|
119
|
+
elif pending and x.strip():
|
|
120
|
+
tail = re.match(r"^\s*(.+?)\s{2,}([MC])\s+(\S+)\s*$", x)
|
|
121
|
+
if tail:
|
|
122
|
+
comps.append({"tag": pending.group(2), "name": f"{pending.group(3).strip()} {tail.group(1).strip()}",
|
|
123
|
+
"status": tail.group(2), "repr": tail.group(3)})
|
|
124
|
+
pending = None
|
|
125
|
+
out[m.group(1)] = {"tag": m.group(1), "name": m.group(2).strip(), "components": comps}
|
|
126
|
+
return out
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _segments(text: str) -> dict:
|
|
130
|
+
out = {}
|
|
131
|
+
for b in _blocks(text):
|
|
132
|
+
head = next((x for x in b if SEGMENT_HEAD.match(x) and x.strip()[:3].isalpha() and x.strip()[:3].isupper()
|
|
133
|
+
and not x.strip().startswith(("Function", "Desc", "Note"))), None)
|
|
134
|
+
if not head or "Function:" not in "\n".join(b):
|
|
135
|
+
continue
|
|
136
|
+
m = SEGMENT_HEAD.match(head)
|
|
137
|
+
items, pending = [], None
|
|
138
|
+
for x in b:
|
|
139
|
+
em = SEG_ELEMENT.match(x)
|
|
140
|
+
if em:
|
|
141
|
+
items.append({"tag": em.group(2), "name": em.group(3).strip(), "status": em.group(4),
|
|
142
|
+
"repeat": int(em.group(5)), "repr": em.group(6) or "", "components": []})
|
|
143
|
+
pending = None
|
|
144
|
+
continue
|
|
145
|
+
sm = SEG_SUB.match(x)
|
|
146
|
+
if sm and items and items[-1]["tag"].startswith("C"):
|
|
147
|
+
items[-1]["components"].append({"tag": sm.group(1), "name": sm.group(2).strip(), "status": sm.group(3),
|
|
148
|
+
"repr": sm.group(4)})
|
|
149
|
+
pending = None
|
|
150
|
+
continue
|
|
151
|
+
wm = re.match(r"^(\d{3})\s+(C\d{3}|\d{4})\s+(.+)$", x)
|
|
152
|
+
if wm:
|
|
153
|
+
pending = ("item", wm)
|
|
154
|
+
continue
|
|
155
|
+
wsm = re.match(r"^\s{4,}(\d{4})\s{2}(.+)$", x)
|
|
156
|
+
if wsm and items and items[-1]["tag"].startswith("C"):
|
|
157
|
+
pending = ("sub", wsm)
|
|
158
|
+
continue
|
|
159
|
+
if pending and x.strip():
|
|
160
|
+
tail = re.match(r"^\s*(.+?)\s{2,}([MC])\s+(\d+)?\s*(\S+)?\s*$", x)
|
|
161
|
+
if tail:
|
|
162
|
+
kind, pm = pending
|
|
163
|
+
if kind == "item":
|
|
164
|
+
items.append({"tag": pm.group(2), "name": f"{pm.group(3).strip()} {tail.group(1).strip()}",
|
|
165
|
+
"status": tail.group(2), "repeat": int(tail.group(3) or 1), "repr": tail.group(4) or "",
|
|
166
|
+
"components": []})
|
|
167
|
+
else:
|
|
168
|
+
items[-1]["components"].append({"tag": pm.group(1), "name": f"{pm.group(2).strip()} {tail.group(1).strip()}",
|
|
169
|
+
"status": tail.group(2), "repr": tail.group(4) or tail.group(3) or ""})
|
|
170
|
+
pending = None
|
|
171
|
+
out[m.group(1)] = {"tag": m.group(1), "name": m.group(2).strip(), "elements": items}
|
|
172
|
+
return out
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _codelists(text: str) -> dict:
|
|
176
|
+
out = {}
|
|
177
|
+
for b in _blocks(text):
|
|
178
|
+
head = next((x for x in b if ELEMENT_HEAD.match(x)), None)
|
|
179
|
+
if not head:
|
|
180
|
+
continue
|
|
181
|
+
tag = ELEMENT_HEAD.match(head).group(1)
|
|
182
|
+
codes, started = [], False
|
|
183
|
+
for x in b:
|
|
184
|
+
if x is head:
|
|
185
|
+
started = True
|
|
186
|
+
continue
|
|
187
|
+
if not started or re.match(r"^\s*(Desc|Repr|Note):", x):
|
|
188
|
+
continue
|
|
189
|
+
cm = CODE_LINE.match(x)
|
|
190
|
+
if cm and (len(x) - len(x.lstrip(" +*#|X-")) <= 6) and not x.startswith(" "):
|
|
191
|
+
codes.append({"value": cm.group(1), "name": cm.group(2).strip()})
|
|
192
|
+
out[tag] = {"tag": tag, "codes": codes}
|
|
193
|
+
return out
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _message(text: str, name: str) -> dict:
|
|
197
|
+
lines = text.splitlines()
|
|
198
|
+
head = {}
|
|
199
|
+
for key, pat in (("type", r"Message Type\s*:\s*(\S+)"), ("version", r"Version\s*:\s*(\S+)"),
|
|
200
|
+
("release", r"Release\s*:\s*(\S+)"), ("agency", r"Contr\. Agency\s*:\s*(\S+)")):
|
|
201
|
+
m = re.search(pat, text)
|
|
202
|
+
head[key] = m.group(1) if m else ""
|
|
203
|
+
n = max((i for i, x in enumerate(lines) if re.match(r"^\s*Pos\s+Tag\s+Name", x)), default=None)
|
|
204
|
+
root = {"group": 0, "status": "M", "repeat": 1, "items": []}
|
|
205
|
+
if n is not None:
|
|
206
|
+
stack = [root]
|
|
207
|
+
for x in lines[n + 1:]:
|
|
208
|
+
gm = GROUP.match(x)
|
|
209
|
+
if gm:
|
|
210
|
+
g = {"group": int(gm.group(2)), "status": gm.group(3), "repeat": int(gm.group(4)), "items": []}
|
|
211
|
+
stack[-1]["items"].append(g)
|
|
212
|
+
stack.append(g)
|
|
213
|
+
continue
|
|
214
|
+
sm = SEGLINE.match(x)
|
|
215
|
+
if sm:
|
|
216
|
+
stack[-1]["items"].append({"tag": sm.group(2), "name": sm.group(3).strip(), "status": sm.group(4),
|
|
217
|
+
"repeat": int(sm.group(5))})
|
|
218
|
+
for _ in range(len(sm.group(7) or "")):
|
|
219
|
+
if len(stack) > 1:
|
|
220
|
+
stack.pop()
|
|
221
|
+
return {"type": head["type"] or name.split("_")[0], "version": head["version"], "release": head["release"],
|
|
222
|
+
"agency": head["agency"], "segment_table": root["items"]}
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def extract(archive: bytes) -> dict:
|
|
226
|
+
"""A directory archive (the zip as downloaded from UNECE) to its definitions."""
|
|
227
|
+
outer = zipfile.ZipFile(io.BytesIO(archive))
|
|
228
|
+
parts = {}
|
|
229
|
+
for name in outer.namelist():
|
|
230
|
+
if name.lower().endswith(".zip"):
|
|
231
|
+
parts[name.lower().split("/")[-1][:-4]] = zipfile.ZipFile(io.BytesIO(outer.read(name)))
|
|
232
|
+
|
|
233
|
+
def one(part: str, prefix: str) -> tuple[str, str]:
|
|
234
|
+
z = parts[part]
|
|
235
|
+
name = next(n for n in z.namelist() if n.upper().split("/")[-1].startswith(prefix + ".")
|
|
236
|
+
and not n.upper().split("/")[-1].startswith(prefix + "I"))
|
|
237
|
+
return name, _text(z.read(name))
|
|
238
|
+
|
|
239
|
+
eded_name, eded = one("eded", "EDED")
|
|
240
|
+
suffix = eded_name.split(".")[-1].upper() # 24A
|
|
241
|
+
version = f"D.{suffix}"
|
|
242
|
+
messages = {}
|
|
243
|
+
for name in parts["edmd"].namelist():
|
|
244
|
+
if name.endswith("/"):
|
|
245
|
+
continue
|
|
246
|
+
m = _message(_text(parts["edmd"].read(name)), name.split("/")[-1])
|
|
247
|
+
messages[m["type"]] = m
|
|
248
|
+
return {"version": version, "elements": _elements(eded), "composites": _composites(one("edcd", "EDCD")[1]),
|
|
249
|
+
"segments": _segments(one("edsd", "EDSD")[1]), "codelists": _codelists(one("uncl", "UNCL")[1]),
|
|
250
|
+
"messages": messages}
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
@dataclass
|
|
254
|
+
class Directory:
|
|
255
|
+
version: str
|
|
256
|
+
elements: dict
|
|
257
|
+
composites: dict
|
|
258
|
+
segments: dict
|
|
259
|
+
codelists: dict
|
|
260
|
+
messages: dict
|
|
261
|
+
sha256: str = ""
|
|
262
|
+
_cache: dict = field(default_factory=dict, repr=False)
|
|
263
|
+
|
|
264
|
+
_loaded: ClassVar[dict] = {}
|
|
265
|
+
|
|
266
|
+
@classmethod
|
|
267
|
+
def load(cls, path: str | os.PathLike | None = None) -> "Directory":
|
|
268
|
+
path = path or os.environ.get(ENV)
|
|
269
|
+
if not path:
|
|
270
|
+
raise DirectoryNotFound(
|
|
271
|
+
f"the UN/EDIFACT directory is needed: download the D.24A archive from UNECE "
|
|
272
|
+
f"(https://unece.org/trade/uncefact/unedifact/download) and set {ENV} to the zip's path")
|
|
273
|
+
p = Path(path)
|
|
274
|
+
if not p.exists():
|
|
275
|
+
raise DirectoryNotFound(f"no UN/EDIFACT directory archive at {p}")
|
|
276
|
+
raw = p.read_bytes()
|
|
277
|
+
sha = hashlib.sha256(raw).hexdigest()
|
|
278
|
+
if sha in cls._loaded:
|
|
279
|
+
return cls._loaded[sha]
|
|
280
|
+
cache = _cache_dir() / f"{sha}-{EXTRACTION}.json"
|
|
281
|
+
if cache.exists():
|
|
282
|
+
data = json.loads(cache.read_text(encoding="utf-8"))
|
|
283
|
+
else:
|
|
284
|
+
data = extract(raw)
|
|
285
|
+
cache.parent.mkdir(parents=True, exist_ok=True)
|
|
286
|
+
cache.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
|
287
|
+
obj = cls(data["version"], data["elements"], data["composites"], data["segments"], data["codelists"],
|
|
288
|
+
data["messages"], sha256=sha)
|
|
289
|
+
cls._loaded[sha] = obj
|
|
290
|
+
return obj
|
|
291
|
+
|
|
292
|
+
def message(self, name: str) -> dict:
|
|
293
|
+
if name not in self.messages:
|
|
294
|
+
raise KeyError(f"{name} is not a message of {self.version}")
|
|
295
|
+
return self.messages[name]
|
|
296
|
+
|
|
297
|
+
def code(self, element: str, value: str) -> dict | None:
|
|
298
|
+
cl = self.codelists.get(element)
|
|
299
|
+
return next((c for c in cl["codes"] if c["value"] == value), None) if cl else None
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""The UN/EDIFACT mapper: a Governed Data Record as a D.24A interchange, by a mapping file.
|
|
2
|
+
|
|
3
|
+
The business documents library links its components to UBL, not to UN/EDIFACT, so where a value goes in a message
|
|
4
|
+
(a segment, a qualifier, a data element) is declared in a mapping file, one per model. Where the model does bind a
|
|
5
|
+
value to a code list the directory uses, the mapping's `binding` transform takes the code from the model and the value
|
|
6
|
+
is a lookup: UBL 2.3's PaymentMeansCode list is UNCL 4461, code for code, and the packaging types are UN/ECE
|
|
7
|
+
Recommendation 21 codes.
|
|
8
|
+
|
|
9
|
+
UN/EDIFACT has no null flavors. A free text segment could hold the words, but no qualifier the directory defines says
|
|
10
|
+
"this value was asked for and refused" in a way a receiving system would act on, so an Exceptional Value is lost, and
|
|
11
|
+
where the directory requires the value, the segment, the group or the message cannot be written, and the report says
|
|
12
|
+
which and why.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
|
|
18
|
+
from sdc_mapper import (DECLARED, JOIN, LOOKUP, LOST, NOT_REPRESENTABLE, EVRule, IncompleteMapping, MapResult,
|
|
19
|
+
MappingError, ValueOutcome, place)
|
|
20
|
+
from sdcreader.record import EXCEPTIONAL_VALUES
|
|
21
|
+
|
|
22
|
+
from . import __version__
|
|
23
|
+
from .check import check
|
|
24
|
+
from .directory import ENV, Directory
|
|
25
|
+
from .writer import Builder
|
|
26
|
+
|
|
27
|
+
CANNOT_HOLD = [
|
|
28
|
+
"Exceptional Values: UN/EDIFACT has no null flavors",
|
|
29
|
+
"validity windows and the time a value was recorded (act, vtb, vte, tr)",
|
|
30
|
+
"the definitions the model binds each component to",
|
|
31
|
+
"the record's provenance and audit clusters",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class EDIFACTMapper:
|
|
36
|
+
name = "edifact"
|
|
37
|
+
target = "UN/EDIFACT"
|
|
38
|
+
target_version = "D.24A"
|
|
39
|
+
package = "sdc-gdr2edifact"
|
|
40
|
+
version = __version__
|
|
41
|
+
ev_table = {code: EVRule(LOST) for code in EXCEPTIONAL_VALUES}
|
|
42
|
+
|
|
43
|
+
def __init__(self, directory: str | os.PathLike | Directory | None = None):
|
|
44
|
+
self._directory = directory
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def directory(self) -> Directory:
|
|
48
|
+
if not isinstance(self._directory, Directory):
|
|
49
|
+
self._directory = Directory.load(self._directory or os.environ.get(ENV))
|
|
50
|
+
return self._directory
|
|
51
|
+
|
|
52
|
+
def map(self, record, mapping) -> MapResult:
|
|
53
|
+
if mapping is None:
|
|
54
|
+
raise MappingError("UN/EDIFACT needs a mapping file: the model does not say where its values go in a message")
|
|
55
|
+
if mapping.carry_rejected:
|
|
56
|
+
raise MappingError("UN/EDIFACT has no agreed place to carry a rejected value; carry_rejected is not available")
|
|
57
|
+
for code, override in mapping.exceptional_values.items():
|
|
58
|
+
if override["class"] != LOST:
|
|
59
|
+
raise MappingError(f"UN/EDIFACT cannot say {code}: it has no null flavors, so an Exceptional Value is lost")
|
|
60
|
+
placements, clusters = place(record, mapping)
|
|
61
|
+
message = next((p.steps[0][0] for p in placements if p.steps), None)
|
|
62
|
+
if message is None:
|
|
63
|
+
raise MappingError("nothing to write: no value of the record reached the message")
|
|
64
|
+
b = Builder(self.directory, message, mapping.cluster_order)
|
|
65
|
+
notes: dict[int, str] = {}
|
|
66
|
+
later = []
|
|
67
|
+
for p in placements:
|
|
68
|
+
v = p.value
|
|
69
|
+
if v.exceptional or p.omitted or p.rule is None or not p.steps:
|
|
70
|
+
continue
|
|
71
|
+
if p.text is None:
|
|
72
|
+
notes[v.position] = p.note or "no value to write"
|
|
73
|
+
continue
|
|
74
|
+
writes = [(p.steps, p.attribute, p.text, v.position, p)]
|
|
75
|
+
writes += [(e.steps, e.attribute, e.text, None, None) for e in p.extras if e.text is not None]
|
|
76
|
+
for w in writes:
|
|
77
|
+
if any(key == JOIN for _, key in w[0]):
|
|
78
|
+
later.append(w)
|
|
79
|
+
else:
|
|
80
|
+
self._write(b, *w)
|
|
81
|
+
# a join waits for the elements it may join: the shallower the deepest join of a write, the earlier it goes
|
|
82
|
+
later.sort(key=lambda w: max(i for i, (_, key) in enumerate(w[0]) if key == JOIN))
|
|
83
|
+
for w in later:
|
|
84
|
+
self._write(b, *w)
|
|
85
|
+
# A cluster's constant data (a NAD's party qualifier, a CPS's packaging level) go on its segment; the segment is
|
|
86
|
+
# written when the cluster's group holds anything of the record's, though no value lands in the segment itself.
|
|
87
|
+
for c in clusters:
|
|
88
|
+
if not c.attributes:
|
|
89
|
+
continue
|
|
90
|
+
node = b.find(c.steps, loose=True)
|
|
91
|
+
parent = b.find(c.steps[:-1], loose=True) if len(c.steps) > 2 else None
|
|
92
|
+
if node is None and parent is not None and parent.children:
|
|
93
|
+
node = b.ensure_under(parent, c.steps[-1])
|
|
94
|
+
if node is not None:
|
|
95
|
+
for k, val in c.attributes.items():
|
|
96
|
+
node.attrs.setdefault(k, val)
|
|
97
|
+
text = b.interchange()
|
|
98
|
+
errors = check(text, self.directory)
|
|
99
|
+
if errors:
|
|
100
|
+
raise MappingError(f"the {message} does not conform to {self.directory.version}: {errors[0]}")
|
|
101
|
+
outcomes, missing = [], []
|
|
102
|
+
for p in placements:
|
|
103
|
+
v = p.value
|
|
104
|
+
if v.exceptional:
|
|
105
|
+
outcomes.append(ValueOutcome(v.position, LOST, note="UN/EDIFACT has no null flavors"))
|
|
106
|
+
elif p.omitted:
|
|
107
|
+
outcomes.append(ValueOutcome(v.position, NOT_REPRESENTABLE, note=p.omitted_by.reason or ""))
|
|
108
|
+
elif p.rule is None:
|
|
109
|
+
missing.append(f"{' / '.join(v.path_labels + [v.label])} ({v.ct_id})")
|
|
110
|
+
elif v.position in notes:
|
|
111
|
+
outcomes.append(ValueOutcome(v.position, NOT_REPRESENTABLE, note=notes[v.position]))
|
|
112
|
+
elif v.position in b.pruned:
|
|
113
|
+
outcomes.append(ValueOutcome(v.position, NOT_REPRESENTABLE, note=b.pruned[v.position]))
|
|
114
|
+
else:
|
|
115
|
+
cls = LOOKUP if p.rule.transform == "binding" else DECLARED
|
|
116
|
+
where = "/".join(n for n, _ in p.steps) + (f"/@{p.attribute}" if p.attribute else "")
|
|
117
|
+
outcomes.append(ValueOutcome(v.position, cls, location=where))
|
|
118
|
+
if missing:
|
|
119
|
+
raise IncompleteMapping(f"mapping {mapping.id} has no rule for {len(missing)} value(s): " + "; ".join(missing[:5]))
|
|
120
|
+
return MapResult(files={f"{message}.edi": text.encode("latin-1")}, outcomes=outcomes,
|
|
121
|
+
cannot_hold=list(CANNOT_HOLD))
|
|
122
|
+
|
|
123
|
+
@staticmethod
|
|
124
|
+
def _write(b: Builder, steps, attribute, text, position, p):
|
|
125
|
+
node = b.put(steps, attribute, text, position)
|
|
126
|
+
if p is not None:
|
|
127
|
+
for k, val in p.attributes.items():
|
|
128
|
+
node.attrs.setdefault(k, val)
|
|
129
|
+
if p.units_attribute and p.units_text is not None:
|
|
130
|
+
node.attrs[p.units_attribute] = p.units_text
|