datex2-xml2json 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. datex2_xml2json/__init__.py +0 -0
  2. datex2_xml2json/json_validator.py +36 -0
  3. datex2_xml2json/merge.py +39 -0
  4. datex2_xml2json/schema/Baustellen-4-00-00.sch +7 -0
  5. datex2_xml2json/schema/DATEXIISchema_2_2_0.xsd +4707 -0
  6. datex2_xml2json/schema/DATEXIISchema_2_2_3.xsd +6783 -0
  7. datex2_xml2json/schema/DATEXII_3.7_Profile/.DS_Store +0 -0
  8. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_AfirEnergyInfrastructure.json +2455 -0
  9. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_AfirFacilities.json +2809 -0
  10. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Common.json +3651 -0
  11. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_CommonExtension.json +919 -0
  12. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_ControlledZone.json +1070 -0
  13. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_D2Payload.json +11 -0
  14. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_EnergyInfrastructure.json +1924 -0
  15. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Facilities.json +2858 -0
  16. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_FaultAndStatus.json +1415 -0
  17. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_LocationExtension.json +186 -0
  18. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_LocationReferencing.json +4847 -0
  19. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Parking.json +5314 -0
  20. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_ReroutingManagementEnhanced.json +1160 -0
  21. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_RoadTrafficData.json +1938 -0
  22. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Situation.json +8040 -0
  23. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_TrafficManagementPlan.json +2358 -0
  24. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_TrafficRegulation.json +3377 -0
  25. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_UrbanExtensions.json +194 -0
  26. datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Vms.json +2544 -0
  27. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/.DS_Store +0 -0
  28. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_AfirEnergyInfrastructure.xsd +2102 -0
  29. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_AfirFacilities.xsd +2060 -0
  30. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Common.xsd +4152 -0
  31. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_CommonExtension.xsd +1102 -0
  32. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_ControlledZone.xsd +601 -0
  33. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_D2Payload.xsd +280 -0
  34. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_EnergyInfrastructure.xsd +1761 -0
  35. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Facilities.xsd +2037 -0
  36. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_FaultAndStatus.xsd +1076 -0
  37. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_LocationExtension.xsd +196 -0
  38. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_LocationReferencing.xsd +4132 -0
  39. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Parking.xsd +4025 -0
  40. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_ReroutingManagementEnhanced.xsd +997 -0
  41. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_RoadTrafficData.xsd +1273 -0
  42. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Situation.xsd +5777 -0
  43. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_TrafficManagementPlan.xsd +1184 -0
  44. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_TrafficRegulation.xsd +2611 -0
  45. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_UrbanExtensions.xsd +148 -0
  46. datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Vms.xsd +2290 -0
  47. datex2_xml2json/schema/MDM_BaustellenInfoDigitalRoadworksProfile_04-03-00.xsd +2401 -0
  48. datex2_xml2json/schema/MDM_Datenmodell_Baustellen_04-00-00.xsd +3232 -0
  49. datex2_xml2json/schema/cifs_schema.json +161 -0
  50. datex2_xml2json/transform/.DS_Store +0 -0
  51. datex2_xml2json/transform/__init__.py +13 -0
  52. datex2_xml2json/transform/base.py +79 -0
  53. datex2_xml2json/transform/cifs_transformer.py +275 -0
  54. datex2_xml2json/transform/geojson_geometry_transformer.py +159 -0
  55. datex2_xml2json/transform/geoparquet_transformer.py +160 -0
  56. datex2_xml2json/transform/jsonata_transformer.py +100 -0
  57. datex2_xml2json/transform/roads_regio_transformer.py +88 -0
  58. datex2_xml2json/transform/source_setter.py +42 -0
  59. datex2_xml2json/transform/transformation_rules.py +526 -0
  60. datex2_xml2json/transformer.py +197 -0
  61. datex2_xml2json/util.py +141 -0
  62. datex2_xml2json-0.0.1.dist-info/METADATA +176 -0
  63. datex2_xml2json-0.0.1.dist-info/RECORD +65 -0
  64. datex2_xml2json-0.0.1.dist-info/WHEEL +4 -0
  65. datex2_xml2json-0.0.1.dist-info/licenses/LICENSE.md +179 -0
@@ -0,0 +1,197 @@
1
+ import json
2
+ import logging
3
+ from argparse import ArgumentParser
4
+ from enum import StrEnum
5
+ from pathlib import Path
6
+
7
+ from .json_validator import JsonValidator
8
+ from .transform import (
9
+ CifsTransformer,
10
+ GeoJSONGeometryTransformer,
11
+ GeoparquetRenderer,
12
+ JsonataTransformer,
13
+ RoadsRegioTransformer,
14
+ RulesTransformer,
15
+ SourceSetter,
16
+ TransformationRuleFactory,
17
+ TransformerPipeline,
18
+ )
19
+ from .util import load_as_json_using_schema, schema_path
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+
24
+ class InputFormat(StrEnum):
25
+ DATEX2_2_2_XML = 'datex2_2_2_xml'
26
+ DATEX2_3_7_XML = 'datex2_3_7_xml'
27
+
28
+
29
+ class OutputFormat(StrEnum):
30
+ RAW = 'raw'
31
+ CIFS = 'cifs'
32
+ GEOJSON = 'geojson'
33
+ DATEX2_3_7_JSON = 'datex2_3_7_json'
34
+ DATEX2_3_7_JSON_LITE = 'datex2_3_7_json_lite'
35
+ GEOPARQUET = 'geoparquet'
36
+
37
+
38
+ FILE_SUFFIXES_PER_FORMAT = {
39
+ OutputFormat.RAW: '.raw.json',
40
+ OutputFormat.CIFS: '.cifs.json',
41
+ OutputFormat.GEOJSON: '.geojson',
42
+ OutputFormat.DATEX2_3_7_JSON: '.json',
43
+ OutputFormat.DATEX2_3_7_JSON_LITE: '.lite.json',
44
+ OutputFormat.GEOPARQUET: '.parquet',
45
+ }
46
+
47
+ DEFAULT_SCHEMAS = {
48
+ InputFormat.DATEX2_2_2_XML: 'schema/DATEXIISchema_2_2_3.xsd',
49
+ InputFormat.DATEX2_3_7_XML: 'schema/DATEXII_3.7_Profile/xml/DATEXII_3_D2Payload.xsd',
50
+ }
51
+
52
+ OUTPUT_SCHEMAS = {
53
+ OutputFormat.DATEX2_3_7_JSON: 'schema/DATEXII_3.7_Profile/DATEXII_3_D2Payload.json',
54
+ OutputFormat.CIFS: 'schema/cifs_schema.json',
55
+ }
56
+
57
+
58
+ def build_transformer(input_format: InputFormat, output_format: OutputFormat):
59
+ if input_format == InputFormat.DATEX2_2_2_XML:
60
+ if output_format == OutputFormat.DATEX2_3_7_JSON:
61
+ return TransformerPipeline(
62
+ [
63
+ RulesTransformer(TransformationRuleFactory().xmlv2_to_json3_7()),
64
+ RoadsRegioTransformer(),
65
+ ]
66
+ )
67
+ elif output_format == OutputFormat.DATEX2_3_7_JSON_LITE:
68
+ return TransformerPipeline(
69
+ [
70
+ RulesTransformer(TransformationRuleFactory().xmlv2_to_json()),
71
+ RoadsRegioTransformer('payload/situation/situationRecord'),
72
+ GeoJSONGeometryTransformer('payload/situation/situationRecord'),
73
+ ]
74
+ )
75
+ elif output_format == OutputFormat.GEOJSON:
76
+ return build_transformer(input_format, OutputFormat.DATEX2_3_7_JSON_LITE).add_transformer(JsonataTransformer())
77
+ elif output_format == OutputFormat.CIFS:
78
+ return build_transformer(input_format, OutputFormat.GEOJSON).add_transformer(CifsTransformer())
79
+ elif output_format == OutputFormat.RAW:
80
+ return TransformerPipeline([])
81
+
82
+ else:
83
+ return TransformerPipeline([RulesTransformer(TransformationRuleFactory().xmlv3_to_json3_7())])
84
+
85
+
86
+ def write(stem: str, json_result: dict, output_format: OutputFormat, outDir: str, validate: bool = False):
87
+ suffix = FILE_SUFFIXES_PER_FORMAT[output_format]
88
+ outfile_name = Path(outDir) / (stem + suffix)
89
+ outfile_name.parent.mkdir(parents=True, exist_ok=True)
90
+
91
+ if output_format != OutputFormat.GEOPARQUET:
92
+ with open(outfile_name, 'w') as f:
93
+ json.dump(json_result, f, indent=2)
94
+ if validate:
95
+ exit(JsonValidator(schema_path(OUTPUT_SCHEMAS[output_format])).validate(outfile_name))
96
+ else:
97
+ GeoparquetRenderer().render(outfile_name, json_result)
98
+
99
+
100
+ def generate_all(stem: str, json_input: dict, outDir: str, validate: bool):
101
+ write(stem, json_input, OutputFormat.RAW, outDir)
102
+ jsonLiteTransformer = build_transformer(InputFormat.DATEX2_2_2_XML, OutputFormat.DATEX2_3_7_JSON_LITE)
103
+ jsonLite_output = jsonLiteTransformer.transform(json_input)
104
+ write(stem, jsonLite_output, OutputFormat.DATEX2_3_7_JSON_LITE, outDir)
105
+ geojson_output = JsonataTransformer().transform(jsonLite_output)
106
+ write(stem, geojson_output, OutputFormat.GEOJSON, outDir)
107
+ write(stem, geojson_output, OutputFormat.GEOPARQUET, outDir)
108
+ cifs_output = CifsTransformer().transform(geojson_output)
109
+ write(stem, cifs_output, OutputFormat.CIFS, outDir, validate)
110
+
111
+
112
+ def transform(
113
+ filename: str,
114
+ outDir: str,
115
+ input_format: InputFormat,
116
+ output_format: OutputFormat,
117
+ validate: bool,
118
+ sourceIdentification: str,
119
+ publicationIdentifier: str,
120
+ ):
121
+ stem = Path(filename).stem
122
+ json_input = load_as_json_using_schema(filename, schema_path(DEFAULT_SCHEMAS[input_format]))
123
+
124
+ if sourceIdentification or publicationIdentifier:
125
+ SourceSetter(publicationIdentifier, sourceIdentification).transform(json_input)
126
+
127
+ if output_format is None:
128
+ generate_all(stem, json_input, outDir, validate)
129
+ else:
130
+ json_output_format = OutputFormat.GEOJSON if output_format == OutputFormat.GEOPARQUET else output_format
131
+ transformer = build_transformer(input_format, json_output_format)
132
+ result = transformer.transform(json_input)
133
+
134
+ write(stem, result, output_format, outDir, validate)
135
+
136
+
137
+ if __name__ == '__main__':
138
+ parser = ArgumentParser()
139
+ parser.add_argument('filename')
140
+ parser.add_argument(
141
+ '-V',
142
+ '--validate',
143
+ help='validate result against DATEXIIv3 schema (requires output format datex2_3_7_json)',
144
+ action='store_true',
145
+ )
146
+ parser.add_argument(
147
+ '-i',
148
+ '--input_format',
149
+ help='input format',
150
+ type=InputFormat,
151
+ choices=list(InputFormat),
152
+ default=InputFormat.DATEX2_2_2_XML,
153
+ )
154
+ parser.add_argument(
155
+ '-f',
156
+ '--output_format',
157
+ help='output format to generate',
158
+ type=OutputFormat,
159
+ choices=list(OutputFormat),
160
+ )
161
+
162
+ parser.add_argument(
163
+ '-o',
164
+ '--outputDir',
165
+ help='output directory the transformed files are written to',
166
+ default='out',
167
+ )
168
+
169
+ parser.add_argument(
170
+ '-s',
171
+ '--sourceIdentification',
172
+ help='sourceIdentification to be set for situationRecords source',
173
+ default=None,
174
+ )
175
+
176
+ parser.add_argument(
177
+ '-p',
178
+ '--publicationIdentifier',
179
+ help='publicationIdentifier to be set as payloadPublicationExtension/publicationIdentifier',
180
+ default=None,
181
+ )
182
+
183
+ args = parser.parse_args()
184
+
185
+ if args.validate and args.output_format is not None and args.output_format not in OUTPUT_SCHEMAS:
186
+ logger.error('Validation is not supported for format %s', args.output_format)
187
+ exit(1)
188
+
189
+ transform(
190
+ args.filename,
191
+ args.outputDir,
192
+ args.input_format,
193
+ args.output_format,
194
+ args.validate,
195
+ args.sourceIdentification,
196
+ args.publicationIdentifier,
197
+ )
@@ -0,0 +1,141 @@
1
+ import json
2
+ from importlib.resources import as_file, files
3
+ from typing import Any
4
+
5
+ from rs_xml2json import convert
6
+
7
+
8
+ def schema_path(schema):
9
+ with as_file(files('datex2_xml2json').joinpath(schema)) as fspath:
10
+ return str(fspath.resolve())
11
+
12
+
13
+ def load_as_json_using_schema(filename: str, schemafile: str) -> dict:
14
+ if filename.endswith('.xml'):
15
+ json_string = convert(filename, schemafile)
16
+ json_input = json.loads(json_string)
17
+ elif filename.endswith('.json'):
18
+ with open(filename) as f:
19
+ json_input = json.load(f)
20
+
21
+ return json_input
22
+
23
+
24
+ def find(a_dict: dict[Any, Any], simplified_xpath: str, default_value: Any = None) -> Any:
25
+ paths = simplified_xpath.split('/')
26
+ elem: Any = a_dict
27
+ for path in paths:
28
+ elem = elem.get(path) if isinstance(elem, dict) else elem
29
+ if elem is None:
30
+ return default_value
31
+
32
+ return elem
33
+
34
+
35
+ def xpath_dict(data, path: str) -> Any:
36
+ """
37
+ Simple XPath-like lookup for nested dict/list structures.
38
+
39
+ Supported:
40
+ - "/" path separator
41
+ """
42
+
43
+ parts = [p for p in path.strip('/').split('/') if p]
44
+
45
+ def _search(current: Any, remaining: list) -> Any:
46
+ if not remaining:
47
+ if isinstance(current, list):
48
+ return current
49
+ else:
50
+ return [current]
51
+
52
+ key = remaining[0]
53
+ rest = remaining[1:]
54
+ results = []
55
+
56
+ if isinstance(current, dict):
57
+ if key in current:
58
+ results.extend(_search(current[key], rest))
59
+
60
+ elif isinstance(current, list):
61
+ try:
62
+ idx = int(key)
63
+ if 0 <= idx < len(current):
64
+ results.extend(_search(current[idx], rest))
65
+ except ValueError:
66
+ for item in current:
67
+ results.extend(_search(item, remaining))
68
+
69
+ return results
70
+
71
+ return _search(data, parts)
72
+
73
+
74
+ def replace(
75
+ value: Any,
76
+ current_key: str | None,
77
+ renamings: dict[str, str],
78
+ value_transforms: dict,
79
+ keys_to_drop: dict[str, list[str]],
80
+ type_mappings: dict[str, str],
81
+ keys_to_pull_up: dict[str, None | list[str]],
82
+ ):
83
+ # renamings: dict of old keys which shall be replaced by new keys
84
+ # keys_to_drop: dict of keys which should be removed from dicts, if their parent key equals keys_to_drop[key]'s value (or value is None)
85
+ # type_mappings: dict of type values which, if present as 'type': type_mapping_key will result in an object {type_mapping_value: {} }
86
+ # keys_to_pull_up: list of keys, which will be replaced by the key/value pairs of its value dict
87
+ if isinstance(value, dict):
88
+ # iterate over keys
89
+ new_dict = {}
90
+ for key in value:
91
+ if (
92
+ value[key] is not None
93
+ and not (key in keys_to_drop and (len(keys_to_drop[key]) == 0 or current_key in keys_to_drop[key]))
94
+ and not (key == 'type' and value['type'] in type_mappings)
95
+ ):
96
+ replaced = replace(
97
+ value[key],
98
+ key,
99
+ renamings,
100
+ value_transforms,
101
+ keys_to_drop,
102
+ type_mappings,
103
+ keys_to_pull_up,
104
+ )
105
+
106
+ new_dict[renamings.get(key, key)] = replaced
107
+
108
+ for parent_key, children_keys in keys_to_pull_up.items():
109
+ if parent_key in new_dict:
110
+ if children_keys and len(children_keys) > 0:
111
+ # copy only intended children
112
+ for child_key in children_keys:
113
+ if child_key in new_dict[parent_key]:
114
+ new_dict[child_key] = new_dict[parent_key].pop(child_key)
115
+ else:
116
+ # copy only intended children
117
+ dict_to_merge = new_dict.pop(parent_key)
118
+ new_dict = new_dict | dict_to_merge
119
+
120
+ if '@xsi:type' in value and value['@xsi:type'] in type_mappings:
121
+ new_dict = {type_mappings[value['@xsi:type']]: new_dict}
122
+
123
+ if current_key is not None and current_key in value_transforms:
124
+ return value_transforms[current_key](new_dict)
125
+ return new_dict
126
+ if isinstance(value, list):
127
+ return [
128
+ replace(
129
+ v,
130
+ current_key,
131
+ renamings,
132
+ value_transforms,
133
+ keys_to_drop,
134
+ type_mappings,
135
+ keys_to_pull_up,
136
+ )
137
+ for v in value
138
+ if v is not None
139
+ ]
140
+ else:
141
+ return value_transforms[current_key](value) if current_key is not None and current_key in value_transforms else value
@@ -0,0 +1,176 @@
1
+ Metadata-Version: 2.4
2
+ Name: datex2-xml2json
3
+ Version: 0.0.1
4
+ Summary: PoC converting DATEX2 v2 (situation) publications to various output formats.
5
+ Keywords: DATEX2,roadworks,road closures,GeoJSON,Geoparquet,WAZE-CIFS
6
+ Author: Holger Bruch
7
+ Author-email: Holger Bruch <hb@mfdz.de>
8
+ License-Expression: EUPL-1.2
9
+ License-File: LICENSE.md
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Topic :: File Formats
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Operating System :: OS Independent
15
+ Requires-Dist: jsonschema>=4.26.0
16
+ Requires-Dist: check-jsonschema>=0.38.2
17
+ Requires-Dist: jsonata-python>=0.7.0
18
+ Requires-Dist: geoparquet-io>=1.6.0
19
+ Requires-Dist: pyarrow>=26.0.0
20
+ Requires-Dist: rs-xml2json>=0.2.0
21
+ Requires-Python: >=3.12
22
+ Project-URL: Homepage, https://github.com/datex2-tools/datex2-xml2json
23
+ Project-URL: Issues, https://github.com/datex2-tools/datex2-xml2json/issues
24
+ Description-Content-Type: text/markdown
25
+
26
+ # DATEX2-XML2JSON (Proof of Concept)
27
+
28
+ This tool provides a PoC converter to convert DATEXII XML to JSON or GeoParquet format.
29
+
30
+ As output format, the following formats are supported
31
+
32
+ * DATEXII v3.7 JSON
33
+ * a custom DATEXII 3.7 (we call it "DATEXII v3.7 JSON Lite") variation which deviates from official schema but is easier to process (see below)
34
+ * a (mostly) flattened GeoJSON
35
+ * a geoparquet format derived from above mostly flat GeoJSON format.
36
+
37
+ Curently, supported DATEXII input is limited to `SituationPublication`s in DATEXII v2.x.
38
+
39
+ Note: This tool uses XML schemas to derive cardinality and type information for the JSON conversion to JSON.
40
+ Custom extensions might require supplying specific schemas to support this conversion.
41
+
42
+ ## Preliminaries
43
+
44
+ This project requires Python >= 3.12 and `uv` installed.
45
+
46
+ Install the needed dependencies via
47
+
48
+ ```
49
+ $ uv sync
50
+ ```
51
+
52
+ ## Usage
53
+
54
+ Below, you find a couple of sample formats to generate different output formats.
55
+
56
+ If you encounter issues, i.e. an invalid DATEXII v3.7, you might consult the [troubleshooting guide](doc/troubleshooting.md).
57
+ If this is does not help, please open an issue and supply a minimal DATEXII XML sample and your command to reproduce the issue.
58
+
59
+ In case you fixed an error or want to contribute, please create an issue as well and feel free to send a PR, thanks.
60
+
61
+ ### DATEXII v3.7
62
+ To convert a DATEXII v2.2 XML publication to DATEXII v3.7, use the following command. It will create `out/situation_publication.json` and will validate it after generation.
63
+
64
+ ```sh
65
+ uv run python -m datex2_xml2json.transformer -f datex2_3_7_json -V situation_publication.xml
66
+ ```
67
+
68
+ ### DATEXII (mostly) flat GeoJSON
69
+
70
+ To convert a DATEXII v2.2 XML situation publication to a (mostly) flat GeoJSON file, use the following command. It will create `out/situation_publication.geojson`. Note that this is a lossy conversion which retains a subset of all possible DATEXII properties. For details which elements are picked, see `jsonata_transformer.py`.
71
+
72
+ ```sh
73
+ uv run python -m datex2_xml2json.transformer -f geojson situation_publication.xml
74
+ ```
75
+
76
+ ### DATEXII Geoparquet
77
+
78
+ To convert a DATEXII v2.2 XML publication to a Geoparquet file, use the following command. It will create out/situation_publication.geoparquet.
79
+
80
+
81
+ Note that all timestamp properties are exported as UTC timestamps, their original offsets applied.
82
+
83
+ ```sh
84
+ uv run python -m datex2_xml2json.transformer -f geoparquet situation_publication.xml
85
+ ```
86
+
87
+ To merge multiple geoparquet files with identical schema into a common one, you may use
88
+
89
+ ```sh
90
+ uv run python -m datex2_xml2json.merge <directory containing parquet files> -o merged.parquet
91
+ ```
92
+
93
+ ### DATEXII v3.7 JSON Lite
94
+
95
+ To convert a DATEXII v2.2 XML publication to a DATEXII v3.7 JSON "Lite" format, use the following command. It will create `out/situation_publication.lite.json`:
96
+
97
+ ```sh
98
+ uv run python -m datex2_xml2json.transformer -f datex2_3_7_json_lite situation_publication.xml
99
+ ```
100
+
101
+ Warning: the JSON resulting from this conversion does not comply with DATEXII JSON Schema.
102
+
103
+ Some differences:
104
+ * this format does not use the DATEXII part 15 xsi-type dependant wrapper objects, but keeps the `xsi:type` attribute as property `type`.
105
+ * it does not wrap text objects in `value` wrappers
106
+ * we try to create a `geoJsonGeometry` location from `gmlLineString` locations. For `locationContainedInGroup`, we collect all linestrings in an `MultiLineString` object
107
+
108
+ ### DATEXII v2.2 RAW JSON
109
+
110
+ To convert a DATEXII v2.2 XML publication to an intermediate JSON which just converts the XML to a JSON file using the DATEXII v2.2 schema, use the following command. It will create `out/situation_publication.raw.json`.
111
+ ```sh
112
+ uv run python -m datex2_xml2json.transformer -f raw situation_publication.xml
113
+ ```
114
+
115
+ ### WAZE CIFS Format
116
+
117
+ To convert a DATEXII v2.2 XML situation publication to [WAZE CIFS Format](https://developers.google.com/waze/data-feed/cifs-specification?hl=de), use the following command. It will create `out/situation_publication.cifs.json`.
118
+
119
+ ```sh
120
+ uv run python -m datex2_xml2json.transformer -f cifs situation_publication.xml
121
+ ```
122
+
123
+ To validate the generated json against the CIFS schema, supply the -v switch.
124
+ ```sh
125
+ uv run python -m datex2_xml2json.transformer -f cifs -V situation_publication.xml
126
+ ```
127
+
128
+ Note: currently, this converter does not handle validityPeriods. I.E. recurring events at specific days/hours are not yet mapped to CIFS schedule.
129
+
130
+
131
+ ### Generating multiple output formats in one run
132
+
133
+ To convert a DATEXII v2.2 XML situation publication to WAZE CIFS Format, GeoJSON and Geoparquet in one run (which avoids repeated parsing/transformation steps), run the following command (without explicitly specifing a format parameter):
134
+ It will create `out/situation_publication.datex2.raw.json`, `out/situation_publication.datex2.lite.json`, `out/situation_publication.datex2.geojson`, `out/situation_publication.datex2.parquet` and`out/situation_publication.cifs.json`.
135
+
136
+ ```sh
137
+ uv run python -m datex2_xml2json.transformer situation_publication.xml
138
+ ```
139
+
140
+ ## Development
141
+
142
+ This section provides information how to develop with this lib.
143
+
144
+ ### How the transformation works
145
+
146
+ This converter runs multiple conversion steps which create intermediate formats which are used as homogenized input for subsequent transformation steps.
147
+
148
+ The following chart describes this conversion chain:
149
+
150
+ ```mermaid
151
+ flowchart TD
152
+ DATEX2v2.2XML -- "load_as_json_using_schema" --> DATEX2v2.2JSON
153
+ DATEX2v2.2JSON -- RulesTransformer\nwith LiteJSON rules --> DATEX2v3.7LiteJSON
154
+ DATEX2v2.2JSON -- RulesTransformer\nwith v3.7 JSON rules --> DATEX2v3.7JSON
155
+ DATEX2MostlyFlatGeoJSON -- CifsTransformer --> WazeCIFS
156
+ DATEX2v3.7LiteJSON -- JsonataTransformer\nGeoJSONGeometryTransformer --> DATEX2MostlyFlatGeoJSON
157
+ DATEX2MostlyFlatGeoJSON -- GeoparquetRenderer --> DATEX2GeoParquet
158
+ ```
159
+
160
+ ### Working with JSONata
161
+ [JSONata](https://jsonata.org/) is a powerful transformation language to define mappings between different JSON formats.
162
+ Easiest way to learn about it, is starting with an existing transformation using the [JSONata.org online editor](https://try.jsonata.org/).
163
+
164
+ E.g. you could start with the following to example transformations (which might not exactly reflect the latest code version of this project):
165
+
166
+ * https://try.jsonata.org/Gk5r-kLek: Mapping DATEX3.7LiteJSON to DATEX2MostlyFlatGeoJSON (Current JSONata expression is in datex2_xml2json/transform/jsonata_transformer.py)
167
+ * https://try.jsonata.org/QEnqmItyF: Mapping DATEX2MostlyFlatGeoJSON to (mostly) WAZE CIFS (Current JSONata expression is in datex2_xml2json/transform/cifs_transformer.py)
168
+
169
+ ### Running tests
170
+
171
+ You can execute the test suite via
172
+
173
+ ```sh
174
+ uv sync --group dev
175
+ uv run pytest
176
+ ```
@@ -0,0 +1,65 @@
1
+ datex2_xml2json/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ datex2_xml2json/json_validator.py,sha256=6xR3dvICX-b3gkJaxd6LR4Xbq_TrmrXpIPOB85StACw,1365
3
+ datex2_xml2json/merge.py,sha256=nnZpsf-0DWwwjdK5KeKELMZA1aJdAYmxoTv6y7GzgWQ,996
4
+ datex2_xml2json/schema/Baustellen-4-00-00.sch,sha256=_w7kxp_uL4__NsCoerba1-VBKB-ks8llC-1TlfONNeQ,227
5
+ datex2_xml2json/schema/DATEXIISchema_2_2_0.xsd,sha256=p5Hjh_AXFYKiSHy8AU46enrSYZY-Wq6rxTg0KagfwFM,249642
6
+ datex2_xml2json/schema/DATEXIISchema_2_2_3.xsd,sha256=4Ewvziqm00NvJYpKff8Y4j4_IiPufxe2vOXrq84eONg,368221
7
+ datex2_xml2json/schema/DATEXII_3.7_Profile/.DS_Store,sha256=w7FLIBjcAn936A1gGCPxY_I9PUCFJGKK5KTY82UMGFc,6148
8
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_AfirEnergyInfrastructure.json,sha256=oQc52I-svUBEqSMpsiEQMrpyDcDLfJYZERtOxHgiePc,87791
9
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_AfirFacilities.json,sha256=2upMburkdFSZLTAafp7La90hDzv2bdXfU17-zjyMhGY,105040
10
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Common.json,sha256=gUek4uFBdFCh7Y5Fen0gLD6wj1C8QgxG6tXT6NVoj1A,131046
11
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_CommonExtension.json,sha256=YTeiqTOOeaXaO4elaYoZY8hDef2-SZD5ZDKBSuAB47M,31319
12
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_ControlledZone.json,sha256=QoOS6IDzoQBRDgVvH84UFsK5RLo5oHhbZT4bQd2Dzo4,39129
13
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_D2Payload.json,sha256=UO-fTNZD5_Ozg8sCyC-p3-LJhr0Y8_-KoF5lTv4j_1U,252
14
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_EnergyInfrastructure.json,sha256=93q67MuaRq7Ann-dBRf-ciB1Mf9FVdlRa9vvtLG_8lI,67802
15
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Facilities.json,sha256=4MOZ9rziyBp9Hk99cLCcMSgkO4AUoNn99JjVO4aoCGc,104538
16
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_FaultAndStatus.json,sha256=UMhJJQxJ2pbCdaLClOdKj0SK_30MWvZ4hG3LMy1b3Wk,50998
17
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_LocationExtension.json,sha256=_SRMX6xKu2EIRcaZgeekFmF3FJ4VFdKfbnGWymooIPk,6001
18
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_LocationReferencing.json,sha256=kEsbdSfDUW6WbBP9RCdAkmjUuiob_pz151oJoQCu4_w,189936
19
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Parking.json,sha256=3qbJ7E19_CjF-CA-J8s-ih2DLMiGYHSjAT7c0jrLYeg,194848
20
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_ReroutingManagementEnhanced.json,sha256=VutLPlHRbosXVeTSlJoruhPiZ-hKd2NoXR33GaCLYWs,44002
21
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_RoadTrafficData.json,sha256=arbtCV1ww4vWPNuC-RFYzOy7s78PliAfVEMOxa8Mm_o,84628
22
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Situation.json,sha256=6HkdIxNrMuuVpVTXR09SHerJuPGwTZeG0H39IKMO1TQ,347235
23
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_TrafficManagementPlan.json,sha256=3CG1IRodTsmV-RHSraD0MugQ1jQPVJJn-ZBifdx9seg,96332
24
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_TrafficRegulation.json,sha256=du5wNP-pkQAKgcLW8z0Cb3ivq-kuVX6J2rGiN__GQ7g,125051
25
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_UrbanExtensions.json,sha256=wC-vy-gMtK8fccEBWXUvrTnT9e1i4xb05l7h7OFNtag,6466
26
+ datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Vms.json,sha256=o-DpytTZbdLPdVfczKN2YnDxjWUJf4oT3QDdOcI_Va0,97980
27
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/.DS_Store,sha256=1lFlJ5EFymdzGAUAaI30vcaaLHt3F1LwpG7xILf9jsM,6148
28
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_AfirEnergyInfrastructure.xsd,sha256=hsdLlbqB1rHRsUa1H-xHGIYBa3zbbZZCr10mNqvCa7w,99840
29
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_AfirFacilities.xsd,sha256=Ct4T_iQR1WRWmFmWD4b9HQ3-6CeGLKEI5fvV8g-BJ54,101216
30
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Common.xsd,sha256=w3yBy6kcT3lYAKHYIy5l4SW7rjUPlxZ71w4285dHKSU,193032
31
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_CommonExtension.xsd,sha256=e7FKDvxXU0cNVoG89KZfBklwEGuGhsqeYcze62dQIyQ,52958
32
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_ControlledZone.xsd,sha256=odLmgZ1SHcO89ChM3t7fRmFc-aajFcvopHbfOW0hGM8,31070
33
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_D2Payload.xsd,sha256=7mhHnLfDvJu_Pv9rdtmr-HZusBYMcOb7lmiIlTQvbLQ,14037
34
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_EnergyInfrastructure.xsd,sha256=Ya1f9lG6XA3giQ8wce_5mY3Gdd6R-klZkDtOnZ2bMf0,81074
35
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Facilities.xsd,sha256=On13G55ZKWDiKhEu32Og3yQUXPkjx03dmWrJGUakuBk,98554
36
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_FaultAndStatus.xsd,sha256=WtCm5hOSuTWT7XWsLNQlekImo8ylxj5DIVp8s3NdDY4,52194
37
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_LocationExtension.xsd,sha256=fiz2zrCfU4Pi7n0iGzdaTspsrQ0sipDIYpsjuvXeHzo,9446
38
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_LocationReferencing.xsd,sha256=mlPZhpf4jiyza_8E8yu9RehGyj_ox7hEYBLpQYUbKmA,200205
39
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Parking.xsd,sha256=xvbHNPuugsdV11GQ9zBNeefihaiUQevAlhTxATvhyYo,200693
40
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_ReroutingManagementEnhanced.xsd,sha256=hwR9AQkwW-F5SbnVgdP9paSVzq-2EHXlvifTQat2G24,50299
41
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_RoadTrafficData.xsd,sha256=j7-2qWeV_q_1Egz-XljWMpaHPSzVpCeJ57vL1rmk0X4,69361
42
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Situation.xsd,sha256=63YKUDOBdHs4eSaBXTG9yS8G3mPNhjX20HN7a9JgoIo,280088
43
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_TrafficManagementPlan.xsd,sha256=yD1gMJjeMr7Aw4VV3LDEdZx0uQX3Q1YdZDSuycE_8K4,66049
44
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_TrafficRegulation.xsd,sha256=B0xpT9uiLrQXDkmy16xm1Rbm1kEbq7_zst5PTV-w4UQ,121452
45
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_UrbanExtensions.xsd,sha256=xNo8EveV9vnHhxBheXPY-XmiidgstB87sYSGAs6eBxg,7186
46
+ datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Vms.xsd,sha256=0ynJLQlxqYBOZ-mluVgrBunJKLf6U3QcJcFm8ye0iLE,114815
47
+ datex2_xml2json/schema/MDM_BaustellenInfoDigitalRoadworksProfile_04-03-00.xsd,sha256=EjhEoiqqWkEDqn0tkJUWt-xqWPHuWZvMTKYROWFMb1Y,119763
48
+ datex2_xml2json/schema/MDM_Datenmodell_Baustellen_04-00-00.xsd,sha256=es6vH4hAK59ryj6JNHtPcZmYNXgLPbwKA2nIrB0-qz0,158614
49
+ datex2_xml2json/schema/cifs_schema.json,sha256=QowoY-X45MCLrAubhoFH_aaUJUlZR4eIl4wW8yn4ttg,5709
50
+ datex2_xml2json/transform/.DS_Store,sha256=1lFlJ5EFymdzGAUAaI30vcaaLHt3F1LwpG7xILf9jsM,6148
51
+ datex2_xml2json/transform/__init__.py,sha256=CE_IqrrYmLGYG-b4zZsT8KpEGBvy47Wtd23bLqG7qBc,630
52
+ datex2_xml2json/transform/base.py,sha256=pi0_7YZMPI6FXcqRaBHr9cbcZzgwRq5DUQEEux-gIB8,2434
53
+ datex2_xml2json/transform/cifs_transformer.py,sha256=pDo64lIBnPvfuaHOiFlDINUsrcv-TQx9VDpTNyEpHpU,14081
54
+ datex2_xml2json/transform/geojson_geometry_transformer.py,sha256=IQPjeTiB4IaYYJGPtX0DIW4JErWrt9NnnMFZuSlF6uA,7002
55
+ datex2_xml2json/transform/geoparquet_transformer.py,sha256=2EpUyJvljwEeTQuUkFcwOmLSgppeRIqNPFySBKr6g_I,6634
56
+ datex2_xml2json/transform/jsonata_transformer.py,sha256=TMwHqUX1JGF7M3fwYp69F36TLK7BuXlyQY4TKmOuGyU,6722
57
+ datex2_xml2json/transform/roads_regio_transformer.py,sha256=BRb71qo2cIxnDla9iw2TzLrBGAaD9iG9Z-gEI8Ax3J8,4024
58
+ datex2_xml2json/transform/source_setter.py,sha256=fQ3-foJkPbrqGCBwExy6QeUV5vAiguKEjuIn_USzgEs,1605
59
+ datex2_xml2json/transform/transformation_rules.py,sha256=HDqrsdpDvU9NMRFpYc8zaVITlUEE3pxf_JvHorDCqKY,23942
60
+ datex2_xml2json/transformer.py,sha256=JW42Y4d6d3Hdeqvsd5VgRIcYHM20nVKjkjuyr8Z2vuQ,6638
61
+ datex2_xml2json/util.py,sha256=XMyZtBJ_4VZgqfZ4G14P7JSvsif8_mvZ5-4ykxBZrS4,4731
62
+ datex2_xml2json-0.0.1.dist-info/licenses/LICENSE.md,sha256=FczCBpjZMhSuHGbPoM2COX2aG2sNvGv8etitdTs9mW8,13818
63
+ datex2_xml2json-0.0.1.dist-info/WHEEL,sha256=si2p8vW03RnhyBnhWl4tj-ObJa78SpE-c9HsEtblGR0,80
64
+ datex2_xml2json-0.0.1.dist-info/METADATA,sha256=G-QYBj-_Chy6hW_otYJwFpfZeez9uxAxFAd5CL__C68,7670
65
+ datex2_xml2json-0.0.1.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: uv 0.13.5
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any