datex2-xml2json 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datex2_xml2json/__init__.py +0 -0
- datex2_xml2json/json_validator.py +36 -0
- datex2_xml2json/merge.py +39 -0
- datex2_xml2json/schema/Baustellen-4-00-00.sch +7 -0
- datex2_xml2json/schema/DATEXIISchema_2_2_0.xsd +4707 -0
- datex2_xml2json/schema/DATEXIISchema_2_2_3.xsd +6783 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/.DS_Store +0 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_AfirEnergyInfrastructure.json +2455 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_AfirFacilities.json +2809 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Common.json +3651 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_CommonExtension.json +919 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_ControlledZone.json +1070 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_D2Payload.json +11 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_EnergyInfrastructure.json +1924 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Facilities.json +2858 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_FaultAndStatus.json +1415 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_LocationExtension.json +186 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_LocationReferencing.json +4847 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Parking.json +5314 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_ReroutingManagementEnhanced.json +1160 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_RoadTrafficData.json +1938 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Situation.json +8040 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_TrafficManagementPlan.json +2358 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_TrafficRegulation.json +3377 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_UrbanExtensions.json +194 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/DATEXII_3_Vms.json +2544 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/.DS_Store +0 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_AfirEnergyInfrastructure.xsd +2102 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_AfirFacilities.xsd +2060 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Common.xsd +4152 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_CommonExtension.xsd +1102 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_ControlledZone.xsd +601 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_D2Payload.xsd +280 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_EnergyInfrastructure.xsd +1761 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Facilities.xsd +2037 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_FaultAndStatus.xsd +1076 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_LocationExtension.xsd +196 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_LocationReferencing.xsd +4132 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Parking.xsd +4025 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_ReroutingManagementEnhanced.xsd +997 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_RoadTrafficData.xsd +1273 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Situation.xsd +5777 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_TrafficManagementPlan.xsd +1184 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_TrafficRegulation.xsd +2611 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_UrbanExtensions.xsd +148 -0
- datex2_xml2json/schema/DATEXII_3.7_Profile/xml/DATEXII_3_Vms.xsd +2290 -0
- datex2_xml2json/schema/MDM_BaustellenInfoDigitalRoadworksProfile_04-03-00.xsd +2401 -0
- datex2_xml2json/schema/MDM_Datenmodell_Baustellen_04-00-00.xsd +3232 -0
- datex2_xml2json/schema/cifs_schema.json +161 -0
- datex2_xml2json/transform/.DS_Store +0 -0
- datex2_xml2json/transform/__init__.py +13 -0
- datex2_xml2json/transform/base.py +79 -0
- datex2_xml2json/transform/cifs_transformer.py +275 -0
- datex2_xml2json/transform/geojson_geometry_transformer.py +159 -0
- datex2_xml2json/transform/geoparquet_transformer.py +160 -0
- datex2_xml2json/transform/jsonata_transformer.py +100 -0
- datex2_xml2json/transform/roads_regio_transformer.py +88 -0
- datex2_xml2json/transform/source_setter.py +42 -0
- datex2_xml2json/transform/transformation_rules.py +526 -0
- datex2_xml2json/transformer.py +197 -0
- datex2_xml2json/util.py +141 -0
- datex2_xml2json-0.0.1.dist-info/METADATA +176 -0
- datex2_xml2json-0.0.1.dist-info/RECORD +65 -0
- datex2_xml2json-0.0.1.dist-info/WHEEL +4 -0
- datex2_xml2json-0.0.1.dist-info/licenses/LICENSE.md +179 -0
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
from .base import SituationRecordTransformer
|
|
4
|
+
|
|
5
|
+
logger = logging.getLogger(__name__)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class GeoJSONGeometryTransformer(SituationRecordTransformer):
|
|
9
|
+
def __init__(
|
|
10
|
+
self,
|
|
11
|
+
situation_record_json_pointer: str = 'payload/sitSituationPublication/situation/situationRecord',
|
|
12
|
+
geom_container_element_json_point: str | None = None,
|
|
13
|
+
):
|
|
14
|
+
super().__init__(situation_record_json_pointer)
|
|
15
|
+
self.geom_container_element_json_point = geom_container_element_json_point
|
|
16
|
+
|
|
17
|
+
def _extract_coordinates_from_gmlLineString(self, gmlLineString: dict) -> list:
|
|
18
|
+
# Coordinate Order Confusion
|
|
19
|
+
# There is often some confusion regarding the correct ordering of coordinate values.
|
|
20
|
+
# While EPSG:4326 usually uses lat,lon ordering, and CRS84 (and GeoJSON) use lon/lat,
|
|
21
|
+
# older GIS software might use lon/lat also for EPSG:4326.
|
|
22
|
+
#
|
|
23
|
+
# Assuming lat,lon in GML's posList when "WGS84 EPSG 4326" is set as srsName,
|
|
24
|
+
# we swap coordinates for GeoJSON.
|
|
25
|
+
is_lat_lon_order = False
|
|
26
|
+
srsName = gmlLineString.get('srsName')
|
|
27
|
+
if srsName is not None:
|
|
28
|
+
if srsName not in {'WGS84 EPSG 4326', 'WGS84'}:
|
|
29
|
+
logger.warning('Unhandled srsName %s, might need reprojection/ latlon order handling.', srsName)
|
|
30
|
+
else:
|
|
31
|
+
logger.info('Detected srsName "%s". We assume latlon gmlLineString coordinates ordering for these.', srsName)
|
|
32
|
+
is_lat_lon_order = True
|
|
33
|
+
posList = gmlLineString.get('posList', '')
|
|
34
|
+
|
|
35
|
+
zipped_coords = zip(*[iter(map(float, posList.split()))] * 2)
|
|
36
|
+
if is_lat_lon_order:
|
|
37
|
+
return [[lon, lat] for lat, lon in zipped_coords]
|
|
38
|
+
else:
|
|
39
|
+
return [[lon, lat] for lon, lat in zipped_coords]
|
|
40
|
+
|
|
41
|
+
def _extract_line_geom(self, linearExtended: dict) -> None | dict:
|
|
42
|
+
gmlLineString = linearExtended.get('gmlLineString')
|
|
43
|
+
if gmlLineString is None:
|
|
44
|
+
logger.warning('No linearExtended/gmlLineString')
|
|
45
|
+
return None
|
|
46
|
+
else:
|
|
47
|
+
geometry = {
|
|
48
|
+
'type': 'LineString',
|
|
49
|
+
'coordinates': self._extract_coordinates_from_gmlLineString(gmlLineString),
|
|
50
|
+
}
|
|
51
|
+
return geometry
|
|
52
|
+
|
|
53
|
+
def _extract_point_geom(self, pointCoordinates: dict | None) -> dict | None:
|
|
54
|
+
if pointCoordinates is None:
|
|
55
|
+
logger.warning('No pointCoordinates')
|
|
56
|
+
return None
|
|
57
|
+
else:
|
|
58
|
+
geometry = {
|
|
59
|
+
'type': 'Point',
|
|
60
|
+
'coordinates': [
|
|
61
|
+
pointCoordinates.get('longitude'),
|
|
62
|
+
pointCoordinates.get('latitude'),
|
|
63
|
+
],
|
|
64
|
+
}
|
|
65
|
+
return geometry
|
|
66
|
+
|
|
67
|
+
def _extract_multi_line_string(self, locationReference: dict) -> dict:
|
|
68
|
+
locations = locationReference.get('locationContainedInGroup', {})
|
|
69
|
+
|
|
70
|
+
lineStrings = []
|
|
71
|
+
for location in locations:
|
|
72
|
+
location_type = location.get('type')
|
|
73
|
+
if not location_type == 'Linear':
|
|
74
|
+
logger.warning(
|
|
75
|
+
'Unexpected location type %s, location is ignored',
|
|
76
|
+
location.get('type'),
|
|
77
|
+
)
|
|
78
|
+
continue
|
|
79
|
+
coordinates = self._extract_coordinates_from_gmlLineString(location.get('gmlLineString', {}))
|
|
80
|
+
if coordinates:
|
|
81
|
+
lineStrings.append(coordinates)
|
|
82
|
+
|
|
83
|
+
return {'type': 'MultiLineString', 'coordinates': lineStrings}
|
|
84
|
+
|
|
85
|
+
def _extract_extendedLinear_geom(self, locationReference: dict) -> dict:
|
|
86
|
+
"""
|
|
87
|
+
Some publications use a custom extension extendedLinear to provide linear geometries using a verbose
|
|
88
|
+
xml encoding.
|
|
89
|
+
We construct geojson geometries from these.
|
|
90
|
+
Note: as this extension is not defined in standard XML schemas, we need to handle some type and cardinality issues,
|
|
91
|
+
e.g. only a single intermediate point vs a list and lat/lon not beeing converted to floats during xml2json conversion.
|
|
92
|
+
"""
|
|
93
|
+
linearByCoordinates = locationReference.get('extendedLinear', {}).get('linearByCoordinates', {})
|
|
94
|
+
coordinates = []
|
|
95
|
+
coordinates.append(
|
|
96
|
+
[
|
|
97
|
+
float(linearByCoordinates.get('start').get('longitude')),
|
|
98
|
+
float(linearByCoordinates.get('start').get('latitude')),
|
|
99
|
+
]
|
|
100
|
+
)
|
|
101
|
+
intermediates = (
|
|
102
|
+
[linearByCoordinates.get('intermediate')]
|
|
103
|
+
if isinstance(linearByCoordinates.get('intermediate'), dict)
|
|
104
|
+
else linearByCoordinates.get('intermediate', [])
|
|
105
|
+
)
|
|
106
|
+
for intermediate in intermediates:
|
|
107
|
+
coordinates.append(
|
|
108
|
+
[
|
|
109
|
+
float(intermediate.get('pointCoordinates').get('longitude')),
|
|
110
|
+
float(intermediate.get('pointCoordinates').get('latitude')),
|
|
111
|
+
]
|
|
112
|
+
)
|
|
113
|
+
coordinates.append(
|
|
114
|
+
[
|
|
115
|
+
float(linearByCoordinates.get('end').get('longitude')),
|
|
116
|
+
float(linearByCoordinates.get('end').get('latitude')),
|
|
117
|
+
]
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
return {'type': 'LineString', 'coordinates': coordinates}
|
|
121
|
+
|
|
122
|
+
def process_situation_record(self, situation_record: dict) -> None:
|
|
123
|
+
locationReference = situation_record.get('locationReference')
|
|
124
|
+
if locationReference is None:
|
|
125
|
+
return
|
|
126
|
+
|
|
127
|
+
geom_container_element = (
|
|
128
|
+
locationReference.get(self.geom_container_element_json_point)
|
|
129
|
+
if self.geom_container_element_json_point is not None
|
|
130
|
+
else locationReference
|
|
131
|
+
)
|
|
132
|
+
if isinstance(geom_container_element, dict):
|
|
133
|
+
geoJsonGeometry = None
|
|
134
|
+
if 'gmlLineString' in geom_container_element:
|
|
135
|
+
geoJsonGeometry = self._extract_line_geom(geom_container_element)
|
|
136
|
+
elif 'pointByCoordinates' in geom_container_element:
|
|
137
|
+
geoJsonGeometry = self._extract_point_geom(geom_container_element.get('pointByCoordinates', {}).get('pointCoordinates'))
|
|
138
|
+
elif 'locationContainedInGroup' in geom_container_element:
|
|
139
|
+
geoJsonGeometry = self._extract_multi_line_string(geom_container_element)
|
|
140
|
+
elif 'extendedLinear' in geom_container_element:
|
|
141
|
+
geoJsonGeometry = self._extract_extendedLinear_geom(geom_container_element)
|
|
142
|
+
elif 'coordinatesForDisplay' in geom_container_element:
|
|
143
|
+
geoJsonGeometry = self._extract_point_geom(geom_container_element.get('coordinatesForDisplay'))
|
|
144
|
+
|
|
145
|
+
if geoJsonGeometry is not None:
|
|
146
|
+
geom_container_element['geoJsonGeometry'] = geoJsonGeometry
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class GeoJSONGeometryTransformer_DATEXII_3_7(GeoJSONGeometryTransformer):
|
|
150
|
+
def __init__(self):
|
|
151
|
+
super().__init__(
|
|
152
|
+
'payload/sitSituationPublication/situation/situationRecord',
|
|
153
|
+
'linearExtended',
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class GeoJSONGeometryTransformer_Simplified(GeoJSONGeometryTransformer):
|
|
158
|
+
def __init__(self):
|
|
159
|
+
super().__init__('payload/situation/situationRecord', None)
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from datetime import datetime, timezone
|
|
4
|
+
from tempfile import NamedTemporaryFile
|
|
5
|
+
|
|
6
|
+
import geoparquet_io as gpio
|
|
7
|
+
import pyarrow as pa
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
fields: list[tuple[str, pa.DataType]] = [
|
|
12
|
+
('publicationTime', pa.timestamp('ms', tz='UTC')),
|
|
13
|
+
('publicationCreator', pa.string()),
|
|
14
|
+
('publicationIdentifier', pa.string()),
|
|
15
|
+
('sourceIdentification', pa.string()),
|
|
16
|
+
('situationId', pa.string()),
|
|
17
|
+
('situationVersionTime', pa.timestamp('ms', tz='UTC')),
|
|
18
|
+
('overallSeverity', pa.string()),
|
|
19
|
+
('confidentiality', pa.string()),
|
|
20
|
+
('informationStatus', pa.string()),
|
|
21
|
+
('id', pa.string()),
|
|
22
|
+
('version', pa.string()),
|
|
23
|
+
('situationRecordCreationTime', pa.timestamp('ms', tz='UTC')),
|
|
24
|
+
('situationRecordVersionTime', pa.timestamp('ms', tz='UTC')),
|
|
25
|
+
('overallStartTime', pa.timestamp('ms', tz='UTC')),
|
|
26
|
+
('overallEndTime', pa.timestamp('ms', tz='UTC')),
|
|
27
|
+
('probabilityOfOccurrence', pa.string()),
|
|
28
|
+
('type', pa.string()),
|
|
29
|
+
('complianceOption', pa.string()),
|
|
30
|
+
('validityStatus', pa.string()),
|
|
31
|
+
('severity', pa.string()),
|
|
32
|
+
('description', pa.string()),
|
|
33
|
+
('roadworksType', pa.string()),
|
|
34
|
+
('roadworksName', pa.string()),
|
|
35
|
+
('heavyDutyTransportRestriction', pa.string()),
|
|
36
|
+
('routeRecommendation', pa.string()),
|
|
37
|
+
('applicableForTrafficDirection', pa.string()),
|
|
38
|
+
('placesAtWhichApplicable', pa.string()),
|
|
39
|
+
('cause', pa.string()),
|
|
40
|
+
('numberOfLanesRestricted', pa.int8()),
|
|
41
|
+
('numberOfOperationalLanes', pa.int8()),
|
|
42
|
+
('residualRoadWidth', pa.float32()),
|
|
43
|
+
('trafficConstrictionType', pa.string()),
|
|
44
|
+
('situationRecordSubtype', pa.string()),
|
|
45
|
+
('actionPlanIdentifier', pa.string()),
|
|
46
|
+
('operatorActionStatus', pa.string()),
|
|
47
|
+
('laneRestriction', pa.string()), # TODO should be json
|
|
48
|
+
('laneStatusCoded', pa.string()),
|
|
49
|
+
('extendedLaneStatusCoded', pa.string()),
|
|
50
|
+
('hardShoulderStatus', pa.string()),
|
|
51
|
+
('residualHeight', pa.float64()),
|
|
52
|
+
('maxVehicleWeight', pa.float64()), # TODO should only be set if valid for allCariageWays
|
|
53
|
+
('capacityRemaining', pa.int32()),
|
|
54
|
+
('originalNumberOfLanes', pa.int32()),
|
|
55
|
+
('numberOfObstructions', pa.int32()),
|
|
56
|
+
('forVehiclesWithCharacteristicsOf', pa.string()), # TODO should be json
|
|
57
|
+
('validPeriods', pa.string()), # TODO should be json
|
|
58
|
+
('subjectTypeOfWorks', pa.string()),
|
|
59
|
+
('applicableForTrafficType', pa.string()),
|
|
60
|
+
('roadworksDurationClassification', pa.string()),
|
|
61
|
+
('mobilityType', pa.string()),
|
|
62
|
+
('roadworksLayout', pa.string()),
|
|
63
|
+
('urlLinkAddress', pa.string()),
|
|
64
|
+
('roadNumber', pa.string()),
|
|
65
|
+
('roadName', pa.string()),
|
|
66
|
+
('fromASBSegmentStation', pa.float64()),
|
|
67
|
+
('fromASBSegmentFromNode', pa.string()),
|
|
68
|
+
('fromASBSegmentTowardsNode', pa.string()),
|
|
69
|
+
('toASBSegmentStation', pa.float64()),
|
|
70
|
+
('toASBSegmentFromNode', pa.string()),
|
|
71
|
+
('toASBSegmentTowardsNode', pa.string()),
|
|
72
|
+
('lengthAffected', pa.float64()),
|
|
73
|
+
('temporarySpeedLimit', pa.float32()),
|
|
74
|
+
('geometry', pa.binary()),
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
MOSTLY_FLAT_SITUATION_RECORD_TARGET_SCHEMA = pa.schema(fields)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class GeoparquetRenderer:
|
|
81
|
+
"""
|
|
82
|
+
GeoparquetRenderer renders a GeoJSON file as a geoparquet file.
|
|
83
|
+
Only properties matchting the supplied target_schema (default ist OSTLY_FLAT_SITUATION_RECORD_TARGET_SCHEMA)
|
|
84
|
+
are exported.
|
|
85
|
+
|
|
86
|
+
Note: As a workaround for https://github.com/geoparquet/geoparquet-io/issues/1152, all timestamp properties
|
|
87
|
+
(according to the supplied target_schema) are converted to UTC strings. Note also, that this modification
|
|
88
|
+
happend in situ, no deep copy of the GeoJSON dict is created.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
def __init__(self, target_schema=MOSTLY_FLAT_SITUATION_RECORD_TARGET_SCHEMA):
|
|
92
|
+
self.target_schema = target_schema
|
|
93
|
+
|
|
94
|
+
def _get_timestamp_fieldnames(self):
|
|
95
|
+
"""
|
|
96
|
+
Return a list of all fieldnames of self.target_schema, whose type is pyarrow.TimestampType.
|
|
97
|
+
"""
|
|
98
|
+
fieldnames = []
|
|
99
|
+
for index, field in enumerate(self.target_schema.types):
|
|
100
|
+
if isinstance(field, pa.TimestampType):
|
|
101
|
+
fieldnames.append(self.target_schema.names[index])
|
|
102
|
+
return fieldnames
|
|
103
|
+
|
|
104
|
+
def _set_timestamps_to_UTC(self, a_geojson_dict: dict):
|
|
105
|
+
"""
|
|
106
|
+
Adds/subtracts optional timezone offset for
|
|
107
|
+
all timestamp properties to assure they are in UTC.
|
|
108
|
+
Note: timestamps without timezone adjustment are assumed to be
|
|
109
|
+
in UTC already.
|
|
110
|
+
This works around https://github.com/geoparquet/geoparquet-io/issues/1152
|
|
111
|
+
and might be obsolete once the it's fix is released in a geoparquet-io
|
|
112
|
+
version > 1.6.0
|
|
113
|
+
"""
|
|
114
|
+
ts_fieldnames = self._get_timestamp_fieldnames()
|
|
115
|
+
for feature in a_geojson_dict['features']:
|
|
116
|
+
props = feature['properties']
|
|
117
|
+
for ts_property in ts_fieldnames:
|
|
118
|
+
if ts_property in props:
|
|
119
|
+
dt = datetime.fromisoformat(props[ts_property])
|
|
120
|
+
# Assume datetime was given in UTC if no timezone was specified
|
|
121
|
+
if not dt.tzinfo:
|
|
122
|
+
dt.replace(tzinfo=timezone.utc)
|
|
123
|
+
props[ts_property] = dt.astimezone(timezone.utc).isoformat()
|
|
124
|
+
|
|
125
|
+
def _cast_fields_and_add_missing(self, table) -> None:
|
|
126
|
+
"""
|
|
127
|
+
Replaces table's pyarrow table by a new one which
|
|
128
|
+
has expected columns in expected order and type.
|
|
129
|
+
"""
|
|
130
|
+
arrays = []
|
|
131
|
+
patable = table._table
|
|
132
|
+
for field in self.target_schema:
|
|
133
|
+
try:
|
|
134
|
+
if field.name in patable.column_names:
|
|
135
|
+
# Cast existing column
|
|
136
|
+
arrays.append(patable[field.name].cast(field.type))
|
|
137
|
+
else:
|
|
138
|
+
# Create null column of the correct type
|
|
139
|
+
arrays.append(pa.nulls(len(patable), type=field.type))
|
|
140
|
+
except Exception:
|
|
141
|
+
logger.exception(f'Exception handling field {field}')
|
|
142
|
+
|
|
143
|
+
patable = pa.Table.from_arrays(arrays, schema=self.target_schema)
|
|
144
|
+
table._table = patable
|
|
145
|
+
|
|
146
|
+
def _as_table(self, a_geojson_dict: dict):
|
|
147
|
+
"""
|
|
148
|
+
Writes a_geojson_dict to a tempory file which is then read
|
|
149
|
+
via geoparquet-io as pyarrow table.
|
|
150
|
+
"""
|
|
151
|
+
with NamedTemporaryFile('w', delete_on_close=False) as f:
|
|
152
|
+
json.dump(a_geojson_dict, f, indent=2)
|
|
153
|
+
f.close()
|
|
154
|
+
return gpio.convert(f.name)
|
|
155
|
+
|
|
156
|
+
def render(self, filename, a_geojson_dict: dict) -> None:
|
|
157
|
+
self._set_timestamps_to_UTC(a_geojson_dict)
|
|
158
|
+
table = self._as_table(a_geojson_dict)
|
|
159
|
+
self._cast_fields_and_add_missing(table)
|
|
160
|
+
table.write(filename)
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import jsonata
|
|
2
|
+
|
|
3
|
+
from .base import AbstractTransformer
|
|
4
|
+
|
|
5
|
+
# ruff: noqa: E501 (JSONata expressions are long...)
|
|
6
|
+
|
|
7
|
+
# TODO: we asume arrays applicableForTrafficType, placesAtWhichApplicable, applicableForTrafficDirection have at most 1 element... We should check this
|
|
8
|
+
mostly_flat_geojson_expression = """
|
|
9
|
+
{
|
|
10
|
+
"type": "FeatureCollection",
|
|
11
|
+
"features": payload.situation[].situationRecord.{
|
|
12
|
+
"type": "Feature",
|
|
13
|
+
"properties": {
|
|
14
|
+
"publicationTime": %.%.publicationTime,
|
|
15
|
+
"publicationCreator": %.%.publicationCreator.nationalIdentifier,
|
|
16
|
+
"publicationIdentifier": %.%.payloadPublicationExtension.publicationIdentifier,
|
|
17
|
+
"sourceIdentification": source.sourceIdentification,
|
|
18
|
+
"situationId": %.id,
|
|
19
|
+
"situationVersionTime": %.situationVersionTime,
|
|
20
|
+
"confidentiality": %.headerInformation.confidentiality,
|
|
21
|
+
"informationStatus": %.headerInformation.informationStatus,
|
|
22
|
+
"overallSeverity": %.overallSeverity,
|
|
23
|
+
"id": id,
|
|
24
|
+
"version": version,
|
|
25
|
+
"situationRecordCreationTime": situationRecordCreationTime,
|
|
26
|
+
"situationRecordVersionTime": situationRecordVersionTime,
|
|
27
|
+
"probabilityOfOccurrence": probabilityOfOccurrence,
|
|
28
|
+
/* description is not declared in BEMaS */
|
|
29
|
+
"description": generalPublicComment[commentType.value="description"].comment.values[lang="de"].value,
|
|
30
|
+
"roadworksType": generalPublicComment[commentExtension.commentExtended.commentType2="roadworksType"].comment.values[lang="de"].value,
|
|
31
|
+
"roadworksName": generalPublicComment[commentExtension.commentExtended.commentType2="roadworksName"].comment.values[lang="de"].value,
|
|
32
|
+
"heavyDutyTransportRestriction": generalPublicComment[commentExtension.commentExtended.commentType2="heavyDutyTransportRestriction"].comment.values[lang="de"].value,
|
|
33
|
+
"routeRecommendation": generalPublicComment[commentExtension.commentExtended.commentType2="routeRecommendation"][0].comment.values[lang="de"].value,
|
|
34
|
+
"type": type,
|
|
35
|
+
"complianceOption": complianceOption,
|
|
36
|
+
"applicableForTrafficDirection":applicableForTrafficDirection[0],
|
|
37
|
+
/* TODO this is not set for BEMaS? */
|
|
38
|
+
"placesAtWhichApplicable":placesAtWhichApplicable[0],
|
|
39
|
+
"cause":cause.managedCause.objectReference.id,
|
|
40
|
+
"numberOfOperationalLanes": impact.numberOfOperationalLanes,
|
|
41
|
+
"numberOfLanesRestricted": impact.numberOfLanesRestricted,
|
|
42
|
+
"residualRoadWidth": impact.residualRoadWidth,
|
|
43
|
+
"trafficConstrictionType": impact.trafficConstrictionType,
|
|
44
|
+
"laneRestriction": impact.impactExtension.impactExtended.laneRestriction,
|
|
45
|
+
"laneStatusCoded": impact.impactExtension.impactExtended.laneStatusCoded,
|
|
46
|
+
"extendedLaneStatusCoded": situationRecordExtension.situationRecordExtended.extendedLaneStatusCoded,
|
|
47
|
+
"hardShoulderStatus": impact.impactExtension.impactExtended.hardShoulderStatus,
|
|
48
|
+
/* Note: other publication (i.e. TrafficMessages) use roadClosed and forVehiclesWithCharacteristicsOf */
|
|
49
|
+
"residualHeight": impact.impactExtension.impactExtended.residualHeight,
|
|
50
|
+
/* Note: other publication (i.e. TrafficMessages) use roadClosed and forVehiclesWithCharacteristicsOf */
|
|
51
|
+
"maxVehicleWeight": impact.impactExtension.impactExtended.laneRestriction[lane="allLanesCompleteCarriageway"].maxVehicleWeight,
|
|
52
|
+
"capacityRemaining": impact.capacityRemaining,
|
|
53
|
+
"originalNumberOfLanes": impact.originalNumberOfLanes,
|
|
54
|
+
/* is numberOfObstructions used?*/
|
|
55
|
+
"numberOfObstructions": impact.numberOfObstructions,
|
|
56
|
+
/* Note: some publications (i.e. BEMaS use impactExtended to communicate height and weight restrictions ) */
|
|
57
|
+
"forVehiclesWithCharacteristicsOf": forVehiclesWithCharacteristicsOf,
|
|
58
|
+
"validityStatus": validity.validityStatus,
|
|
59
|
+
"overallStartTime": validity.validityTimeSpecification.overallStartTime,
|
|
60
|
+
"overallEndTime": validity.validityTimeSpecification.overallEndTime,
|
|
61
|
+
"validPeriods": validity.validityTimeSpecification.validPeriod,
|
|
62
|
+
"subjectTypeOfWorks": subjects.subjectTypeOfWorks,
|
|
63
|
+
"severity": severity,
|
|
64
|
+
"applicableForTrafficType":applicableForTrafficType[0],
|
|
65
|
+
/* TODO Check which types are defined with maxoccurs=*, check which have value */
|
|
66
|
+
"situationRecordSubtype": obstructionType[0] ?? reroutingManagementType[0] ?? constructionWorkType ?? generalInstructionToRoadUsersType ?? infrastructureDamageType ?? accidentType[0] ?? abnormalTrafficType ?? speedManagementType ?? roadMaintenanceType[0] ?? animalPresenceType ?? environmentalObstructionType ?? generalNetworkManagementType ?? authorityOperationType ?? roadOrCarriagewayOrLaneManagementType ?? vehicleObstructionType ?? poorEnvironmentType,
|
|
67
|
+
"actionPlanIdentifier": actionPlanIdentifier,
|
|
68
|
+
"operatorActionStatus": operatorActionStatus,
|
|
69
|
+
"roadworksDurationClassification": roadworksDurationClassification,
|
|
70
|
+
"mobilityType": mobility.mobilityType,
|
|
71
|
+
"roadworksLayout": roadworksExtension.roadworksExtended.roadworksLayout,
|
|
72
|
+
"urlLinkAddress": urlLink[0].urlLinkAddress,
|
|
73
|
+
"roadNumber": locationReference.linearWithinLinearElement.linearElement.locLinearElement.roadNumber,
|
|
74
|
+
"roadName": locationReference.linearWithinLinearElement.linearElement.locLinearElement.roadName.values[0].value,
|
|
75
|
+
"fromASBSegmentFromNode": locationReference.linearWithinLinearElement.fromPoint.fromReferent.referentIdentifier,
|
|
76
|
+
"fromASBSegmentTowardsNode": locationReference.linearWithinLinearElement.fromPoint.towardsReferent.referentIdentifier,
|
|
77
|
+
"fromASBSegmentStation": locationReference.linearWithinLinearElement.fromPoint.distanceAlong,
|
|
78
|
+
"toASBSegmentFromNode": locationReference.linearWithinLinearElement.toPoint.fromReferent.referentIdentifier,
|
|
79
|
+
"toASBSegmentTowardsNode": locationReference.linearWithinLinearElement.toPoint.towardsReferent.referentIdentifier,
|
|
80
|
+
"toASBSegmentStation": locationReference.linearWithinLinearElement.toPoint.distanceAlong,
|
|
81
|
+
"lengthAffected": locationReference.supplementaryPositionalDescription.lengthAffected,
|
|
82
|
+
"temporarySpeedLimit": temporarySpeedLimit ?? operatorActionExtension.operatorActionExtended.mainSpeedLimit
|
|
83
|
+
},
|
|
84
|
+
"geometry": locationReference.geoJsonGeometry
|
|
85
|
+
} ?? []
|
|
86
|
+
}
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class JsonataTransformer(AbstractTransformer):
|
|
91
|
+
def __init__(self, jsonata_expression=mostly_flat_geojson_expression):
|
|
92
|
+
self.jsonata_expr = jsonata.Jsonata(jsonata_expression)
|
|
93
|
+
|
|
94
|
+
def transform(self, a_dict: dict) -> dict:
|
|
95
|
+
result = self.jsonata_expr.evaluate(a_dict)
|
|
96
|
+
|
|
97
|
+
return self.post_transform(result)
|
|
98
|
+
|
|
99
|
+
def post_transform(self, a_dict: dict) -> dict:
|
|
100
|
+
return a_dict
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
from .base import SituationRecordTransformer
|
|
4
|
+
|
|
5
|
+
logger = logging.getLogger(__name__)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class RoadsRegioTransformer(SituationRecordTransformer):
|
|
9
|
+
def __init__(self, path: str = 'payload/sitSituationPublication/situation/situationRecord'):
|
|
10
|
+
super().__init__(path)
|
|
11
|
+
|
|
12
|
+
def _filter_empty_comments(self, situation_record: dict):
|
|
13
|
+
# Remove comments without value
|
|
14
|
+
if 'generalPublicComment' in situation_record:
|
|
15
|
+
comments = [
|
|
16
|
+
c
|
|
17
|
+
for c in situation_record['generalPublicComment']
|
|
18
|
+
if len(c['comment']['values']) > 0 and c['comment']['values'][0].get('value') is not None
|
|
19
|
+
]
|
|
20
|
+
situation_record['generalPublicComment'] = comments
|
|
21
|
+
|
|
22
|
+
def _handle_originalNumberOfLanes(self, situation_record: dict):
|
|
23
|
+
# Removes originalNumberOfLanes from impact.
|
|
24
|
+
|
|
25
|
+
if 'impact' in situation_record:
|
|
26
|
+
# TODO originalNumberOfLanes probably should they be added elsewhere
|
|
27
|
+
situation_record['impact'].pop('originalNumberOfLanes', None)
|
|
28
|
+
|
|
29
|
+
def _handle_bemas_locations(self, situation_record: dict):
|
|
30
|
+
# Removes originalNumberOfLanes from impact.
|
|
31
|
+
|
|
32
|
+
# BEMaS
|
|
33
|
+
# Expectation:
|
|
34
|
+
# locationGroup/
|
|
35
|
+
# linearExtended/gmlLineString/roadInformation
|
|
36
|
+
# . linearWithinLinearElement[0]/linearElement
|
|
37
|
+
# Result:
|
|
38
|
+
# linearElement/locLinearElementByLineString/
|
|
39
|
+
# . gmlLineString/locGmlLineString
|
|
40
|
+
# . roadNumber
|
|
41
|
+
# . roadName
|
|
42
|
+
locationGroup = situation_record.get('locationReference', {}).get('locSingleRoadLinearLocation', {})
|
|
43
|
+
|
|
44
|
+
linearExtended = locationGroup.pop('linearExtended', None)
|
|
45
|
+
gmlLineString = None
|
|
46
|
+
if isinstance(linearExtended, dict) and 'gmlLineString' in linearExtended:
|
|
47
|
+
gmlLineString = linearExtended['gmlLineString']
|
|
48
|
+
# roadInformation is not part of gmlLineString any more, remove it
|
|
49
|
+
gmlLineString.pop('roadInformation', {})
|
|
50
|
+
|
|
51
|
+
linearWithinLinearElements = locationGroup.get('linearWithinLinearElement', [])
|
|
52
|
+
|
|
53
|
+
# Copy roadinformation and gmlLineString to locLinearElementByLineString
|
|
54
|
+
if len(linearWithinLinearElements) == 1:
|
|
55
|
+
linearElement = linearWithinLinearElements[0].get('linearElement', {})
|
|
56
|
+
locLinearElementByLineString = {'gmlLineString': {'locGmlLineString': gmlLineString}} if gmlLineString else {}
|
|
57
|
+
if 'roadNumber' in linearElement:
|
|
58
|
+
locLinearElementByLineString['roadNumber'] = linearElement.pop('roadNumber', None)
|
|
59
|
+
if 'roadName' in linearElement:
|
|
60
|
+
locLinearElementByLineString['roadName'] = linearElement.pop('roadName', None)
|
|
61
|
+
|
|
62
|
+
linearElement['locLinearElementByLineString'] = locLinearElementByLineString
|
|
63
|
+
linearWithinLinearElements[0]['linearElement'] = linearElement
|
|
64
|
+
|
|
65
|
+
def _fix_missing_linearElement(self, situation_record: dict):
|
|
66
|
+
# Removes originalNumberOfLanes from impact.
|
|
67
|
+
|
|
68
|
+
# BEMaS
|
|
69
|
+
# Expectation:
|
|
70
|
+
# locationGroup/
|
|
71
|
+
# linearExtended/gmlLineString/roadInformation
|
|
72
|
+
# . linearWithinLinearElement[0]/linearElement
|
|
73
|
+
# Result:
|
|
74
|
+
# linearElement/locLinearElementByLineString/
|
|
75
|
+
# . gmlLineString/locGmlLineString
|
|
76
|
+
# . roadNumber
|
|
77
|
+
# . roadName
|
|
78
|
+
locationGroup = situation_record.get('locationReference', {}).get('locSingleRoadLinearLocation', {})
|
|
79
|
+
linearWithinLinearElements = locationGroup.get('linearWithinLinearElement', [])
|
|
80
|
+
if len(linearWithinLinearElements) == 1:
|
|
81
|
+
if 'linearElement' not in linearWithinLinearElements[0]:
|
|
82
|
+
linearWithinLinearElements[0]['linearElement'] = {}
|
|
83
|
+
|
|
84
|
+
def process_situation_record(self, situation_record: dict) -> None:
|
|
85
|
+
# self._handle_bemas_locations(situation_record)
|
|
86
|
+
self._fix_missing_linearElement(situation_record)
|
|
87
|
+
self._handle_originalNumberOfLanes(situation_record)
|
|
88
|
+
self._filter_empty_comments(situation_record)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
from datex2_xml2json.transform.base import SituationRecordTransformer
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def recurse(data, level=0):
|
|
5
|
+
if level > 2:
|
|
6
|
+
return
|
|
7
|
+
if type(data) is dict:
|
|
8
|
+
for key, value in data.items():
|
|
9
|
+
print(level * ' ', key)
|
|
10
|
+
recurse(value, level + 1)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class SourceSetter(SituationRecordTransformer):
|
|
14
|
+
"""
|
|
15
|
+
This Transformer sets situationRecord/source/sourceIdentification and the
|
|
16
|
+
custom extension payloadPublication/payloadPublicationExtension/publicationIdentifier,
|
|
17
|
+
if they are specified in the constructor call.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
sourceIdentification: str | None = None
|
|
21
|
+
publicationIdentifier: str | None = None
|
|
22
|
+
|
|
23
|
+
def __init__(self, publicationIdentifier: str | None, sourceIdentification: str | None):
|
|
24
|
+
super().__init__('d2LogicalModel/payloadPublication/situation/situationRecord')
|
|
25
|
+
self.publicationIdentifier = publicationIdentifier
|
|
26
|
+
self.sourceIdentification = sourceIdentification
|
|
27
|
+
|
|
28
|
+
def process_situation_record(self, situation_record: dict) -> None:
|
|
29
|
+
situation_record['source'] = {'sourceIdentification': self.sourceIdentification}
|
|
30
|
+
|
|
31
|
+
def transform(self, publication: dict) -> dict:
|
|
32
|
+
recurse(publication)
|
|
33
|
+
if self.sourceIdentification is not None:
|
|
34
|
+
publication = super().transform(publication)
|
|
35
|
+
|
|
36
|
+
recurse(publication)
|
|
37
|
+
if self.publicationIdentifier is not None:
|
|
38
|
+
publication['d2LogicalModel']['payloadPublication'].setdefault('payloadPublicationExtension', {})['publicationIdentifier'] = (
|
|
39
|
+
self.publicationIdentifier
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
return publication
|