datex2-xml2json 0.0.1__py3-none-any.whl

datex2_xml2json/schema/cifs_schema.json

@@ -0,0 +1,161 @@
+{
+  "$schema": "urn:json-schema:2020-12",
+  "type": "object",
+  "properties": {
+    "incidents": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string",
+            "description": "Specifies an alphanumeric or numeric identifier. The ID must be globally unique to your feed and remain stable over an incident’s lifetime.",
+            "minLength": 3
+          },
+          "parent_event": {
+            "type": "string",
+            "description": "For events with multiple closures such as marathons or festivals, you can associate the incident to an event object."
+          },
+          "creationtime": {
+            "type": "string",
+            "description": "Datetime when the incident was created.",
+            "format": "date-time"
+          },
+          "updatetime": {
+            "type": "string",
+            "description": "Datetime when the incident was last updated.",
+            "format": "date-time"
+          },
+          "source": {
+            "type": "object",
+            "description": "Encapsulates the elements that specify the source of information for a single incident.",
+            "properties": {
+              "reference": {
+                "type": "string",
+                "description": "Uses a reference ID to identify the organization that sourced the information.",
+                "minLength": 1
+              },
+              "name": {
+                "type": "string",
+                "description": "Identifies the organization from which the information was sourced, by the organization’s name.",
+                "minLength": 1
+              },
+              "url": {
+                "type": "string",
+                "description": "Specifies the URL for the organization from which the information was sourced.",
+                "minLength": 1
+              }
+            },
+            "required": [
+              "reference"
+            ]
+          },
+          "type": {
+            "description": "Specifies the type of incident.",
+            "enum": [
+              "ACCIDENT",
+              "CONSTRUCTION",
+              "HAZARD",
+              "ROAD_CLOSED",
+              "POLICE",
+              "CHIT_CHAT",
+              "JAM"
+            ]
+          },
+          "subtype": {
+            "description": "Further refinement of the information in the \"type\" element.",
+            "enum": [
+              "",
+              "ACCIDENT_MAJOR",
+              "ACCIDENT_MINOR",
+              "HAZARD_ON_ROAD",
+              "HAZARD_ON_ROAD_CAR_STOPPED",
+              "HAZARD_ON_ROAD_CONSTRUCTION",
+              "HAZARD_ON_ROAD_ICE",
+              "HAZARD_ON_ROAD_LANE_CLOSED",
+              "HAZARD_ON_ROAD_OBJECT",
+              "HAZARD_ON_ROAD_OIL",
+              "HAZARD_ON_ROAD_POT_HOLE",
+              "HAZARD_ON_ROAD_ROAD_KILL",
+              "HAZARD_ON_ROAD_TRAFFIC_LIGHT_FAULT",
+              "HAZARD_ON_SHOULDER",
+              "HAZARD_ON_SHOULDER_ANIMALS",
+              "HAZARD_ON_SHOULDER_CAR_STOPPED",
+              "HAZARD_ON_SHOULDER_MISSING_SIGN",
+              "HAZARD_WEATHER",
+              "HAZARD_WEATHER_FLOOD",
+              "HAZARD_WEATHER_FOG",
+              "HAZARD_WEATHER_FREEZING_RAIN",
+              "HAZARD_WEATHER_HAIL",
+              "HAZARD_WEATHER_HEAT_WAVE",
+              "HAZARD_WEATHER_HEAVY_RAIN",
+              "HAZARD_WEATHER_HEAVY_SNOW",
+              "HAZARD_WEATHER_HURRICANE",
+              "HAZARD_WEATHER_MONSOON",
+              "HAZARD_WEATHER_TORNADO",
+              "JAM_HEAVY_TRAFFIC",
+              "JAM_LIGHT_TRAFFIC",
+              "JAM_MODERATE_TRAFFIC",
+              "JAM_STAND_STILL_TRAFFIC",
+              "ROAD_CLOSED_CONSTRUCTION",
+              "ROAD_CLOSED_EVENT",
+              "ROAD_CLOSED_HAZARD",
+              "POLICE_VISIBLE",
+              "POLICE_HIDING"
+            ]
+          },
+          "description": {
+            "type": "string",
+            "description": "Describes the incident, including the possible cause and consequences of the disruption.",
+            "minLength": 1
+          },
+          "street": {
+            "type": "string",
+            "description": "Specifies the name of the street on which the incident is occurring.",
+            "minLength": 2
+          },
+
+          "polyline": {
+            "type": "string",
+            "description": "Specifies the WGS84 latitude/longitude coordinates that describe the location of the incident. The decimal value should have at least 6 digits (to ensure accuracy of 0.11m or better). For any incident, you must submit at least two coordinates."
+          },
+          "direction": {
+            "description": "Specifies whether the disruption or closure affects one or both sides of the road.",
+            "enum": [
+              "ONE_DIRECTION",
+              "BOTH_DIRECTIONS"
+            ]
+          },
+          "starttime": {
+            "type": "string",
+            "description": "Specifies the start datetime for the period of disruption.",
+            "format": "date-time"
+          },
+          "endtime": {
+            "type": "string",
+            "description": "Specifies the end datetime for the period of disruption.",
+            "format": "date-time"
+          }
+        },
+        "additionalProperties": true,
+        "required": [
+          "id",
+          "description",
+          "starttime",
+          "street",
+          "type",
+          "polyline"
+        ]
+      }
+    },
+    "timestamp": {
+      "type": "string",
+      "description": "Specifies the creation time of the feed.",
+      "format": "date-time"
+    }
+  },
+  "additionalProperties": true,
+  "required": [
+    "incidents"
+  ]
+}

datex2_xml2json/transform/__init__.py

@@ -0,0 +1,13 @@
+from .base import (
+    TransformerPipeline as TransformerPipeline,
+    RulesTransformer as RulesTransformer,
+)
+from .jsonata_transformer import JsonataTransformer as JsonataTransformer
+from .transformation_rules import TransformationRuleFactory as TransformationRuleFactory
+from .roads_regio_transformer import RoadsRegioTransformer as RoadsRegioTransformer
+from .geojson_geometry_transformer import (
+    GeoJSONGeometryTransformer as GeoJSONGeometryTransformer,
+)
+from .geoparquet_transformer import GeoparquetRenderer as GeoparquetRenderer
+from .cifs_transformer import CifsTransformer
+from .source_setter import SourceSetter

datex2_xml2json/transform/base.py

@@ -0,0 +1,79 @@
+from datex2_xml2json.util import replace, xpath_dict
+
+from .transformation_rules import TransformationRules
+
+
+class AbstractTransformer:
+    def transform(self, a_dict: dict) -> dict:
+        return a_dict
+
+
+class TransformerPipeline(AbstractTransformer):
+    def __init__(self, transformers: list[AbstractTransformer]):
+        self.transformers = list(transformers)
+
+    def add_transformer(self, transformer: AbstractTransformer):
+        self.transformers.append(transformer)
+        return self
+
+    def transform(self, a_dict: dict) -> dict:
+        result = a_dict
+        for t in self.transformers:
+            result = t.transform(result)
+
+        return result
+
+
+class RulesTransformer(AbstractTransformer):
+    def __init__(self, rules: TransformationRules):
+        self.rules = rules
+
+    def transform(self, a_dict: dict) -> dict:
+        # Transforms the result of xml2json transformation into DATEXIIv3 json.
+        # This includes:
+        # * renamings of certain properties according to v2 to v3 name changes
+        # * renamings of certain properties according to changed writing in json
+        # * casts values of certain properties to intended types
+        # * ...
+
+        return replace(
+            a_dict,
+            None,
+            self.rules.renamings,
+            self.rules.value_transforms,
+            self.rules.keys_to_drop,
+            self.rules.type_mappings,
+            self.rules.keys_to_pull_up,
+        )
+
+
+class ProcessingTransformer(AbstractTransformer):
+    def __init__(self, xpath: str):
+        self.xpath = xpath
+
+    def transform(self, a_dict: dict) -> dict:
+        for elem in xpath_dict(a_dict, self.xpath):
+            self.process(elem)
+
+        return a_dict
+
+    def process(self, a_dict: dict):
+        pass
+
+
+class SituationRecordTransformer(ProcessingTransformer):
+    def __init__(self, xpath: str):
+        super().__init__(xpath)
+
+    def process_situation_record(self, situation_record: dict) -> None:
+        pass
+
+    def process(self, situation_record_wrapper: dict) -> None:
+        if isinstance(situation_record_wrapper, dict) and len(situation_record_wrapper) == 1:
+            situation_record = situation_record_wrapper.get(next(iter(situation_record_wrapper.keys())))
+        else:
+            situation_record = situation_record_wrapper
+        if situation_record is None:
+            return
+
+        self.process_situation_record(situation_record)

datex2_xml2json/transform/cifs_transformer.py

@@ -0,0 +1,275 @@
+import copy
+import logging
+import re
+
+from .jsonata_transformer import JsonataTransformer
+
+logger = logging.getLogger(__name__)
+
+# TODO: PublicEvent notificcations and GeneralInstructionOrMessageToRoadUsers do not really match
+# TODO how can we deduce direction for NRW incidents
+cifs_expression = """
+{
+    "incidents": features[
+        properties.type!="CarParks" and
+        properties.type!="ReroutingManagement" and
+        properties.type!="EquipmentOrSystemFault" and
+        properties.validityStatus!="suspended"
+        ][].{
+        "id": properties.id,
+        "creationtime": properties.situationRecordCreationTime,
+        "updatetime": properties.situationRecordVersionTime,
+        "description": properties.roadworksName ? properties.roadworksName : properties.description ?? "-",
+        "starttime": properties.overallStartTime,
+        "endtime": properties.overallEndTime,
+        "direction": "ONE_DIRECTION",
+        "street": $trim(properties.roadNumber?($contains(properties.roadName, properties.roadNumber)?
+          properties.roadName: properties.roadNumber & " " & properties.roadName): $split(properties.description, "\n")[0] ??
+          "Unbekannte Straße"),
+        "type": (
+            $typeBySubtypeLookup := {
+                "carriagewayClosures": "ROAD_CLOSED",
+                "closedPermanentlyForTheWinter": "ROAD_CLOSED",
+                "overnightClosures": "ROAD_CLOSED",
+                "roadClosed": "ROAD_CLOSED",
+                "unprotectedAccidentArea": "ACCIDENT"
+                };
+            $typeLookup := {
+                "Accident": "ACCIDENT",
+                "AnimalPresence": "HAZARD",
+                "AnimalPresenceObstruction": "HAZARD",
+                "AbnormalTraffic": "JAM",
+                "AuthorityOperation": "HAZARD",
+                "Conditions": "HAZARD",
+                "ConstructionWorks": "CONSTRUCTION",
+                "EnvironmentalObstruction": "HAZARD",
+                "GeneralObstruction": "HAZARD",
+                "GeneralNetworkManagement": "HAZARD",
+                "InfrastructureDamageObstruction": "HAZARD",
+                "MaintenanceWorks": "CONSTRUCTION",
+                "Obstruction": "HAZARD",
+                "RoadOrCarriagewayOrLaneManagement":"HAZARD",
+                "SpeedManagement": "HAZARD",
+                "VehicleObstruction": "HAZARD",
+                "PublicEvent": "CHIT_CHAT",
+                "GeneralInstructionOrMessageToRoadUsers": "CHIT_CHAT",
+                "NonWeatherRelatedRoadConditions": "HAZARD",
+                "WeatherRelatedRoadConditions": "HAZARD",
+                "DisturbanceActivity": "CHIT_CHAT",
+                "TransitInformation": "CHIT_CHAT"
+                };
+            $lookup($typeBySubtypeLookup , properties.situationRecordSubtype)?:$lookup($typeLookup , properties.type)?:"TODO"),
+        "subtype": (
+            $subtypeByTypeLookup := {
+                /* Accident (no subtypes, as DATEXII does not provide accident severity) */
+                /* AnimalPresence has no exact correspondance, we prefer HAZARD_ON_SHOULDER_ANIMALS over HAZARD_ON_ROAD_ROAD_KILL */
+                "AnimalPresence": "HAZARD_ON_SHOULDER_ANIMALS",
+                "AnimalPresenceObstruction": "HAZARD_ON_SHOULDER_ANIMALS",
+                "AuthorityOperation": "HAZARD_ON_ROAD",
+                "GeneralNetworkManagement": "HAZARD_ON_ROAD",
+                "InfrastructureDamageObstruction": "HAZARD_ON_ROAD",
+                "MaintenanceWorks": "HAZARD_ON_ROAD_CONSTRUCTION",
+                "Obstruction": "HAZARD_ON_ROAD",
+                "RoadOrCarriagewayOrLaneManagement": "HAZARD_ON_ROAD",
+                "VehicleObstruction": "HAZARD_ON_ROAD",
+                "WeatherRelatedRoadConditions": "HAZARD_WEATHER"
+                };
+            $subtypeBySubtypeLookup := {
+                /* AbnormalTraffic (no subtype for unspecifiedAbnormalTraffic and other)*/
+                "stationaryTraffic": "JAM_STAND_STILL_TRAFFIC",
+                "queuingTraffic": "JAM_HEAVY_TRAFFIC",
+                "slowTraffic": "JAM_MODERATE_TRAFFIC",
+                "heavyTraffic": "JAM_LIGHT_TRAFFIC",
+                /* Accident (no subtypes, as DATEXII does not provide accident severity) */
+                "roadworks": "HAZARD_ON_ROAD_CONSTRUCTION",
+                /* EnvironmentalObstruction*/
+                "avalanches": "HAZARD_WEATHER",
+                "earthquakeDamage": "HAZARD_ON_ROAD",
+                "fallenTrees": "HAZARD_ON_ROAD_OBJECT",
+                "fallingIce": "HAZARD_WEATHER_HAIL",
+                "fallingLightIceOrSnow": "HAZARD_WEATHER_FREEZING_RAIN",
+                "flashFloods": "HAZARD_WEATHER_MONSOON",
+                "flooding": "HAZARD_WEATHER_FLOOD",
+                "forestFire": "HAZARD_WEATHER",
+                "grassFire": "HAZARD_WEATHER",
+                "landslips": "HAZARD_ON_ROAD",
+                "mudSlide": "HAZARD_ON_ROAD",
+                "sewerOverflow": "HAZARD_ON_ROAD",
+                "rockfalls": "HAZARD_ON_ROAD",
+                "seriousFire": "HAZARD_WEATHER",
+                "smokeOrFumes": "HAZARD_WEATHER",
+                "stormDamage": "HAZARD_ON_ROAD",
+                "subsidence": "HAZARD_ON_ROAD",
+                /* GeneralNetworkManagement (default defined via subtypeByTypeLookup, no exceptions )*/
+                /* Obstruction (default defined via subtypeByTypeLookup, here only exceptions ) */
+                "objectOnTheRoad": "HAZARD_ON_ROAD_OBJECT",
+                "objectsFallingFromMovingVehicle": "HAZARD_ON_ROAD_OBJECT",
+                "obstructionOnTheRoad": "HAZARD_ON_ROAD_OBJECT",
+                /* RoadOrCarriagewayOrLaneManagement (default defined via subtypeByTypeLookup, here only exceptions ) */
+                "carriagewayClosures": "ROAD_CLOSED_CONSTRUCTION",/* TODO: subtype needs to be figured out via sibling situationRecord*/
+                "closedPermanentlyForTheWinter": "ROAD_CLOSED_HAZARD",
+                "doNotUseSpecifiedLanesOrCarriageways": "HAZARD_ON_ROAD_LANE_CLOSED",
+                "laneClosures": "HAZARD_ON_ROAD_LANE_CLOSED",
+                "overnightClosures": "ROAD_CLOSED_CONSTRUCTION",/* TODO: subtype needs to be figured out via sibling situationRecord*/
+                "roadClosed": "ROAD_CLOSED_CONSTRUCTION", /* TODO: subtype needs to be figured out via sibling situationRecord*/
+                /* VehicleObstruction (default defined via subtypeByTypeLookup, here only exceptions )*/
+                "abandonedVehicle": "HAZARD_ON_ROAD_CAR_STOPPED",
+                "brokenDownBus": "HAZARD_ON_ROAD_CAR_STOPPED",
+                "brokenDownHeavyLorry": "HAZARD_ON_ROAD_CAR_STOPPED",
+                "brokenDownVehicle": "HAZARD_ON_ROAD_CAR_STOPPED",
+                "damagedVehicle": "HAZARD_ON_ROAD_CAR_STOPPED",
+                "emergencyVehicle": "HAZARD_ON_ROAD_EMERGENCY_VEHICLE",
+                "unlitVehicleOnTheRoad": "HAZARD_ON_ROAD_CAR_STOPPED",
+                "vehicleOnFire": "HAZARD_ON_ROAD_CAR_STOPPED",
+                /* NonWeatherRelatedRoadConditions */
+                "roadSurfaceInPoorCondition": "HAZARD_ON_ROAD_POT_HOLE"
+                };
+            $lookup($subtypeBySubtypeLookup, properties.situationRecordSubtype)?:$lookup($subtypeByTypeLookup, properties.type)
+        ),
+        /* Note: As geometries might be LineStrings, Points or even MultiLineStrings, we just copy them and
+           handle them in a post_transform. Note: For Points, a zero-length polyline linestring will be created,
+           multilines will result in multiple CIFS events. */
+        "geometry": geometry
+    } ?? []
+}
+"""
+
+
+class CifsTransformer(JsonataTransformer):
+    # This pattern is used to exract the common ID prefix of all situation records which
+    # should be grouped in a single CIFS entry.
+    # Currently expected patterns are:
+    # 1. four hyphen separated numeric values, followed by no or any subsring
+    # 2. Any string/char prefix followed by dot + number
+    situation_record_group_prefix_regexp = r'^(\d+-\d\d\d\dV\d+)$|^(\d+-\d+-\d+-\d+-gegen).*|^(\d+-\d+-\d+-\d+).*|^(.*)\.\d+$'
+
+    def __init__(self, **kwargs):
+        super().__init__(cifs_expression)
+
+    def _group_by_common_situation_record_id_substring(self, incidents, pattern):
+        grouped_incidents = {}
+        p = re.compile(pattern)
+
+        for incident in incidents:
+            m = p.match(incident['id'])
+            if m is None:
+                logger.warning(f'No match for id {incident["id"]} with pattern {pattern} ')
+                continue
+            common_id_substring = next(group for group in m.groups() if group is not None)
+            if common_id_substring in grouped_incidents:
+                grouped_incidents[common_id_substring].append(incident)
+            else:
+                grouped_incidents[common_id_substring] = [incident]
+
+        return grouped_incidents
+
+    def _get_type_sub_type_prio(self, incident):
+        if incident.get('type') == 'ROAD_CLOSED':
+            return 10
+        if incident.get('subtype') == 'HAZARD_ON_ROAD_LANE_CLOSED':
+            return 4
+        if incident.get('subtype') == 'HAZARD_ON_ROAD_CONSTRUCTION':
+            return 3
+        if incident.get('subtype') == 'HAZARD_ON_ROAD' or incident.get('type') in ['CHIT_CHAT']:
+            return 1
+        else:
+            return 2
+
+    def _merge_incident_group(self, incident_group):
+        # Assumptions: besides type and subtype, all properties are assumed to be equals
+        # (TODO: we might check this explicitly)
+        #
+        # For now, all incidents are merged into one single incident which should represent all of them.
+        # Rules are as follows:
+        # * highest priority subtype/type wins
+        # TODO: subtype for road_closed to be set according to other incidents
+        # TODO: should jam be communicated in addition?
+        highest_prio = None
+        highest_prio_type = None
+        highest_prio_subtype = None
+        highest_prio_incident = None
+
+        for incident in incident_group:
+            current_prio = self._get_type_sub_type_prio(incident)
+            if highest_prio is None or current_prio > highest_prio:
+                logger.debug(
+                    f'For {incident["id"]}, replace {highest_prio} {highest_prio_type} {highest_prio_subtype}'
+                    f' by {incident["type"]} {incident.get("subtype")}'
+                )
+                highest_prio = current_prio
+                highest_prio_type = incident['type']
+                highest_prio_subtype = incident.get('subtype')
+                highest_prio_incident = incident
+            else:
+                logger.debug(
+                    f'For {incident["id"]}, do not replace {highest_prio} {highest_prio_type} {highest_prio_subtype}'
+                    f' by {incident["type"]} {incident.get("subtype")}'
+                )
+        return highest_prio_incident
+
+    def _transform_polyline_geom(self, linestring_coordinates) -> str:
+        """
+        Converts GeoJSON linestring coordinates array into CIFS polyline string.
+        The list of lon/lat coords (like [[10.0, 49.0], [11.0, 49.0]]) is converted into a
+        lat/lon ordered string ("49.0 10.0 49.0 11.0").
+        """
+        return ' '.join([f'{coords[1]} {coords[0]}' for coords in linestring_coordinates])
+
+    def _transform_incident_geometry(self, incident: dict) -> list[dict]:
+        """
+        Converts GeoJSON geometry of incident into a CIFS polyline.
+        As the GeoJSON geometry might be a MultiLineString, we return an array
+        of (in case of a multilinestring) multiple CIFS incidents with linear polylines.
+        Note: Point geometries will be converted into zero-length linestrings.
+        """
+        geometry = incident.pop('geometry')
+        geometry_type = geometry['type']
+        coordinates = geometry['coordinates']
+
+        if geometry_type == 'MultiLineString':
+            exploded_incidents = []
+            cnt = 0
+            for linestring_coordinates in coordinates:
+                cloned_incident = copy.deepcopy(incident)
+                cloned_incident['id'] = f'{cloned_incident["id"]}.{cnt:02d}'
+                cloned_incident['polyline'] = self._transform_polyline_geom(linestring_coordinates)
+                exploded_incidents.append(cloned_incident)
+                cnt += 1
+
+            return exploded_incidents
+
+        if geometry_type == 'Point':
+            incident['polyline'] = f'{coordinates[1]} {coordinates[0]}'
+        elif geometry_type == 'LineString':
+            incident['polyline'] = self._transform_polyline_geom(coordinates)
+        else:
+            logger.error('Unexpected geometry type %s for incident %s, incident is omitted', geometry_type, incident.get('id'))
+            return []
+
+        return [incident]
+
+    def _merge_grouped_incidents_and_handle_geometry(self, grouped_incidents):
+        incidents = []
+        for key in grouped_incidents:
+            try:
+                incident = self._merge_incident_group(grouped_incidents[key])
+                potentially_exploded_incidents = self._transform_incident_geometry(incident)
+                incidents.extend(potentially_exploded_incidents)
+            except Exception:
+                logger.exception('Error while merging for incidents of key %s', key)
+
+        return incidents
+
+    def post_transform(self, a_dict: dict) -> dict:
+        """
+        Retains the most expressive incident of a group of incidents describing
+        the same situation.
+        Currently, a common substring of the situationRecord IDs is used to group these events.
+        As this strategy is very brittle, other strategies should be considered, like
+        grouping by equivalent geometries/validity periods for situation records of the same situation.
+        """
+        grouped_incidents = self._group_by_common_situation_record_id_substring(
+            a_dict['incidents'], self.situation_record_group_prefix_regexp
+        )
+        a_dict['incidents'] = self._merge_grouped_incidents_and_handle_geometry(grouped_incidents)
+        return a_dict