edfh-data 0.5.2__py3-none-any.whl

edfh_data/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.2"

edfh_data/db/crud.py

@@ -0,0 +1,218 @@
+import datetime as dt
+import json
+
+from sqlmodel import and_
+from sqlmodel import desc
+from sqlmodel import select
+from sqlmodel import Session
+
+from edfh_data.db.models import FactionStateDB
+from edfh_data.db.models import MarketCommodityDB
+from edfh_data.db.models import StationDB
+from edfh_data.db.models import SystemDB
+from edfh_data.db.models import SystemFactionDB
+from edfh_data.db.models import SystemFactionHistoryDB
+from edfh_data.db.models import SystemPowerDB
+from edfh_data.db.utils import common_keys_equal
+from edfh_data.db.utils import engine
+from edfh_data.models import Station
+from edfh_data.models import StationMarket
+from edfh_data.models import System
+
+UTC = dt.timezone.utc
+
+
+def create_update_system(system: System) -> None:
+
+    system_data = system.model_dump(
+        exclude_none=True, exclude={"system_factions", "system_powers"}
+    )
+
+    with Session(engine) as session:
+        stmt = select(SystemDB).where(SystemDB.system_name == system.system_name)
+        res = session.exec(stmt)
+        system_db = res.one_or_none()
+
+        if system_db:
+            system_update = system.update_datetime.replace(tzinfo=UTC)
+            system_db_update = system_db.update_datetime.replace(tzinfo=UTC)
+            if system_update > system_db_update:
+                system_db = system_db | system_data
+            else:
+                return
+        else:
+            system_db = SystemDB(**system_data)
+            session.add(system_db)
+            session.commit()
+            session.refresh(system_db)
+
+        system_factions = []
+        for faction in system.system_factions:
+
+            faction_data = faction.model_dump(exclude_none=True, exclude={"states"})
+
+            states_data = None
+            if faction.states:
+                states_data = json.dumps(faction.model_dump(mode="json")["states"])
+
+            stmt = select(SystemFactionDB).where(
+                and_(
+                    SystemFactionDB.system_id == system_db.system_id,
+                    SystemFactionDB.faction_name == faction.faction_name,
+                )
+            )
+            res = session.exec(stmt)
+            system_faction = res.one_or_none()
+
+            if system_faction:
+                system_faction = system_faction | faction_data
+            else:
+                system_faction = SystemFactionDB(**faction_data)
+
+            system_faction.faction_states = [
+                FactionStateDB(**state.model_dump()) for state in faction.states
+            ]
+
+            system_factions.append(system_faction)
+
+            # Get latest faction data from history
+            stmt = (
+                select(SystemFactionHistoryDB)
+                .where(
+                    and_(
+                        SystemFactionHistoryDB.system_id == system_db.system_id,
+                        SystemFactionHistoryDB.faction_name == faction.faction_name,
+                    )
+                )
+                .order_by(desc(SystemFactionHistoryDB.update_datetime))
+            )
+            res = session.exec(stmt)
+            system_faction_latest = res.first()
+
+            if system_faction_latest:
+                # Only update the update_datetime field if faction data has not changed
+                if common_keys_equal(faction_data, system_faction_latest.model_dump()):
+                    system_faction_latest.update_datetime = system.update_datetime
+                    session.add(system_faction_latest)
+                # Create new entry only if values have updated
+                else:
+                    system_faction_latest = SystemFactionHistoryDB(
+                        system_id=system_db.system_id,  # type: ignore
+                        update_datetime=system.update_datetime,
+                        states=states_data,
+                        **faction_data,
+                    )
+            else:
+                system_faction_latest = SystemFactionHistoryDB(
+                    system_id=system_db.system_id,  # type: ignore
+                    update_datetime=system.update_datetime,
+                    states=states_data,
+                    **faction_data,
+                )
+
+            session.add(system_faction_latest)
+
+        system_db.system_factions = system_factions
+
+        system_powers = []
+        for power in system.system_powers:
+
+            stmt = select(SystemPowerDB).where(
+                and_(
+                    SystemPowerDB.system_id == system_db.system_id,
+                    SystemPowerDB.power == power,
+                )
+            )
+            res = session.exec(stmt)
+            system_power = res.one_or_none()
+
+            if not system_power:
+                system_power = SystemPowerDB(
+                    system_id=system_db.system_id,  # type: ignore
+                    power=power,
+                )
+
+            system_powers.append(system_power)
+
+        system_db.system_powers = system_powers
+
+        session.add(system_db)
+        session.commit()
+
+
+def create_update_station(station: Station) -> None:
+
+    station_data = station.model_dump(exclude_none=True)
+
+    with Session(engine) as session:
+        # Get the parent system from the DB if it exists
+        stmt = select(SystemDB).where(SystemDB.system_name == station.system_name)
+        res = session.exec(stmt)
+        system_db = res.one_or_none()
+
+        # Stop if the system does not exist in the DB
+        if system_db is None:
+            return
+
+        # Get the station from the DB if it exists
+        stmt = select(StationDB).where(StationDB.market_id == station.market_id)
+        res = session.exec(stmt)
+        station_db = res.one_or_none()
+
+        if station_db:
+            station_update = station.update_datetime.replace(tzinfo=UTC)
+            station_db_update = station_db.update_datetime.replace(tzinfo=UTC)
+            if station_update > station_db_update:
+                station_db = station_db | station_data
+                station_db.system_id = system_db.system_id  # type: ignore
+            else:
+                return
+        else:
+            station_db = StationDB(
+                system_id=system_db.system_id,  # type: ignore
+                **station_data,
+            )
+
+        session.add(station_db)
+        session.commit()
+
+
+def create_update_station_market(market: StationMarket):
+
+    with Session(engine) as session:
+        # Get the parent station from the DB
+        stmt = select(StationDB).where(StationDB.market_id == market.market_id)
+        res = session.exec(stmt)
+        station_db = res.one_or_none()
+
+        # Stop if the station does not exist in the DB
+        if station_db is None:
+            return
+
+        market_update = market.update_datetime.replace(tzinfo=UTC)
+        station_market_update = station_db.market_update_datetime
+        if (
+            station_market_update is None
+            or market_update > station_market_update.replace(tzinfo=UTC)
+        ):
+            existing_commodities = {c.name: c for c in station_db.market_commodities}
+            commodities = []
+
+            for commodity in market.commodities:
+                if commodity.name in existing_commodities:
+                    commodities.append(
+                        existing_commodities[commodity.name] | commodity.model_dump()
+                    )
+                else:
+                    commodities.append(
+                        MarketCommodityDB(
+                            station_id=station_db.station_id,  # type: ignore
+                            **commodity.model_dump(),
+                        )
+                    )
+
+            station_db.market_update_datetime = market.update_datetime
+            station_db.market_commodities = commodities
+
+            session.add(station_db)
+            session.commit()

edfh_data/db/models.py

@@ -0,0 +1,164 @@
+import datetime as dt
+from typing import Mapping, Self
+
+from sqlmodel import BigInteger
+from sqlmodel import Column
+from sqlmodel import Field
+from sqlmodel import Relationship
+from sqlmodel import SQLModel
+from sqlmodel import String
+from sqlmodel import TIMESTAMP
+
+
+class UpdatableSQLModel(SQLModel):
+
+    def update(self, **kwargs: Mapping) -> Self:
+        """Update the speficied fields of the model instance and return the instance."""
+        for field, value in kwargs.items():
+            if field in self.__class__.model_fields:
+                setattr(self, field, value)
+        return self
+
+    def __or__(self, other: Mapping) -> Self:
+        """Update the speficied fields of the model instance from a dictionary using
+        the or ('|') operator"""
+        return self.update(**other)
+
+
+class SystemDB(UpdatableSQLModel, table=True):
+    __tablename__ = "systems"  # type: ignore
+    system_id: int | None = Field(default=None, primary_key=True, sa_type=BigInteger)
+    system_name: str
+    system_address: int = Field(sa_type=BigInteger)
+    x: float
+    y: float
+    z: float
+    population: int = Field(sa_type=BigInteger)
+    system_factions: list["SystemFactionDB"] = Relationship(
+        back_populates="system", cascade_delete=True
+    )
+    update_datetime: dt.datetime = Field(
+        sa_column=Column(TIMESTAMP(timezone=True), nullable=False)
+    )
+    system_allegiance: str | None = Field(default=None)
+    controlling_faction: str | None = Field(default=None)
+    controlling_power: str | None = Field(default=None)
+    powerplay_state: str | None = Field(default=None)
+    powerplay_state_control_progress: float | None = Field(default=None)
+    powerplay_state_reinforcement: int | None = Field(default=None)
+    powerplay_state_undermining: int | None = Field(default=None)
+    primary_economy: str | None = Field(default=None)
+    secondary_economy: str | None = Field(default=None)
+    security: str | None = Field(default=None)
+    government: str | None = Field(default=None)
+    system_powers: list["SystemPowerDB"] = Relationship(
+        back_populates="system", cascade_delete=True
+    )
+    system_factions_history: list["SystemFactionHistoryDB"] = Relationship(
+        back_populates="system", cascade_delete=True
+    )
+    stations: list["StationDB"] = Relationship(
+        back_populates="system", cascade_delete=True
+    )
+
+
+class SystemFactionDB(UpdatableSQLModel, table=True):
+    __tablename__ = "system_factions"  # type: ignore
+    system_faction_id: int | None = Field(
+        default=None, primary_key=True, sa_type=BigInteger
+    )
+    system_id: int = Field(
+        foreign_key="systems.system_id", index=True, sa_type=BigInteger
+    )
+    faction_name: str
+    influence: float
+    faction_states: list["FactionStateDB"] = Relationship(
+        back_populates="system_faction", cascade_delete=True
+    )
+    system: SystemDB = Relationship(back_populates="system_factions")
+
+
+class SystemPowerDB(UpdatableSQLModel, table=True):
+    __tablename__ = "system_powers"  # type: ignore
+    system_power_id: int | None = Field(
+        default=None, primary_key=True, sa_type=BigInteger
+    )
+    system_id: int = Field(
+        foreign_key="systems.system_id", index=True, sa_type=BigInteger
+    )
+    power: str
+    system: SystemDB = Relationship(back_populates="system_powers")
+
+
+class SystemFactionHistoryDB(SQLModel, table=True):
+    __tablename__ = "system_factions_history"  # type: ignore
+    system_faction_history_id: int | None = Field(
+        default=None, primary_key=True, sa_type=BigInteger
+    )
+    system_id: int = Field(
+        foreign_key="systems.system_id", index=True, sa_type=BigInteger
+    )
+    faction_name: str
+    influence: float
+    states: str | None = Field(default=None, sa_type=String(1024))
+    update_datetime: dt.datetime = Field(
+        sa_column=Column(TIMESTAMP(timezone=True), nullable=False)
+    )
+    system: SystemDB = Relationship(back_populates="system_factions_history")
+
+
+class FactionStateDB(SQLModel, table=True):
+    __tablename__ = "faction_states"  # type: ignore
+    faction_state_id: int | None = Field(
+        default=None, primary_key=True, sa_type=BigInteger
+    )
+    system_faction_id: int = Field(
+        foreign_key="system_factions.system_faction_id", index=True, sa_type=BigInteger
+    )
+    state_phase: str
+    state: str
+    system_faction: SystemFactionDB = Relationship(back_populates="faction_states")
+
+
+class StationDB(UpdatableSQLModel, table=True):
+    __tablename__ = "stations"  # type: ignore
+    station_id: int | None = Field(default=None, primary_key=True, sa_type=BigInteger)
+    system_id: int = Field(
+        foreign_key="systems.system_id", index=True, sa_type=BigInteger
+    )
+    station_name: str
+    distance_from_star: float
+    market_id: int = Field(unique=True, index=True, sa_type=BigInteger)
+    station_type: str
+    is_planetary: bool
+    max_landing_pad_size: str
+    primary_economy: str
+    secondary_economy: str | None = Field(default=None)
+    station_faction_name: str = Field(index=True)
+    update_datetime: dt.datetime = Field(
+        sa_column=Column(TIMESTAMP(timezone=True), nullable=False)
+    )
+    system: SystemDB = Relationship(back_populates="stations")
+    market_commodities: list["MarketCommodityDB"] = Relationship(
+        back_populates="station", cascade_delete=True
+    )
+    market_update_datetime: dt.datetime | None = Field(
+        default=None, sa_column=Column(TIMESTAMP(timezone=True))
+    )
+
+
+class MarketCommodityDB(UpdatableSQLModel, table=True):
+    __tablename__ = "market_commodities"  # type: ignore
+    market_commodity_id: int | None = Field(
+        default=None, primary_key=True, sa_type=BigInteger
+    )
+    station_id: int = Field(
+        foreign_key="stations.station_id", index=True, sa_type=BigInteger
+    )
+    name: str
+    mean_price: int
+    buy_price: int
+    stock: int
+    sell_price: int
+    demand: int
+    station: StationDB = Relationship(back_populates="market_commodities")

edfh_data/db/utils.py

@@ -0,0 +1,47 @@
+import os
+from pathlib import Path
+from typing import Mapping
+
+from sqlmodel import create_engine
+from sqlmodel import SQLModel
+
+if Path(".env").is_file():
+    from dotenv import load_dotenv
+
+    load_dotenv()
+
+
+def db_connect_string():
+    """Generate the DB connection string from different environment variables."""
+    user = os.getenv("DB_USER", "dbuser")
+    passwd = os.getenv("DB_PASSWD")
+    host = os.getenv("DB_HOST", "localhost")
+    port = os.getenv("DB_PORT", 3306)
+    name = os.getenv("DB_NAME")
+
+    return f"mysql+pymysql://{user}:{passwd}@{host}:{port}/{name}"
+
+
+engine = create_engine(db_connect_string())
+
+
+def init_db():
+    SQLModel.metadata.create_all(engine)
+
+
+def update_entry_fields(entry: SQLModel, update: Mapping) -> SQLModel:
+    for field, value in update.items():
+        if field in entry.__class__.model_fields:
+            setattr(entry, field, value)
+    return entry
+
+
+def exclude_none(data):
+    return {k: v for k, v in data.items() if v is not None}
+
+
+def common_keys_equal(first: Mapping, second: Mapping) -> bool:
+    """Compares the values of the common keys in two dictionaries. Returns True if all
+    values are equal, False otherwise."""
+    common_keys = set(first.keys()) & set(second.keys())
+    return all(first[k] == second[k] for k in common_keys)

edfh_data/eddn/handlers.py

@@ -0,0 +1,147 @@
+import datetime as dt
+import os
+from typing import Mapping
+import zlib
+
+import orjson
+import pika
+from sqlalchemy.exc import OperationalError
+
+from edfh_data.db.crud import create_update_station
+from edfh_data.db.crud import create_update_station_market
+from edfh_data.db.crud import create_update_system
+from edfh_data.db.utils import init_db
+from edfh_data.exceptions import EventTooOldError
+from edfh_data.models import construction_station_types
+from edfh_data.models import Station
+from edfh_data.models import StationMarket
+from edfh_data.models import System
+
+JOURNAL_V1_SCHEMA = "https://eddn.edcd.io/schemas/journal/1"
+COMMODITY_V3_SCHEMA = "https://eddn.edcd.io/schemas/commodity/3"
+
+
+def handle_journal_v1_fsdjump(message: Mapping) -> None:
+    """Handle the journal/1 messages with event==FSDJump."""
+
+    system = System(**message)
+
+    create_update_system(system)
+
+
+def handle_journal_v1_docked(message: Mapping) -> None:
+    """Handle the journal/1 message with event==Docked."""
+    # Ignore colonisation ships & construction sites
+    if (
+        message["StationType"] in construction_station_types
+        or "colonisationship" in message["StationName"].lower()
+    ):
+        return
+
+    station = Station(**message)
+
+    create_update_station(station)
+
+
+def handle_commodity_v3(message: Mapping) -> None:
+    """Handle the commodity/3 message."""
+
+    market = StationMarket(**message)
+
+    create_update_station_market(market)
+    pass
+
+
+def handle_eddn_event(
+    event: dict, max_age: dt.timedelta = dt.timedelta(hours=1)
+) -> bool:
+    """Handle any eddn event."""
+    handled = False
+    message = event["message"]
+
+    now = dt.datetime.now(tz=dt.timezone.utc)
+    message_datetime = dt.datetime.fromisoformat(message["timestamp"]).replace(
+        tzinfo=dt.timezone.utc
+    )
+    messag_age = now - message_datetime
+
+    if messag_age > max_age:
+        raise EventTooOldError(
+            f"The event timestamp is too old: {message['timestamp']}"
+        )
+
+    if event["$schemaRef"] == JOURNAL_V1_SCHEMA and message.get("event") == "FSDJump":
+        handle_journal_v1_fsdjump(message)
+        handled = True
+
+    elif event["$schemaRef"] == JOURNAL_V1_SCHEMA and message.get("event") == "Docked":
+        handle_journal_v1_docked(message)
+        handled = True
+
+    elif event["$schemaRef"] == COMMODITY_V3_SCHEMA:
+        handle_commodity_v3(message)
+        handled = True
+
+    return handled
+
+
+def run():
+
+    init_db()
+
+    RMQ_EXCHANGE_NAME = "eddn_raw"
+    RMQ_QUEUE_NAME = "eddn_raw_queue"
+
+    rmq_connection = pika.BlockingConnection(
+        pika.ConnectionParameters(
+            host=os.getenv("RMQ_HOST", "localhost"),
+            credentials=pika.PlainCredentials(
+                username=os.getenv("RMQ_USER", "guest"),
+                password=os.getenv("RMQ_PASSWD", "guest"),
+            ),
+        )
+    )
+
+    rmq_channel = rmq_connection.channel()
+    rmq_channel.exchange_declare(exchange=RMQ_EXCHANGE_NAME, exchange_type="fanout")
+
+    rmq_channel.queue_declare(queue=RMQ_QUEUE_NAME, durable=True)
+    rmq_channel.queue_bind(exchange=RMQ_EXCHANGE_NAME, queue=RMQ_QUEUE_NAME)
+
+    def callback(ch, method, properties, body):
+        try:
+            event_json = zlib.decompress(body)
+            event = orjson.loads(event_json)
+            print(f" [x] Received {event["$schemaRef"]}")
+
+            handle_eddn_event(event, max_age=dt.timedelta(days=7))
+
+            ch.basic_ack(delivery_tag=method.delivery_tag)
+
+        except OperationalError as e:
+            if e.orig and e.orig.args[0] == 1020:
+                # OperationalError(1020, "Record has changed since last read in
+                # table 'systems'")
+                # Do not aknowledge and requeue for re-processing
+                ch.basic_nack(delivery_tag=method.delivery_tag, requeue=True)
+            else:
+                ch.basic_ack(delivery_tag=method.delivery_tag)
+                print(e)
+
+        except Exception as e:
+            ch.basic_ack(delivery_tag=method.delivery_tag)
+            print(e)
+
+    rmq_channel.basic_qos(prefetch_count=int(os.getenv("RMQ_HANDLER_PREFETCH", 1)))
+    rmq_channel.basic_consume(queue=RMQ_QUEUE_NAME, on_message_callback=callback)
+
+    try:
+        rmq_channel.start_consuming()
+    except KeyboardInterrupt:
+        rmq_channel.stop_consuming()
+
+    rmq_connection.close()
+
+
+if __name__ == "__main__":
+    run()

edfh_data/eddn/listener.py

@@ -0,0 +1,96 @@
+import os
+from pathlib import Path
+import sys
+import time
+import zlib
+
+import orjson
+import pika
+import zmq
+
+
+def run():
+
+    if Path(".env").is_file():
+        from dotenv import load_dotenv
+
+        load_dotenv()
+
+    EDDN_RELAY_URL = "tcp://eddn.edcd.io:9500"
+    EDDN_SOCKET_TIMEOUT = 600000
+    RMQ_EXCHANGE_NAME = "eddn_raw"
+
+    context = zmq.Context()
+    subscriber = context.socket(zmq.SUB)
+
+    subscriber.setsockopt(zmq.SUBSCRIBE, b"")
+    subscriber.setsockopt(zmq.RCVTIMEO, EDDN_SOCKET_TIMEOUT)
+
+    rmq_credentials = pika.PlainCredentials(
+        username=os.getenv("RMQ_USER", "guest"),
+        password=os.getenv("RMQ_PASSWD", "guest"),
+    )
+    rmq_connect_params = pika.ConnectionParameters(
+        host=os.getenv("RMQ_HOST", "localhost"),
+        credentials=rmq_credentials,
+        heartbeat=20,
+    )
+
+    while True:
+        try:
+            subscriber.connect(EDDN_RELAY_URL)
+
+            rmq_connection = None
+            rmq_connection = pika.BlockingConnection(rmq_connect_params)
+            rmq_channel = rmq_connection.channel()
+            rmq_channel.exchange_declare(
+                exchange=RMQ_EXCHANGE_NAME, exchange_type="fanout"
+            )
+
+            while True:
+                try:
+                    event_raw = subscriber.recv()
+
+                    if event_raw is False:
+                        subscriber.disconnect(EDDN_RELAY_URL)
+                        break
+
+                    rmq_channel.basic_publish(
+                        exchange=RMQ_EXCHANGE_NAME,
+                        routing_key="",
+                        body=event_raw,
+                        properties=pika.BasicProperties(
+                            delivery_mode=pika.DeliveryMode.Persistent
+                        ),
+                    )
+
+                    event_json = zlib.decompress(event_raw)
+                    event = orjson.loads(event_json)
+                    print(f" [x] Published {event.get("$schemaRef")}")
+
+                except KeyError as e:
+                    print(e)
+
+        except KeyboardInterrupt:
+            subscriber.disconnect(EDDN_RELAY_URL)
+            rmq_connection.close()
+            sys.exit()
+
+        except Exception as e:
+            print(repr(e))
+            try:
+                subscriber.disconnect(EDDN_RELAY_URL)
+            except Exception:
+                pass
+
+            try:
+                if rmq_connection and rmq_connection.is_open:
+                    rmq_connection.close()
+            except Exception:
+                pass
+
+            time.sleep(5)
+
+
+if __name__ == "__main__":
+    run()

edfh_data/exceptions.py

@@ -0,0 +1,11 @@
+class RejectedEventError(ValueError):
+    """Top level exception for EDDN events that pass validation but are rejected for
+    other reasons."""
+
+    pass
+
+
+class EventTooOldError(RejectedEventError):
+    """The EDDN event timestamp is too old."""
+
+    pass

edfh_data/models.py

@@ -0,0 +1,292 @@
+import datetime as dt
+from typing import Annotated, Literal, Self
+
+from fdev_ids import load_table
+from pydantic import AfterValidator
+from pydantic import AliasPath
+from pydantic import BaseModel
+from pydantic import BeforeValidator
+from pydantic import Field
+from pydantic import model_validator
+
+economy_ids = load_table("economy")
+security_ids = load_table("security")
+government_ids = load_table("government")
+happiness_ids = load_table("happiness")
+factionstate_ids = load_table("factionstate")
+commodity_symbols = {c["symbol"].lower(): c for c in load_table("commodity").values()}
+rare_commodity_symbols = {
+    c["symbol"].lower(): c for c in load_table("rare_commodity").values()
+}
+
+planetary_station_types = (
+    "CraterOutpost",
+    "CraterPort",
+    "OnFootSettlement",
+    "SurfaceStation",
+)
+
+construction_station_types = (
+    "PlanetaryConstructionDepot",
+    "SpaceConstructionDepot",
+)
+
+stronghold_carrier_names = (
+    "Stronghold Carrier",
+    "Hochburg-Carrier",
+    "Porte-vaisseaux de forteresse",
+    "Portanaves bastión",
+    "Носитель-база",
+    "Transportadora da potência",
+)
+
+
+def replace_none(value: str | None) -> str | None:
+    """Replace 'None' string and empty string values with None."""
+    if value in ("None", ""):
+        return None
+    return value
+
+
+def map_economy(economy_id: str) -> str | None:
+    return replace_none(economy_ids.get(economy_id))
+
+
+def map_security(security_id: str) -> str | None:
+    return security_ids.get(security_id)
+
+
+def map_government(government_id: str) -> str | None:
+    return replace_none(government_ids.get(government_id))
+
+
+def map_happiness(happiness_id: str) -> str | None:
+    return happiness_ids.get(happiness_id)
+
+
+def map_faction_state(factionstate_id: str) -> str | None:
+    return replace_none(factionstate_ids.get(factionstate_id))
+
+
+def map_commodity(commodity_symbol: str) -> str | None:
+    all_commodities = commodity_symbols | rare_commodity_symbols
+    return all_commodities[commodity_symbol.lower()]["name"]
+
+
+class FactionState(BaseModel):
+    """A state of a faction in a system."""
+
+    state_phase: Literal["pending", "active", "recovering"]
+    state: Annotated[str, BeforeValidator(map_faction_state)]
+
+
+class SystemFaction(BaseModel):
+    """Status of a faction in a star system.
+
+    Aliases are defined so that an instance can be created from a record in the
+    'Factions' field of the message of a journal/1 event with `event=FSDJump`
+    """
+
+    faction_name: str = Field(alias="Name")
+    allegiance: str = Field(alias="Allegiance")
+    faction_state: str = Field(alias="FactionState")
+    government: str = Field(alias="Government")
+    Happiness: Annotated[str | None, BeforeValidator(map_happiness)] = Field(
+        default=None, alias="Happiness"
+    )
+    influence: float = Field(alias="Influence")
+    states: list[FactionState] = Field(default_factory=list)
+
+    @model_validator(mode="before")
+    @classmethod
+    def map_states(cls, data: dict) -> dict:
+        pending = [
+            FactionState(state=s["State"], state_phase="pending")
+            for s in data.get("PendingStates", list())
+        ]
+        active = [
+            FactionState(state=s["State"], state_phase="active")
+            for s in data.get("ActiveStates", list())
+        ]
+        recovering = [
+            FactionState(state=s["State"], state_phase="recovering")
+            for s in data.get("RecoveringStates", list())
+        ]
+        data["states"] = pending + active + recovering
+
+        return data
+
+
+def influence_total_is_one(system_factions: list[SystemFaction]) -> list[SystemFaction]:
+    """Verify that the sum of faction influences is one (within 0.00001)."""
+    is_one = 1.00001 >= sum(f.influence for f in system_factions) >= 0.99999
+
+    if not is_one:
+        raise ValueError("Sum of faction influences must be one")
+
+    return system_factions
+
+
+class System(BaseModel):
+    """Star system.
+
+    Aliases are defined so that an instance can be created from the message of a
+    journal/1 event with `event=FSDJump`.
+    """
+
+    system_name: str = Field(alias="StarSystem")
+    system_address: int = Field(alias="SystemAddress")
+    population: int = Field(alias="Population")
+    x: float = Field(validation_alias=AliasPath("StarPos", 0))
+    y: float = Field(validation_alias=AliasPath("StarPos", 1))
+    z: float = Field(validation_alias=AliasPath("StarPos", 2))
+    update_datetime: dt.datetime = Field(alias="timestamp")
+    system_allegiance: Annotated[str | None, AfterValidator(replace_none)] = Field(
+        default=None, alias="SystemAllegiance"
+    )
+    controlling_faction: Annotated[str | None, AfterValidator(replace_none)] = Field(
+        default=None, validation_alias=AliasPath("SystemFaction", "Name")
+    )
+    controlling_power: Annotated[str | None, AfterValidator(replace_none)] = Field(
+        default=None, alias="ControllingPower"
+    )
+    powerplay_state: Annotated[str | None, AfterValidator(replace_none)] = Field(
+        default=None, alias="PowerplayState"
+    )
+    powerplay_state_control_progress: float | None = Field(
+        default=None, alias="PowerplayStateControlProgress"
+    )
+    powerplay_state_reinforcement: int | None = Field(
+        default=None, alias="PowerplayStateReinforcement"
+    )
+    powerplay_state_undermining: int | None = Field(
+        default=None, alias="PowerplayStateUndermining"
+    )
+    primary_economy: Annotated[str | None, BeforeValidator(map_economy)] = Field(
+        default=None, alias="SystemEconomy"
+    )
+    secondary_economy: Annotated[str | None, BeforeValidator(map_economy)] = Field(
+        default=None, alias="SystemSecondEconomy"
+    )
+    security: Annotated[str | None, BeforeValidator(map_security)] = Field(
+        default=None, alias="SystemSecurity"
+    )
+    government: Annotated[str | None, BeforeValidator(map_government)] = Field(
+        default=None, alias="SystemGovernment"
+    )
+    system_factions: Annotated[
+        list[SystemFaction], AfterValidator(influence_total_is_one)
+    ] = Field(default_factory=list, alias="Factions")
+    system_powers: list[str] = Field(default_factory=list, alias="Powers")
+
+
+def construction_type_not_allowed(station_type: str) -> str:
+    """Raise an exception if the station type is a construction depot."""
+    if station_type in construction_station_types:
+        raise ValueError("Construction site not allowed as station")
+    return station_type
+
+
+def colonisation_ship_not_allowed(station_name: str) -> str:
+    """Raise an exception if the station name indicates a colonisation ship."""
+    if "colonisationship" in station_name.lower():
+        raise ValueError("Colonisation ship not allowed as station")
+    return station_name
+
+
+class Station(BaseModel):
+    """Station.
+
+    Aliases are defined so that an instance can be created from the message of a
+    journal/1 event with `event=Docked`.
+    """
+
+    system_name: str = Field(alias="StarSystem")
+    station_name: Annotated[str, AfterValidator(colonisation_ship_not_allowed)] = Field(
+        alias="StationName"
+    )
+    distance_from_star: float = Field(alias="DistFromStarLS")
+    market_id: int | None = Field(default=None, alias="MarketID")
+    station_type: Annotated[str, AfterValidator(construction_type_not_allowed)] = Field(
+        alias="StationType"
+    )
+    is_planetary: bool = Field(default=False)
+    max_landing_pad_size: Literal["L", "M", "S"]
+    primary_economy: Annotated[str, BeforeValidator(map_economy)] = Field(
+        alias="StationEconomy"
+    )
+    secondary_economy: str | None = Field(default=None)
+    station_faction_name: str = Field(
+        validation_alias=AliasPath("StationFaction", "Name")
+    )
+    update_datetime: dt.datetime = Field(alias="timestamp")
+
+    @model_validator(mode="before")
+    @classmethod
+    def _secondary_economy(cls, data: dict) -> dict:
+        if len(data["StationEconomies"]) >= 2:
+            data["secondary_economy"] = map_economy(data["StationEconomies"][1]["Name"])
+        else:
+            data["secondary_economy"] = None
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def _is_planetary(cls, data: dict) -> dict:
+        data["is_planetary"] = data["StationType"] in planetary_station_types
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def _max_landing_pad_size(cls, data: dict) -> dict:
+        pads = data["LandingPads"]
+        if pads.get("Large", 0) > 0:
+            size = "L"
+        elif pads.get("Medium", 0) > 0:
+            size = "M"
+        else:
+            size = "S"
+        data["max_landing_pad_size"] = size
+        return data
+
+    @model_validator(mode="after")
+    def update_stronghold_carrier(self) -> Self:
+        if self.station_name in stronghold_carrier_names:
+            self.station_name = "Stronghold Carrier"
+            self.station_type = "StrongholdCarrier"
+            self.is_planetary = False
+        return self
+
+
+class Commodity(BaseModel):
+    """A single commodity.
+
+    Aliases are defined so that an instance can be created from a record in the
+    commodities field of the commodity/3 event message.
+    """
+
+    name: Annotated[str, BeforeValidator(map_commodity)]
+    mean_price: int = Field(alias="meanPrice")
+    buy_price: int = Field(alias="buyPrice")
+    stock: int
+    sell_price: int = Field(alias="sellPrice")
+    demand: int
+
+
+def exclude_zero_commodities(commodities: list[Commodity]) -> list[Commodity]:
+    """Exclude commodities with zero supply & demand."""
+    return [c for c in commodities if not all([c.stock == 0, c.demand == 0])]
+
+
+class StationMarket(BaseModel):
+    """List of commodities in a station/market.
+
+    Aliases are defined so that an instance can be created the message of the
+    commodity/3 event.
+    """
+
+    market_id: int = Field(alias="marketId")
+    commodities: Annotated[
+        list[Commodity], AfterValidator(exclude_zero_commodities)
+    ] = Field(default_factory=list, alias="commodities")
+    update_datetime: dt.datetime = Field(alias="timestamp")

edfh_data-0.5.2.dist-info/METADATA

@@ -0,0 +1,123 @@
+Metadata-Version: 2.4
+Name: edfh-data
+Version: 0.5.2
+Summary: Python library for the E:D Faction Hub application backend data services
+License-Expression: MIT
+Project-URL: Homepage, https://codeberg.org/jdlbt/edfh-data
+Project-URL: Issue tracker, https://codeberg.org/jdlbt/edfh-data/issues
+Project-URL: Source, https://codeberg.org/jdlbt/edfh-data
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: fdev-ids
+Requires-Dist: pydantic
+Provides-Extra: db
+Requires-Dist: PyMySQL; extra == "db"
+Requires-Dist: sqlmodel; extra == "db"
+Provides-Extra: eddn
+Requires-Dist: edfh-data[db]; extra == "eddn"
+Requires-Dist: orjson; extra == "eddn"
+Requires-Dist: pika; extra == "eddn"
+Requires-Dist: zmq; extra == "eddn"
+Provides-Extra: dev
+Requires-Dist: edfh-data[db,eddn]; extra == "dev"
+Requires-Dist: alembic; extra == "dev"
+Requires-Dist: black==25.9.0; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: flake8==7.3.0; extra == "dev"
+Requires-Dist: isort==6.1.0; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: python-dotenv; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: license-file
+
+# edfh-data
+
+Python library for the E:D Faction Hub application backend data services:
+
+- EDDN listener service, receives messages from the EDDN stream and publishes them to a RabbitMQ exchange.
+- EDDN handler service, polls message from a RabbitMQ queue, parses them and stores the relevant info in a relational database.
+
+## Requirements
+
+- Python 3.13 available in the sytem path.
+- Running RabbitMQ and MariaDB instances (the included `docker-compose.yaml` file can set them up for development).
+
+## Configuration
+
+The following configuration variables are defined are defined as environment variables, or, for local development, specified in a `.env` configuration file:
+
+```ini
+DB_USER=dbuser
+DB_PASSWD=<db_password>
+DB_HOST=localhost
+DB_PORT=3306
+DB_NAME=fgs_infra_db
+RMQ_USER=rmquser
+RMQ_PASSWD=<rabbitmq_password>
+RMQ_HOST=localhost
+RMQ_HANDLER_PREFETCH=10
+```
+
+## Usage
+
+Install the package with the `eddn` extra:
+
+```
+python -m pip install edfh-data[eddn]
+```
+
+Launch the EDDN message handler service with:
+
+```
+eddn-handle
+```
+
+Launch the EDDN listener service with:
+
+```
+eddn-listen
+```
+
+## Development
+
+### Python environment
+
+Create a virtual environment using Python 3.13:
+
+```
+python3.13 -m venv .venv --prompt edfh-data --upgrade-deps
+source .venv/bin/activate
+```
+
+Install the package in development mode along with the development dependencies:
+
+```
+python -m pip install -e .[dev]
+```
+
+Install pre-commit hooks:
+
+```
+pre-commit install
+```
+
+### Database service
+
+Migrate the database with:
+
+```
+alembic upgrade head
+```
+
+Generate a new revision:
+
+```
+alembic revision --autogenerate -m "Revision message"
+```

edfh_data-0.5.2.dist-info/RECORD

@@ -0,0 +1,16 @@
+edfh_data/__init__.py,sha256=isJrmDBLRag7Zc2UK9ZovWGOv7ji1Oh-zJtJMNJFkXw,22
+edfh_data/exceptions.py,sha256=AxBIxYYBciw17UrGspx2_r95ZBwNgBs7Q3Jz-tRdIq8,258
+edfh_data/models.py,sha256=otcDn_G8by1PZDyhlxuXBVVkudZVWetWHA2x6yYXQ04,10051
+edfh_data/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+edfh_data/db/crud.py,sha256=0vFWX2raeo1gzmu4BjHrX10h1HK81iOJP4WCc_z6ZsY,7786
+edfh_data/db/models.py,sha256=3W9zI7S_Jda4vXK5-Cth7vQC-xcu32yEheYpFBrKe8w,6000
+edfh_data/db/utils.py,sha256=lc403cOPDkbTbfGjMtf0V0WNLpj8j5scMkqZzNEyTJQ,1302
+edfh_data/eddn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+edfh_data/eddn/handlers.py,sha256=hvJPbdPvThIOadoZhlw44G4r0PxuA7-kV8-7FrDsfuA,4391
+edfh_data/eddn/listener.py,sha256=-yGCrwiYWGrZSwS5JpyOORXNbsfTA1fDs0M-nTYt4uk,2592
+edfh_data-0.5.2.dist-info/licenses/LICENSE,sha256=U1KpKZcXdpnNNT4Xh5cA-aRMvC0eaDEu_azpC70VT_I,1062
+edfh_data-0.5.2.dist-info/METADATA,sha256=2fvxWecaFZ7CEwJ7D-d5FCfeLSUQX219XhA-ORsK4G8,3108
+edfh_data-0.5.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+edfh_data-0.5.2.dist-info/entry_points.txt,sha256=NeJ61ITr5YFhdh-p2akshiPQF8LI9530iUD4SYko_CU,102
+edfh_data-0.5.2.dist-info/top_level.txt,sha256=RsVo4mZIyr5Q_PZQ_-ix9ec1FHhvXra_8c-HIZA2nK8,10
+edfh_data-0.5.2.dist-info/RECORD,,

edfh_data-0.5.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

edfh_data-0.5.2.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+eddn-handle = edfh_data.eddn.handlers:run
+eddn-listen = edfh_data.eddn.listener:run

edfh_data-0.5.2.dist-info/licenses/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2025 jdlbt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

edfh_data-0.5.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+edfh_data