pyopensky 2.4__tar.gz → 2.6__tar.gz

{pyopensky-2.4 → pyopensky-2.6}/PKG-INFO

@@ -1,7 +1,8 @@
 Metadata-Version: 2.1
 Name: pyopensky
-Version: 2.4
+Version: 2.6
 Summary: A Python interface for OpenSky database
+Home-page: https://github.com/open-aviation/pyopensky/
 License: MIT
 Author: Xavier Olive
 Author-email: git@xoolive.org
@@ -21,13 +22,14 @@ Requires-Dist: pyjwt (>=2.7.0)
 Requires-Dist: tqdm (>=4.65.0)
 Requires-Dist: trino[sqlalchemy] (>=0.322)
 Requires-Dist: typing-extensions (>=4.5.0)
+Project-URL: Repository, https://github.com/open-aviation/pyopensky/
 Description-Content-Type: text/markdown
 
 # pyopensky
 
 The `pyopensky` Python library provides functions to download data from the OpenSky Network live API and historical databases. It aims at making ADS-B and Mode S data from OpenSky easily accessible in the Python programming environment.
 
-Full documentation on https://open-aviation.github.io/pyopensky
+Full documentation on [https://open-aviation.github.io/pyopensky](https://open-aviation.github.io/pyopensky)
 
 ## Installation
 
@@ -55,13 +57,16 @@ See details in the [documentation](https://open-aviation.github.io/pyopensky/cre
 
 ## Usage
 
+> [!IMPORTANT]
+> The Impala shell is now deprecated. Please upgrade to Trino.
+
 - from the [REST API](https://open-aviation.github.io/pyopensky/rest.html):
 
   ```python
   from pyopensky.rest import REST
-  
+
   rest = REST()
-  
+
   rest.states()
   rest.tracks(icao24)
   rest.routes(callsign)
@@ -69,25 +74,16 @@ See details in the [documentation](https://open-aviation.github.io/pyopensky/cre
   rest.arrival(airport, begin, end)
   rest.departure(airport, begin, end)
   ```
-- from the [Impala](https://open-aviation.github.io/pyopensky/impala.html) shell (requires authentication):
 
-  ```python
-  from pyopensky.impala import Impala
-  
-  impala = Impala()
-  # full description of the whole set of parameters in the documentation
-  impala.flightlist(start, stop, *, airport, callsign, icao24)
-  impala.history(start, stop, *, callsign, icao24, bounds)
-  impala.rawdata(start, stop, *, callsign, icao24, bounds)
-  ```
 - from the [Trino](https://open-aviation.github.io/pyopensky/trino.html) database (requires authentication):
 
   ```python
   from pyopensky.trino import Trino
-  
+
   trino = Trino()
   # full description of the whole set of parameters in the documentation
   trino.flightlist(start, stop, *, airport, callsign, icao24)
   trino.history(start, stop, *, callsign, icao24, bounds)
   trino.rawdata(start, stop, *, callsign, icao24, bounds)
   ```
+

{pyopensky-2.4 → pyopensky-2.6}/pyproject.toml

@@ -1,7 +1,8 @@
 [tool.poetry]
 name = "pyopensky"
-version = "2.4"
+version = "2.6"
 description = "A Python interface for OpenSky database"
+repository = "https://github.com/open-aviation/pyopensky/"
 authors = ["Xavier Olive <git@xoolive.org>", "Junzi Sun <j.sun-1@tudelft.nl>"]
 license = "MIT"
 readme = "readme.md"

{pyopensky-2.4 → pyopensky-2.6}/readme.md

@@ -2,7 +2,7 @@
 
 The `pyopensky` Python library provides functions to download data from the OpenSky Network live API and historical databases. It aims at making ADS-B and Mode S data from OpenSky easily accessible in the Python programming environment.
 
-Full documentation on https://open-aviation.github.io/pyopensky
+Full documentation on [https://open-aviation.github.io/pyopensky](https://open-aviation.github.io/pyopensky)
 
 ## Installation
 
@@ -30,13 +30,16 @@ See details in the [documentation](https://open-aviation.github.io/pyopensky/cre
 
 ## Usage
 
+> [!IMPORTANT]
+> The Impala shell is now deprecated. Please upgrade to Trino.
+
 - from the [REST API](https://open-aviation.github.io/pyopensky/rest.html):
 
   ```python
   from pyopensky.rest import REST
-  
+
   rest = REST()
-  
+
   rest.states()
   rest.tracks(icao24)
   rest.routes(callsign)
@@ -44,25 +47,15 @@ See details in the [documentation](https://open-aviation.github.io/pyopensky/cre
   rest.arrival(airport, begin, end)
   rest.departure(airport, begin, end)
   ```
-- from the [Impala](https://open-aviation.github.io/pyopensky/impala.html) shell (requires authentication):
 
-  ```python
-  from pyopensky.impala import Impala
-  
-  impala = Impala()
-  # full description of the whole set of parameters in the documentation
-  impala.flightlist(start, stop, *, airport, callsign, icao24)
-  impala.history(start, stop, *, callsign, icao24, bounds)
-  impala.rawdata(start, stop, *, callsign, icao24, bounds)
-  ```
 - from the [Trino](https://open-aviation.github.io/pyopensky/trino.html) database (requires authentication):
 
   ```python
   from pyopensky.trino import Trino
-  
+
   trino = Trino()
   # full description of the whole set of parameters in the documentation
   trino.flightlist(start, stop, *, airport, callsign, icao24)
   trino.history(start, stop, *, callsign, icao24, bounds)
   trino.rawdata(start, stop, *, callsign, icao24, bounds)
-  ```
+  ```

{pyopensky-2.4 → pyopensky-2.6}/src/pyopensky/schema.py

@@ -67,6 +67,17 @@ class UTCTimestampFloat(TypeDecorator[pd.Timestamp]):
         return super().process_bind_param(value, dialect)
 
 
+class AddressString(TypeDecorator[str]):
+    """Automatic coercing of icao24 addresses into low characters."""
+
+    impl = String
+
+    def process_bind_param(self, value: str | None, dialect: Dialect) -> Any:
+        if isinstance(value, str):
+            return value.lower()
+        return super().process_bind_param(value, dialect)
+
+
 class CallsignString(TypeDecorator[str]):
     """Automatic coercing of callsigns into 8-char left padded strings.
 
@@ -122,7 +133,7 @@ class TrackRow:
 
 @dataclass
 class AirportCandidateRow:
-    icao24: str
+    icao24: Address
     horizdistance: int
     vertdistance: int
 
@@ -180,6 +191,7 @@ Track = List[TrackRow]
 Sensors = List[SensorRow]
 
 Callsign = Annotated[str, mapped_column(CallsignString)]
+Address = Annotated[str, mapped_column(AddressString)]
 
 
 class Base(DeclarativeBase):
@@ -188,6 +200,7 @@ class Base(DeclarativeBase):
         AirportCandidates: AirportCandidateType,
         Track: TrackType,
         Callsign: CallsignString,
+        Address: AddressString,
         Sensors: SensorsType,
     }
 
@@ -196,7 +209,7 @@ class StateVectorsData4(Base):
     __tablename__ = "state_vectors_data4"
 
     time: Mapped[pd.Timestamp]
-    icao24: Mapped[str]
+    icao24: Mapped[Address]
     lat: Mapped[float]
     lon: Mapped[float]
     velocity: Mapped[float]
@@ -218,7 +231,7 @@ class StateVectorsData4(Base):
 class FlightsData4(Base):
     __tablename__ = "flights_data4"
 
-    icao24: Mapped[str]
+    icao24: Mapped[Address]
     firstseen: Mapped[pd.Timestamp]
     estdepartureairport: Mapped[str]
     lastseen: Mapped[pd.Timestamp]
@@ -243,7 +256,7 @@ class FlightsData4(Base):
 class FlightsData5(Base):
     __tablename__ = "flights_data5"
 
-    icao24: Mapped[str]
+    icao24: Mapped[Address]
     firstseen: Mapped[pd.Timestamp]
     estdepartureairport: Mapped[str]
     lastseen: Mapped[pd.Timestamp]
@@ -280,17 +293,7 @@ class RawTable(Protocol):
     mintime: Mapped[pd.Timestamp]
     maxtime: Mapped[pd.Timestamp]
     msgcount: Mapped[int]
-    icao24: Mapped[str]
-    message: Mapped[str]
-    isid: Mapped[bool]
-    flightstatus: Mapped[int]
-    downlinkrequest: Mapped[int]
-    utilitymsg: Mapped[int]
-    interrogatorid: Mapped[int]
-    identifierdesignator: Mapped[int]
-    valuecode: Mapped[int]
-    altitude: Mapped[float]
-    identity: Mapped[str]
+    icao24: Mapped[Address]
 
     # Whatever, we pick this one as primary key BUT
     #   - this is not true
@@ -306,7 +309,7 @@ class AcasData4(Base):
     mintime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     maxtime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     msgcount: Mapped[int]
-    icao24: Mapped[str]
+    icao24: Mapped[Address]
     message: Mapped[str]
     isid: Mapped[bool]
     flightstatus: Mapped[int]
@@ -332,7 +335,7 @@ class AllcallRepliesData4(Base):
     mintime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     maxtime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     msgcount: Mapped[int]
-    icao24: Mapped[str]
+    icao24: Mapped[Address]
     message: Mapped[str]
     isid: Mapped[bool]
     flightstatus: Mapped[int]
@@ -358,16 +361,9 @@ class IdentificationData4(Base):
     mintime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     maxtime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     msgcount: Mapped[int]
-    icao24: Mapped[str]
-    message: Mapped[str]
-    isid: Mapped[bool]
-    flightstatus: Mapped[int]
-    downlinkrequest: Mapped[int]
-    utilitymsg: Mapped[int]
-    interrogatorid: Mapped[int]
-    identifierdesignator: Mapped[int]
-    valuecode: Mapped[int]
-    altitude: Mapped[float]
+    icao24: Mapped[Address]
+    emittercategory: Mapped[int]
+    ftc: Mapped[int]
     identity: Mapped[str]
 
     # Whatever, we pick this one as primary key BUT
@@ -385,16 +381,33 @@ class OperationalStatusData4(Base):
     maxtime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     msgcount: Mapped[int]
     icao24: Mapped[str]
-    message: Mapped[str]
-    isid: Mapped[bool]
-    flightstatus: Mapped[int]
-    downlinkrequest: Mapped[int]
-    utilitymsg: Mapped[int]
-    interrogatorid: Mapped[int]
-    identifierdesignator: Mapped[int]
-    valuecode: Mapped[int]
-    altitude: Mapped[float]
-    identity: Mapped[str]
+    subtypecode: Mapped[int]
+    unknowncapcode: Mapped[bool]
+    unknownopcode: Mapped[bool]
+    hasoperationaltcas: Mapped[int]
+    has1090esin: Mapped[bool]
+    supportsairreferencedvelocity: Mapped[int]
+    haslowtxpower: Mapped[int]
+    supportstargetstatereport: Mapped[int]
+    supportstargetchangereport: Mapped[int]
+    hasuatin: Mapped[bool]
+    nacv: Mapped[int]
+    nicsupplementc: Mapped[int]
+    hastcasresolutionadvisory: Mapped[bool]
+    hasactiveidentswitch: Mapped[bool]
+    usessingleantenna: Mapped[bool]
+    systemdesignassurance: Mapped[int]
+    gpsantennaoffset: Mapped[int]
+    airplanelength: Mapped[int]
+    airplanewidth: Mapped[float]
+    version: Mapped[int]
+    nicsupplementa: Mapped[bool]
+    positionnac: Mapped[float]
+    geometricverticalaccuracy: Mapped[int]
+    sourceintegritylevel: Mapped[int]
+    barometricaltitudeintegritycode: Mapped[int]
+    trackheadinginfo: Mapped[int]
+    horizontalreferencedirection: Mapped[bool]
 
     # Whatever, we pick this one as primary key BUT
     #   - this is not true
@@ -410,7 +423,7 @@ class PositionData4(Base):
     mintime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     maxtime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     msgcount: Mapped[int]
-    icao24: Mapped[str]
+    icao24: Mapped[Address]
     nicsuppla: Mapped[bool]
     hcr: Mapped[float]
     nic: Mapped[int]
@@ -442,7 +455,7 @@ class RollcallRepliesData4(Base):
     mintime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     maxtime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     msgcount: Mapped[int]
-    icao24: Mapped[str]
+    icao24: Mapped[Address]
     message: Mapped[str]
     isid: Mapped[bool]
     flightstatus: Mapped[int]
@@ -468,17 +481,48 @@ class VelocityData4(Base):
     mintime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     maxtime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
     msgcount: Mapped[int]
-    icao24: Mapped[str]
-    message: Mapped[str]
-    isid: Mapped[bool]
-    flightstatus: Mapped[int]
-    downlinkrequest: Mapped[int]
-    utilitymsg: Mapped[int]
-    interrogatorid: Mapped[int]
-    identifierdesignator: Mapped[int]
-    valuecode: Mapped[int]
-    altitude: Mapped[float]
-    identity: Mapped[str]
+    icao24: Mapped[Address]
+    supersonic: Mapped[bool]
+    intentchange: Mapped[bool]
+    ifrcapability: Mapped[bool]
+    nac: Mapped[int]
+    ewvelocity: Mapped[float]
+    nsvelocity: Mapped[float]
+    baro: Mapped[bool]
+    vertrate: Mapped[float]
+    geominurbaro: Mapped[float]  # typo confirmed
+    heading: Mapped[float]
+    velocity: Mapped[float]
+
+    # Whatever, we pick this one as primary key BUT
+    #   - this is not true
+    #   - we still need one column with a primary key
+    hour: Mapped[pd.Timestamp] = mapped_column(primary_key=True)
+
+
+class FlarmRaw(Base):
+    __tablename__ = "flarm_raw"
+
+    sensortype: Mapped[str]
+    sensorlatitude: Mapped[float]
+    sensorlongitude: Mapped[float]
+    sensoraltitude: Mapped[int]
+    timeatserver: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
+    timeatsensor: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
+    timestamp: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
+    rawmessage: Mapped[str]
+    crc: Mapped[str]
+    rawsoftmessage: Mapped[str]
+    sensorname: Mapped[str]
+    ntperror: Mapped[float]
+    userfreqcorrection: Mapped[float]
+    autofreqcorrection: Mapped[float]
+    frequency: Mapped[float]
+    channel: Mapped[int]
+    snrdetector: Mapped[float]
+    snrdemodulator: Mapped[float]
+    typeogn: Mapped[bool]
+    crccorrect: Mapped[bool]
 
     # Whatever, we pick this one as primary key BUT
     #   - this is not true

{pyopensky-2.4 → pyopensky-2.6}/src/pyopensky/trino.py

@@ -33,6 +33,7 @@ import pandas as pd
 from .api import HasBounds, OpenSkyDBAPI
 from .config import cache_path, trino_password, trino_username
 from .schema import (
+    FlarmRaw,
     FlightsData4,
     FlightsData5,
     RawTable,
@@ -119,6 +120,8 @@ class Trino(OpenSkyDBAPI):
                 if token is not None
                 else OAuth2Authentication(),
                 http_scheme="https",
+                # https://github.com/trinodb/trino-python-client/discussions/456
+                legacy_prepared_statements=True,
             ),
         )
         return engine
@@ -381,9 +384,11 @@ class Trino(OpenSkyDBAPI):
         departure_airport: None | str = None,
         arrival_airport: None | str = None,
         airport: None | str = None,
+        time_buffer: None | str | pd.Timedelta = None,
         cached: bool = True,
         compress: bool = False,
         limit: None | int = None,
+        selected_columns: tuple[InstrumentedAttribute[Any] | str, ...] = (),
         **kwargs: Any,
     ) -> None | pd.DataFrame:
         """Get Traffic from the OpenSky Trino database.
@@ -413,6 +418,10 @@ class Trino(OpenSkyDBAPI):
         :param bounds: sets a geographical footprint. Either an **airspace or
             shapely shape** (requires the bounds attribute); or a **tuple of
             float** (west, south, east, north);
+        :param selected_columns: specify the columns you want to retrieve. When
+            empty, use all columns of the StateVectorsData4 table. You may
+            escape column names as str. **Always** escape names from the
+            FlightsData4 table.
 
         **Airports**
 
@@ -428,6 +437,10 @@ class Trino(OpenSkyDBAPI):
         :param airport: a string for the ICAO identifier of the airport. Selects
             flights departing from or arriving at the airport between the two
             timestamps;
+        :param time_buffer: (default: None) time buffer used to extend time
+            bounds for flights in the OpenSky flight tables: requests will get
+            flights between ``start - time_buffer`` and ``stop + time_buffer``.
+            If no airport is specified, the parameter is ignored.
 
         .. warning::
 
@@ -475,7 +488,7 @@ class Trino(OpenSkyDBAPI):
                 )
                 .where(
                     FlightsData4.day >= start_ts.floor("1d"),
-                    FlightsData4.day <= start_ts.ceil("1d"),
+                    FlightsData4.day <= stop_ts.ceil("1d"),
                 )
             )
 
@@ -512,18 +525,26 @@ class Trino(OpenSkyDBAPI):
 
             flight_query = flight_table.subquery()
             fd4 = aliased(FlightsData4, alias=flight_query, adapt_on_names=True)
-            stmt = (
-                select(StateVectorsData4)
-                .join(
-                    flight_query,
-                    (fd4.icao24 == StateVectorsData4.icao24)
-                    & (fd4.callsign == StateVectorsData4.callsign),
-                )
-                .where(
+
+            if isinstance(time_buffer, str):
+                time_buffer = pd.Timedelta(time_buffer)
+
+            stmt = select(StateVectorsData4).join(
+                flight_query,
+                (fd4.icao24 == StateVectorsData4.icao24)
+                & (fd4.callsign == StateVectorsData4.callsign),
+            )
+
+            if time_buffer is None:
+                stmt = stmt.where(
                     StateVectorsData4.time >= fd4.firstseen,
                     StateVectorsData4.time <= fd4.lastseen,
                 )
-            )
+            else:
+                stmt = stmt.where(
+                    StateVectorsData4.time >= (fd4.firstseen - time_buffer),
+                    StateVectorsData4.time <= (fd4.lastseen + time_buffer),
+                )
 
         stmt = self.stmt_where_str(stmt, icao24, StateVectorsData4.icao24)
         stmt = self.stmt_where_str(stmt, callsign, StateVectorsData4.callsign)
@@ -568,8 +589,86 @@ class Trino(OpenSkyDBAPI):
         stmt = stmt.where(
             StateVectorsData4.time >= start_ts,
             StateVectorsData4.time <= stop_ts,
-            StateVectorsData4.hour >= start_ts.floor("1H"),
-            StateVectorsData4.hour < stop_ts.ceil("1H"),
+            StateVectorsData4.hour >= start_ts.floor("1h"),
+            StateVectorsData4.hour < stop_ts.ceil("1h"),
+        )
+
+        def transform_column(col: str) -> InstrumentedAttribute[Any]:
+            if "." in col:
+                table, column = col.split(".")
+                MATCH = {
+                    "StateVectorsData4": StateVectorsData4,
+                    "FlightsData4": fd4,
+                }
+                return getattr(MATCH[table], column)  # type: ignore
+            if candidate := getattr(StateVectorsData4, col, None):
+                return candidate  # type: ignore
+            if candidate := getattr(fd4, col, None):
+                return candidate  # type: ignore
+            raise ValueError(f"Unknown column: {col}")
+
+        if len(selected_columns) > 0:
+            columns = (
+                transform_column(col) if isinstance(col, str) else col
+                for col in selected_columns
+            )
+            stmt = stmt.with_only_columns(*columns)
+
+        if limit is not None:
+            stmt = stmt.limit(limit)
+
+        res = self.query(stmt, cached=cached, compress=compress)
+
+        if res.shape[0] == 0:
+            return None
+
+        return res
+
+    def flarm(
+        self,
+        start: timelike,
+        stop: None | timelike = None,
+        *args: ColumnExpressionArgument[bool],
+        sensor_name: None | str | list[str] = None,
+        cached: bool = True,
+        compress: bool = False,
+        limit: None | int = None,
+        correct_only: bool = True,
+        extra_columns: tuple[InstrumentedAttribute[Any], ...] = (),
+        **kwargs: Any,
+    ) -> None | pd.DataFrame:
+        start_ts = to_datetime(start)
+        stop_ts = (
+            to_datetime(stop)
+            if stop is not None
+            else start_ts + pd.Timedelta("1d")
+        )
+        stmt = select(FlarmRaw).with_only_columns(
+            FlarmRaw.sensoraltitude,
+            FlarmRaw.sensorlatitude,
+            FlarmRaw.sensorlongitude,
+            FlarmRaw.sensorname,
+            FlarmRaw.timestamp,
+            FlarmRaw.timeatserver,
+            FlarmRaw.timeatsensor,
+            FlarmRaw.rawmessage,
+            *extra_columns,
+        )
+        if correct_only:
+            stmt = stmt.where(FlarmRaw.crccorrect)
+            stmt = stmt.where(FlarmRaw.rawmessage.is_not(None))
+
+        if sensor_name is not None:
+            stmt = self.stmt_where_str(stmt, sensor_name, FlarmRaw.sensorname)
+
+        for condition in args:
+            stmt = stmt.where(condition)
+
+        stmt = stmt.where(
+            FlarmRaw.timestamp >= start_ts,
+            FlarmRaw.timestamp <= stop_ts,
+            FlarmRaw.hour >= start_ts.floor("1h"),
+            FlarmRaw.hour < stop_ts.ceil("1h"),
         )
 
         if limit is not None:
@@ -712,7 +811,7 @@ class Trino(OpenSkyDBAPI):
                 )
                 .where(
                     FlightsData4.day >= start_ts.floor("1d"),
-                    FlightsData4.day <= start_ts.ceil("1d"),
+                    FlightsData4.day <= stop_ts.ceil("1d"),
                 )
             )
 
@@ -766,8 +865,8 @@ class Trino(OpenSkyDBAPI):
                 )
                 .group_by(StateVectorsData4.icao24)
                 .where(
-                    StateVectorsData4.hour >= start_ts.floor("1H"),
-                    StateVectorsData4.hour <= stop_ts.ceil("1H"),
+                    StateVectorsData4.hour >= start_ts.floor("1h"),
+                    StateVectorsData4.hour <= stop_ts.ceil("1h"),
                     StateVectorsData4.time >= start_ts,
                     StateVectorsData4.time <= stop_ts,
                 )
@@ -819,8 +918,8 @@ class Trino(OpenSkyDBAPI):
         stmt = stmt.where(
             Table.mintime >= start_ts,
             Table.mintime <= stop_ts,
-            Table.hour >= start_ts.floor("1H"),
-            Table.hour < stop_ts.ceil("1H"),
+            Table.hour >= start_ts.floor("1h"),
+            Table.hour < stop_ts.ceil("1h"),
         )
 
         if limit is not None: