pyopensky 2.17__tar.gz → 2.18__tar.gz

{pyopensky-2.17 → pyopensky-2.18}/.github/workflows/build-docs.yml

@@ -23,7 +23,7 @@ jobs:
         run: curl -LsSf https://astral.sh/uv/install.sh | sh
 
       - name: Set up Python
-        uses: actions/setup-python@v7.0.0
+        uses: actions/setup-python@v7.1.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 

{pyopensky-2.17 → pyopensky-2.18}/.github/workflows/pypi_publish.yml

@@ -15,7 +15,7 @@ jobs:
         run: curl -LsSf https://astral.sh/uv/install.sh | sh
 
       - name: Set up Python
-        uses: actions/setup-python@v7.0.0
+        uses: actions/setup-python@v7.1.0
 
       - name: Cache folder for uv
         uses: actions/cache@v6

{pyopensky-2.17 → pyopensky-2.18}/.github/workflows/run_tests.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - uses: astral-sh/setup-uv@v10.1.0
+      - uses: astral-sh/setup-uv@v10.3.2
         with:
           enable-cache: true
 
@@ -29,7 +29,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - uses: astral-sh/setup-uv@v10.1.0
+      - uses: astral-sh/setup-uv@v10.3.2
         with:
           enable-cache: true
 
@@ -45,7 +45,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: astral-sh/setup-uv@v10.1.0
+      - uses: astral-sh/setup-uv@v10.3.2
         with:
           enable-cache: true
 
@@ -103,7 +103,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: astral-sh/setup-uv@v10.1.0
+      - uses: astral-sh/setup-uv@v10.3.2
         with:
           enable-cache: true
 

{pyopensky-2.17 → pyopensky-2.18}/.gitignore

@@ -33,3 +33,4 @@ var/
 *.manifest
 *.spec
 
+.secrets

{pyopensky-2.17 → pyopensky-2.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pyopensky
-Version: 2.17
+Version: 2.18
 Summary: A Python interface for OpenSky database
 Project-URL: Repository, https://github.com/open-aviation/pyopensky/
 Author-email: Xavier Olive <git@xoolive.org>, Junzi Sun <j.sun-1@tudelft.nl>
@@ -138,5 +138,32 @@ trino.history(start, stop, *, callsign, icao24, bounds)
 trino.rebuild(start, stop, *, icao24, bounds)
 ```
 
+For several routes in the same time window, use one query with exact directed
+airport pairs:
+
+```python
+df = trino.history(
+    "2025-01-01",
+    "2025-01-02",
+    routes=[("KJFK", "EGLL"), ("KMIA", "EIDW"), ("LFPG", "KJFK")],
+    selected_columns=("time", "icao24", "callsign", "lat", "lon"),
+)
+```
+
+The same `routes` filter is available in `flightlist()` for metadata queries and
+`rawdata()` for raw messages.
+`history()` clips trajectories to the requested time window, including flights
+that cross midnight.
+
+Optional Trino session settings apply to all queries on a client. For selective
+flight joins whose filtered flight table fits in each worker's memory:
+
+```python
+trino = Trino(session_properties={"join_distribution_type": "BROADCAST"})
+```
+
+`Trino()` uses the server's default settings. Use `cached=False` on each call when
+benchmarking query performance.
+
 > [!IMPORTANT]
 > Refer to the [documentation](https://mode-s.org/pyopensky) for advanced usage and parameter details.

{pyopensky-2.17 → pyopensky-2.18}/docs/source/trino_advanced.rst

@@ -70,6 +70,155 @@ Use ``with_only_columns()`` to reduce data transfer and improve query performanc
    - Schema attribute: ``StateVectorsData4.lat``
    - Qualified string: ``"StateVectorsData4.lat"`` or ``"FlightsData4.estdepartureairport"``
 
+Configuring Trino sessions
+---------------------------
+
+Pass ``session_properties`` to ``Trino`` to configure each connection opened by
+that client. Names and values are strings. For example, a broadcast join can
+reduce CPU and worker network traffic for selective flight queries:
+
+.. code-block:: python
+
+    from pyopensky.trino import Trino
+
+    trino = Trino(
+        session_properties={"join_distribution_type": "BROADCAST"}
+    )
+    df = trino.history(
+        "2025-06-01 12:00",
+        "2025-06-01 13:00",
+        routes=[("EGLL", "KJFK"), ("LFPG", "KJFK"), ("EHAM", "KJFK")],
+        selected_columns=("time", "icao24", "callsign", "lat", "lon"),
+    )
+
+``Trino()`` uses the server's default session settings. Broadcast joins are
+optional: the filtered build table must fit in each worker's memory. For large
+flight selections, let the server choose the join strategy or choose another
+session configuration. See `Trino join distribution selection
+<https://trino.io/docs/current/optimizer/cost-based-optimizations.html#join-distribution-selection>`_.
+
+Session properties apply to ``query()``, ``history()``, ``flightlist()``, and
+``rawdata()`` on that client. The constructor copies the supplied mapping.
+Result-cache keys include configured properties, so different configurations do
+not reuse each other's results. To measure query performance, pass
+``cached=False`` to every measured call.
+
+Batching directed routes
+------------------------
+
+Use ``routes`` to retrieve several directed airport pairs in one query for the
+same time window. This avoids submitting one state-vector query per route.
+
+.. code-block:: python
+
+    from pyopensky.trino import Trino
+
+    trino = Trino()
+    routes = [
+        ("KJFK", "EGLL"),
+        ("KMIA", "EIDW"),
+        ("LFPG", "KJFK"),
+    ]
+    df = trino.history(
+        "2025-01-01",
+        "2025-01-02",
+        routes=routes,
+        selected_columns=(
+            "time", "icao24", "callsign", "lat", "lon", "baroaltitude",
+            "FlightsData4.firstseen", "FlightsData4.lastseen",
+            "FlightsData4.estdepartureairport",
+            "FlightsData4.estarrivalairport",
+        ),
+    )
+
+Each pair contains the estimated departure and arrival ICAO identifiers, in that
+order. Only the listed pairs match. For example, ``[("KJFK", "EGLL"),
+("KMIA", "EIDW")]`` does not include KJFK to EIDW or the reverse directions.
+Separate ``departure_airport`` and ``arrival_airport`` lists select every
+combination between those lists.
+
+Route identifiers match exactly; SQL wildcards are not expanded. An empty route
+list selects no flights. Repeating a pair does not duplicate its results. Other
+filters, such as ``icao24``, ``callsign``, bounds and airport filters, further
+restrict the selected routes.
+
+For long periods, loop over time windows and include all required routes in each
+call. Select the columns needed for the analysis. Actual scan savings depend on
+the Trino query plan and table layout.
+
+If only flight metadata is needed, use the same route list with
+``trino.flightlist(start, stop, routes=routes)``. With routes alone,
+``flightlist()`` filters by ``lastseen``. Setting ``departure_airport`` also
+selects its existing departure-time behavior.
+
+.. note::
+
+   ``history()`` returns state vectors within ``start`` and ``stop``. These
+   timestamps do not select flights by their departure or arrival time. A flight
+   that crosses midnight can be partial in a daily result. ``time_buffer``
+   extends the bounds around each flight's ``firstseen`` and ``lastseen`` but
+   does not extend the requested state-vector window. Each call submits one
+   query; pyopensky does not split that window automatically.
+
+Filtering flight metadata in history queries
+--------------------------------------------
+
+``history()`` accepts ``FlightsData4`` conditions alongside state-vector
+conditions. Flight columns refer to the flight table joined to the state vectors.
+A flight condition also enables this join when no airport or route is specified.
+Selecting flight columns also enables this join. Flight schema attributes can
+be used directly in ``selected_columns``.
+
+.. code-block:: python
+
+    from pyopensky.schema import FlightsData4, StateVectorsData4
+    from pyopensky.trino import Trino
+
+    df = Trino().history(
+        "2025-01-01",
+        "2025-01-02",
+        FlightsData4.estarrivalairporthorizdistance < 5000,
+        routes=[("KJFK", "EGLL"), ("LFPG", "KJFK")],
+        selected_columns=(
+            StateVectorsData4.time,
+            StateVectorsData4.icao24,
+            FlightsData4.estdepartureairport,
+            FlightsData4.estarrivalairport,
+        ),
+    )
+
+Raw messages for directed routes
+--------------------------------
+
+``rawdata()`` also accepts exact ``routes``. It joins raw messages to matching
+flights by aircraft address and the flight's ``firstseen``/``lastseen`` interval,
+then clips messages to the requested time window. FlightsData4 conditions and
+extra columns use this joined flight table.
+
+.. code-block:: python
+
+    from pyopensky.schema import FlightsData4, PositionData4
+    from pyopensky.trino import Trino
+
+    df = Trino().rawdata(
+        "2025-01-01",
+        "2025-01-02",
+        FlightsData4.estarrivalairporthorizdistance < 5000,
+        routes=[("KJFK", "EGLL"), ("LFPG", "KJFK")],
+        Table=PositionData4,
+        extra_columns=(FlightsData4.estdepartureairport,),
+    )
+
+An empty route list selects no messages. Repeated pairs do not duplicate results.
+Airport filters and ``icao24`` further restrict the selected routes. Flight
+conditions or flight extra columns enable a flight join even without an airport
+or route argument.
+
+``rawdata()`` supports one selection mode per call: flight metadata (airports,
+routes, flight conditions or flight extra columns), geographic bounds, or
+callsign. These modes cannot be combined. Conditions on the raw table can
+further restrict any mode. Its existing raw-column output is preserved.
+
 
 Spatial queries with bounds
 ---------------------------
@@ -181,7 +330,8 @@ Filter by sensor serial numbers to get data from specific ground stations:
 Extending temporal range for airport queries
 --------------------------------------------
 
-Use ``time_buffer`` to include aircraft that departed/arrived slightly outside the time window:
+Use ``time_buffer`` to include state vectors just before a flight's ``firstseen``
+or after its ``lastseen``, within the requested time window:
 
 .. code-block:: python
 
@@ -189,7 +339,7 @@ Use ``time_buffer`` to include aircraft that departed/arrived slightly outside t
     
     trino = Trino()
     
-    # Get flights with 30-minute buffer before/after time window
+    # Allow samples up to 30 minutes before/after each flight's time bounds
     df = trino.history(
         start="2024-03-16 09:00",
         stop="2024-03-16 11:00",
@@ -197,7 +347,8 @@ Use ``time_buffer`` to include aircraft that departed/arrived slightly outside t
         time_buffer="30m",  # Also accepts "1h", "45min", etc.
     )
 
-This is useful when you want complete flight trajectories that may start or end outside your time range.
+The result remains clipped to 09:00 through 11:00. To retrieve samples outside
+that window, extend ``start`` or ``stop`` explicitly.
 
 Filtering with boolean expressions
 ----------------------------------

{pyopensky-2.17 → pyopensky-2.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pyopensky"
-version = "2.17"
+version = "2.18"
 description = "A Python interface for OpenSky database"
 authors = [{ name = "Xavier Olive", email = "git@xoolive.org" }, { name = "Junzi Sun", email = "j.sun-1@tudelft.nl" }]
 license = "MIT"

{pyopensky-2.17 → pyopensky-2.18}/readme.md

@@ -109,5 +109,32 @@ trino.history(start, stop, *, callsign, icao24, bounds)
 trino.rebuild(start, stop, *, icao24, bounds)
 ```
 
+For several routes in the same time window, use one query with exact directed
+airport pairs:
+
+```python
+df = trino.history(
+    "2025-01-01",
+    "2025-01-02",
+    routes=[("KJFK", "EGLL"), ("KMIA", "EIDW"), ("LFPG", "KJFK")],
+    selected_columns=("time", "icao24", "callsign", "lat", "lon"),
+)
+```
+
+The same `routes` filter is available in `flightlist()` for metadata queries and
+`rawdata()` for raw messages.
+`history()` clips trajectories to the requested time window, including flights
+that cross midnight.
+
+Optional Trino session settings apply to all queries on a client. For selective
+flight joins whose filtered flight table fits in each worker's memory:
+
+```python
+trino = Trino(session_properties={"join_distribution_type": "BROADCAST"})
+```
+
+`Trino()` uses the server's default settings. Use `cached=False` on each call when
+benchmarking query performance.
+
 > [!IMPORTANT]
 > Refer to the [documentation](https://mode-s.org/pyopensky) for advanced usage and parameter details.

pyopensky-2.18/scripts/benchmark_trino_routes.py

@@ -0,0 +1,214 @@
+"""Compare separate route queries at a Git revision with one routes query.
+
+Run from the repository root, for example:
+    uv run --env-file .secrets scripts/benchmark_trino_routes.py \
+        --baseline-ref origin/master --output /tmp/trino-routes-benchmark.json
+
+This runs live Trino queries. Every query uses cached=False and an isolated,
+temporary result-cache directory. Server and storage caches are not controlled.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import platform
+import subprocess
+import time
+import types
+from contextlib import contextmanager
+from importlib.metadata import version
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Iterable, Iterator
+from unittest.mock import patch
+
+from sqlalchemy import CursorResult
+
+import pandas as pd
+import pyopensky.trino as current
+from pyopensky.trino import Trino
+
+COLUMNS = ("time", "icao24", "callsign", "lat", "lon", "baroaltitude")
+DEFAULT_ROUTES = [("EGLL", "KJFK"), ("LFPG", "KJFK"), ("EHAM", "KJFK")]
+
+
+@contextmanager
+def instrument(client: Trino, records: list[dict[str, Any]]) -> Iterator[None]:
+    process_result = client.process_result
+
+    def record(
+        result: CursorResult[Any], batch_size: int = 50_000
+    ) -> Iterable[pd.DataFrame]:
+        cursor = result.cursor
+        yield from process_result(result, batch_size)
+        assert cursor is not None
+        records.append(dict(cursor.stats))
+
+    with patch.object(client, "process_result", record):
+        yield
+
+
+def run(
+    client: Trino,
+    mode: str,
+    start: str,
+    stop: str,
+    routes: list[tuple[str, str]],
+) -> tuple[pd.DataFrame, float]:
+    frames = []
+    begin = time.perf_counter()
+    if mode == "before":
+        for departure, arrival in routes:
+            frame = client.history(
+                start,
+                stop,
+                departure_airport=departure,
+                arrival_airport=arrival,
+                selected_columns=COLUMNS,
+                cached=False,
+            )
+            if frame is not None:
+                frames.append(frame)
+    else:
+        frame = client.history(
+            start, stop, routes=routes, selected_columns=COLUMNS, cached=False
+        )
+        if frame is not None:
+            frames.append(frame)
+    result = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
+    return result, time.perf_counter() - begin
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--baseline-ref", required=True)
+    parser.add_argument("--start", default="2025-06-01 12:00")
+    parser.add_argument("--stop", default="2025-06-01 13:00")
+    parser.add_argument("--route", action="append", help="DEPARTURE:ARRIVAL")
+    parser.add_argument("--rounds", type=int, default=2)
+    parser.add_argument(
+        "--after-session-property",
+        action="append",
+        default=[],
+        help="NAME=VALUE; configure the batched client only (repeatable)",
+    )
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    if args.rounds < 1:
+        parser.error("--rounds must be at least 1")
+    session_properties = {}
+    for setting in args.after_session_property:
+        name, separator, value = setting.partition("=")
+        if not separator or not name or not value:
+            parser.error(
+                "--after-session-property must have the form NAME=VALUE"
+            )
+        session_properties[name] = value
+    routes = DEFAULT_ROUTES
+    if args.route:
+        routes = []
+        for route in args.route:
+            parts = route.split(":")
+            if len(parts) != 2 or not all(parts):
+                parser.error("--route must have the form DEPARTURE:ARRIVAL")
+            routes.append((parts[0], parts[1]))
+    routes = list(dict.fromkeys(routes))
+
+    baseline_commit = subprocess.check_output(
+        ["git", "rev-parse", "--verify", f"{args.baseline_ref}^{{commit}}"],
+        text=True,
+    ).strip()
+    source = subprocess.check_output(
+        ["git", "show", f"{baseline_commit}:src/pyopensky/trino.py"], text=True
+    )
+    baseline = types.ModuleType("pyopensky._routes_benchmark_before")
+    baseline.__package__ = "pyopensky"
+    exec(
+        compile(source, f"{baseline_commit}:trino.py", "exec"),
+        baseline.__dict__,
+    )
+
+    report: dict[str, Any] = {
+        "baseline_commit": baseline_commit,
+        "after_source_sha256": hashlib.sha256(
+            Path(current.__file__).read_bytes()
+        ).hexdigest(),
+        "started_at_utc": pd.Timestamp.now(tz="UTC").isoformat(),
+        "start": args.start,
+        "stop": args.stop,
+        "timezone": "UTC",
+        "routes": routes,
+        "selected_columns": COLUMNS,
+        "cached": False,
+        "after_session_properties": session_properties,
+        "cache_directory": "isolated temporary directory",
+        "server_and_storage_caches": "not controlled",
+        "python": platform.python_version(),
+        "packages": {
+            name: version(name) for name in ("pyopensky", "trino", "sqlalchemy")
+        },
+        "runs": [],
+    }
+    with TemporaryDirectory(prefix="pyopensky-routes-benchmark-") as directory:
+        current.cache_path = Path(directory)
+        setattr(baseline, "cache_path", Path(directory))
+        after = Trino(session_properties=session_properties)
+        after.token()  # Authenticate before timing either workload.
+        before = baseline.Trino()
+        before._token = after._token
+        clients = {"before": before, "after": after}
+        reference = None
+        for round_number in range(args.rounds):
+            order = (
+                ("before", "after")
+                if round_number % 2 == 0
+                else ("after", "before")
+            )
+            for mode in order:
+                records: list[dict[str, Any]] = []
+                client = clients[mode]
+                with instrument(client, records):
+                    frame, seconds = run(
+                        client, mode, args.start, args.stop, routes
+                    )
+                canonical = (
+                    frame.sort_values(list(COLUMNS)).reset_index(drop=True)
+                    if len(frame)
+                    else frame
+                )
+                if reference is None:
+                    reference = canonical
+                else:
+                    pd.testing.assert_frame_equal(reference, canonical)
+                assert not list(Path(directory).iterdir()), "Cache was written"
+                run_report = {
+                    "round": round_number + 1,
+                    "mode": mode,
+                    "rows": len(frame),
+                    "seconds": seconds,
+                    "query_count": len(records),
+                    "physical_input_bytes": sum(
+                        record["physicalInputBytes"] for record in records
+                    ),
+                    "cpu_ms": sum(
+                        record["cpuTimeMillis"] for record in records
+                    ),
+                    "queries": records,
+                }
+                report["runs"].append(run_report)
+                args.output.write_text(json.dumps(report, indent=2))
+                print(
+                    json.dumps(
+                        {k: v for k, v in run_report.items() if k != "queries"}
+                    ),
+                    flush=True,
+                )
+    report["identical_results"] = True
+    report["result_cache_files_written"] = 0
+    args.output.write_text(json.dumps(report, indent=2))
+
+
+if __name__ == "__main__":
+    main()

{pyopensky-2.17 → pyopensky-2.18}/src/pyopensky/api.py

@@ -26,6 +26,7 @@ class OpenSkyDBAPI(Protocol):
         departure_airport: str | list[str] | None = None,
         arrival_airport: str | list[str] | None = None,
         airport: str | list[str] | None = None,
+        routes: list[tuple[str, str]] | None = None,
         callsign: str | list[str] | None = None,
         icao24: str | list[str] | None = None,
         cached: bool = True,
@@ -39,16 +40,16 @@ class OpenSkyDBAPI(Protocol):
         start: timelike,
         stop: timelike | None = None,
         *args: Any,
-        # date_delta: timedelta = timedelta(hours=1),
         callsign: str | list[str] | None = None,
         icao24: str | list[str] | None = None,
         serials: int | Iterable[int] | None = None,
         bounds: (
             str | HasBounds | tuple[float, float, float, float] | None
         ) = None,
-        departure_airport: str | None = None,
-        arrival_airport: str | None = None,
-        airport: str | None = None,
+        departure_airport: str | list[str] | None = None,
+        arrival_airport: str | list[str] | None = None,
+        airport: str | list[str] | None = None,
+        routes: list[tuple[str, str]] | None = None,
         time_buffer: str | pd.Timedelta | None = None,
         cached: bool = True,
         compress: bool = False,
@@ -65,9 +66,10 @@ class OpenSkyDBAPI(Protocol):
         serials: int | Iterable[int] | None = None,
         bounds: HasBounds | tuple[float, float, float, float] | None = None,
         callsign: str | list[str] | None = None,
-        departure_airport: str | None = None,
-        arrival_airport: str | None = None,
-        airport: str | None = None,
+        departure_airport: str | list[str] | None = None,
+        arrival_airport: str | list[str] | None = None,
+        airport: str | list[str] | None = None,
+        routes: list[tuple[str, str]] | None = None,
         cached: bool = True,
         compress: bool = False,
         limit: int | None = None,