FlightRadarAPI 1.6.2__tar.gz → 1.7.0__tar.gz

{flightradarapi-1.6.2 → flightradarapi-1.7.0}/FlightRadarAPI/__init__.py

@@ -12,7 +12,7 @@ https://www.flightradar24.com/terms-and-conditions
 """
 
 __author__ = "Jean Loui Bernard Silva de Jesus"
-__version__ = "1.6.2"
+__version__ = "1.7.0"
 
 from .api import FlightRadar24API
 from .core import Countries

{flightradarapi-1.6.2 → flightradarapi-1.7.0}/FlightRadarAPI/api.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import dataclasses
+import logging
 import math
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
@@ -11,9 +12,11 @@ from .entities.airport import Airport
 from .entities.flight import Flight
 from .errors import AirportNotFoundError, LoginError
 from .flight_tracker_config import FlightTrackerConfig
-from .parsers import country_to_slug, parse_airlines_html, parse_airports_json
+from .parsers import country_to_slug, parse_airlines_html, parse_airports_index, parse_airports_page, parse_inertia_page
 from .request import APIClient, RetryPolicy
 
+_logger = logging.getLogger(__name__)
+
 # Some FR24 live-feed backends answer 200 with a well-formed envelope but no
 # flight entries -- indistinguishable from a legitimately empty result. The
 # "AWSALB" cookie then pins the session to that backend, so dropping it is what
@@ -193,6 +196,12 @@ class FlightRadar24API:
         """
         Return a list with all airports, optionally narrowed to some countries.
 
+        FR24 lists airports one page per country, and one per state for the
+        largest countries, so this fetches one page per country asked for (a few
+        hundred for every airport), `max_workers` at a time. The listing carries
+        the name, codes and country only: use get_airport(code) for the position
+        and altitude of an airport.
+
         :param countries: Country names from the Countries enum, or their slug strings,
             as any iterable or a single value. Every country when omitted.
         """
@@ -207,13 +216,82 @@ class FlightRadar24API:
         if wanted is not None and len(wanted) == 0:
             return []
 
-        response = self.__client.request(
-            Core.airports_json_url, headers=Core.json_headers, timeout=self.timeout,
+        # The index names every country, and declares the asset version the
+        # JSON requests below have to quote.
+        response = self.__client.request(Core.airports_data_url, headers=Core.html_headers, timeout=self.timeout)
+        index = parse_inertia_page(response.get_bytes_content())
+        version = index.get("version") if index is not None else None
+
+        if wanted is None:
+            slugs = [country["slug"] for country in parse_airports_index(index or {})]
+        else:
+            slugs = [country_to_slug(country) for country in wanted]
+
+        slugs = list(dict.fromkeys(slug for slug in slugs if slug))
+        version = version if isinstance(version, str) else None
+
+        country_pages = self.__get_airport_pages(
+            [Core.airports_by_country_url.format(slug) for slug in slugs], version,
         )
 
-        # get_content(), not get_json_content(): an html body reaches the parser's
-        # guard; a malformed JSON body still raises. The parser slugifies `wanted`.
-        return parse_airports_json(response.get_content(), wanted)
+        # Second pass for the countries split by state, every state at once.
+        state_owners = [index for index, (_, paths) in enumerate(country_pages) for _ in paths]
+        state_pages = self.__get_airport_pages(
+            [
+                path if path.startswith("http") else Core.flightradar_base_url + path
+                for _, paths in country_pages for path in paths
+            ],
+            version,
+        )
+
+        # Grouped by country, in the order asked for, each country's states after it.
+        airports_by_country: List[List[Airport]] = [list(found) for found, _ in country_pages]
+
+        for owner, (found, _) in zip(state_owners, state_pages):
+            airports_by_country[owner].extend(found)
+
+        airports = [airport for found in airports_by_country for airport in found]
+
+        # A country FR24 does not know answers with the index page, which lists
+        # neither airports nor states.
+        missing = [slug for slug, (found, paths) in zip(slugs, country_pages) if not found and not paths]
+
+        if missing:
+            _logger.warning(
+                "get_airports: no airports found for %s — check the Countries enum.", ", ".join(missing),
+            )
+
+        return airports
+
+    def __get_airport_pages(self, urls: List[str], version: Optional[str]) -> List[Tuple[List[Airport], List[str]]]:
+        """
+        Fetch and parse the airport listing pages at `urls`, in order, max_workers at a time.
+        """
+        if not urls:
+            return []
+
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            return list(executor.map(lambda url: parse_airports_page(self.__get_inertia_page(url, version)), urls))
+
+    def __get_inertia_page(self, url: str, version: Optional[str]) -> Union[Dict, bytes]:
+        """
+        Fetch a FR24 data page as its Inertia page object, falling back to the HTML.
+
+        With the asset version FR24 answers the bare page JSON, a fraction of the
+        HTML. A 409 means the assets were redeployed since the version was read:
+        the HTML carries the same page object and needs no version.
+        """
+        if version is not None:
+            headers = {**Core.inertia_headers, "x-inertia-version": version}
+            response = self.__client.request_standalone(
+                url, headers=headers, timeout=self.timeout, allowed_error_codes=[409],
+            )
+
+            if response.get_status_code() != 409:
+                return response.get_content()
+
+        response = self.__client.request_standalone(url, headers=Core.html_headers, timeout=self.timeout)
+        return response.get_bytes_content()
 
     def get_bookmarks(self) -> Dict:
         """

{flightradarapi-1.6.2 → flightradarapi-1.7.0}/FlightRadarAPI/core.py

@@ -30,10 +30,12 @@ class Core:
     # Airports data URLs.
     api_airport_data_url = api_flightradar_base_url + "/airport.json"
     airport_data_url = flightradar_base_url + "/airports/traffic-stats/?airport={}"
+    # The listing is split by country, and by state for the largest countries.
+    # These are Inertia pages: the data rides JSON-encoded in the data-page
+    # attribute of the HTML, or comes as the whole body when the request carries
+    # `inertia_headers` and the asset version the index page declares.
     airports_data_url = flightradar_base_url + "/data/airports"
-
-    # Every airport in one payload; the per-country pages above are client-rendered.
-    airports_json_url = flightradar_base_url + "/_json/airports.php"
+    airports_by_country_url = airports_data_url + "/{}"
 
     # Airlines data URL.
     airlines_data_url = flightradar_base_url + "/data/airlines"
@@ -86,6 +88,12 @@ class Core:
     image_headers = headers.copy()
     image_headers["accept"] = "image/gif, image/jpg, image/jpeg, image/png"
 
+    # Asks an Inertia page for its page object instead of the HTML around it.
+    # `x-inertia-version` must match the deployed assets or FR24 answers 409.
+    inertia_headers = json_headers.copy()
+    inertia_headers["x-inertia"] = "true"
+    inertia_headers["x-requested-with"] = "XMLHttpRequest"
+
     html_headers = {
         "accept": (
             "text/html,application/xhtml+xml,application/xml;q=0.9,"

flightradarapi-1.7.0/FlightRadarAPI/parsers.py

@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+
+import json
+import logging
+import re
+import unicodedata
+from enum import Enum
+from typing import Dict, List, Optional, Tuple, Union
+
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+
+from .entities.airport import Airport
+
+_logger = logging.getLogger(__name__)
+
+# An airline code badge: a two-letter IATA code or a three-letter ICAO code.
+# ASCII only, so a stray Unicode letter does not pass for a code.
+AIRLINE_CODE_PATTERN = re.compile(r"^[A-Z0-9]{2,3}$", re.ASCII)
+
+# A fleet size as the page prints it: "340" or "3,878".
+AIRCRAFT_COUNT_PATTERN = re.compile(r"^\d{1,3}(?:[,.]\d{3})*$|^\d+$", re.ASCII)
+
+AIRLINES_PATH_PREFIX = "/data/airlines/"
+
+
+def _is_airline_link(href: object) -> bool:
+    return isinstance(href, str) and href.startswith(AIRLINES_PATH_PREFIX)
+
+
+def _leaf_texts(root: Tag, tag: str) -> List[str]:
+    """
+    Text of the `tag` elements under `root` that hold no other element.
+
+    The name element is left out by its `title` attribute: an airline called
+    "TAP" would otherwise read as an ICAO code.
+    """
+    return [
+        element.get_text(strip=True)
+        for element in root.find_all(tag)
+        if element.find() is None and not element.has_attr("title")
+    ]
+
+
+def _airline_name(link: Tag) -> str:
+    """
+    The airline name of a listing entry.
+
+    The name sits in the element titled with it, with the logo's aria-label as a
+    fallback; the anchor text also carries the codes and the fleet size.
+    """
+    for attribute in ("title", "aria-label"):
+        element = link.find(lambda tag, name=attribute: tag.has_attr(name))
+
+        if isinstance(element, Tag):
+            return str(element[attribute]).strip()
+
+    return link.get_text(strip=True)
+
+
+def parse_airlines_html(html: bytes) -> List[Dict]:
+    """
+    Parse the airlines listing page into a list of airline dicts.
+
+    Every airline is a link to its data page, with the IATA and ICAO codes as
+    badges next to the name. The "Largest fleets" section repeats a few airlines
+    with their aircraft count; an airline the page lists twice becomes one entry
+    carrying everything the page said about it.
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    links = soup.find_all("a", href=_is_airline_link)
+
+    if not links:
+        _logger.warning(
+            "parse_airlines_html: no airline links in response — FR24 page layout may have changed."
+        )
+        return []
+
+    airlines: Dict[str, Dict] = {}
+
+    for link in links:
+        name = _airline_name(link)
+
+        if len(name) < 2:
+            continue
+
+        codes = [text for text in _leaf_texts(link, "span") if AIRLINE_CODE_PATTERN.match(text)]
+        iata = next((code for code in codes if len(code) == 2), None)
+        icao = next((code for code in codes if len(code) == 3), None)
+
+        # Section links ("All") point under /data/airlines/ too; a code tells
+        # an airline apart from them.
+        if iata is None and icao is None:
+            continue
+
+        n_aircrafts = next(
+            (int(re.sub(r"\D", "", text)) for text in _leaf_texts(link, "strong") if AIRCRAFT_COUNT_PATTERN.match(text)),
+            None,
+        )
+
+        airline = airlines.setdefault(
+            str(link["href"]), {"Name": name, "ICAO": None, "IATA": None, "n_aircrafts": None},
+        )
+
+        for key, value in (("ICAO", icao), ("IATA", iata), ("n_aircrafts", n_aircrafts)):
+            if airline[key] is None:
+                airline[key] = value
+
+    return list(airlines.values())
+
+
+def country_to_slug(country: object) -> str:
+    """
+    Slugify a country name the way FR24 spells it in its data page URLs, so feed
+    rows can be matched against the Countries enum ("United States" -> "united-states").
+    """
+    # `.value` unwraps a Countries member, which str() would render as
+    # "Countries.BRAZIL". Diacritics are stripped so a future "Curaçao" still
+    # matches "curacao"; `is None` rather than truthiness because str(0 or "") is
+    # "" here while String(0 ?? "") is "0" in Node.
+    if isinstance(country, Enum):
+        country = country.value
+
+    decomposed = unicodedata.normalize("NFKD", "" if country is None else str(country))
+    ascii_only = "".join(char for char in decomposed if not unicodedata.combining(char))
+
+    # Punctuation becomes a hyphen rather than being deleted: FR24's own assets are
+    # named that way, e.g. flags-small/cote-d-ivoire.svg.
+    return re.sub(r"[^a-z0-9]+", "-", ascii_only.lower()).strip("-")
+
+
+def _to_text(value: object) -> str:
+    """
+    Keep a text field as a string, or "" when the feed sends anything else.
+
+    Airport documents these as strings and callers treat them as such:
+    get_country_flag(airport.country) would slugify a dict into "object-object"
+    and request that flag.
+    """
+    return value if isinstance(value, str) else ""
+
+
+def parse_inertia_page(payload: Union[bytes, str, Dict]) -> Optional[Dict]:
+    """
+    Return the Inertia page object (component, props, version) of a FR24 data page.
+
+    Served as HTML, the page keeps it JSON-encoded in the data-page attribute of
+    its root element; asked with Core.inertia_headers, FR24 sends it as the body,
+    which `request.get_content()` has already decoded into a dict.
+    """
+    if isinstance(payload, dict):
+        page: object = payload
+    else:
+        root = BeautifulSoup(payload, "html.parser").find(lambda tag: tag.has_attr("data-page"))
+
+        if not isinstance(root, Tag):
+            _logger.warning("parse_inertia_page: no data-page attribute in response — FR24 page layout may have changed.")
+            return None
+
+        try:
+            page = json.loads(str(root["data-page"]))
+        except ValueError:
+            _logger.warning("parse_inertia_page: data-page is not valid JSON — FR24 page layout may have changed.")
+            return None
+
+    if not isinstance(page, dict) or not isinstance(page.get("props"), dict):
+        _logger.warning("parse_inertia_page: no page props in response — FR24 page layout may have changed.")
+        return None
+
+    return page
+
+
+def parse_airports_index(payload: Union[bytes, str, Dict]) -> List[Dict]:
+    """
+    Parse the airports index (Core.airports_data_url) into its countries.
+
+    Each entry has the country "name" as FR24 prints it ("UNITED STATES") and the
+    "slug" of its listing page, which is also its Countries enum value.
+    """
+    page = parse_inertia_page(payload)
+    rows = page["props"].get("airportsByCountry") if page is not None else None
+
+    if not isinstance(rows, list):
+        _logger.warning('parse_airports_index: no "airportsByCountry" list in response — FR24 page layout may have changed.')
+        return []
+
+    countries = []
+
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+
+        name = _to_text(row.get("name"))
+        slug = country_to_slug(name)
+
+        if slug:
+            countries.append({"name": name, "slug": slug})
+
+    return countries
+
+
+def parse_airports_page(payload: Union[bytes, str, Dict]) -> Tuple[List[Airport], List[str]]:
+    """
+    Parse a country or state airports page into (airports, state_paths).
+
+    A country with many airports lists its states instead, each with the path of
+    its own page; those paths come back for the caller to fetch. The listing has
+    no coordinates or altitude: `get_airport(code)` has them for one airport.
+    """
+    page = parse_inertia_page(payload)
+
+    if page is None:
+        return [], []
+
+    props = page["props"]
+    country = props.get("country")
+    country_name = _to_text(country.get("name")) if isinstance(country, dict) else ""
+
+    rows = props.get("airports")
+    states = props.get("states")
+
+    airports = [
+        Airport(basic_info={
+            "name": _to_text(row.get("name")),
+            "icao": _to_text(row.get("icao")),
+            "iata": _to_text(row.get("iata")),
+            "lat": None,
+            "lon": None,
+            "alt": None,
+            "country": country_name,
+        })
+        for row in (rows if isinstance(rows, list) else [])
+        if isinstance(row, dict)
+    ]
+
+    state_paths = [
+        state["url"]
+        for state in (states if isinstance(states, list) else [])
+        if isinstance(state, dict) and isinstance(state.get("url"), str) and state["url"]
+    ]
+
+    return airports, state_paths

{flightradarapi-1.6.2 → flightradarapi-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: FlightRadarAPI
-Version: 1.6.2
+Version: 1.7.0
 Summary: SDK for FlightRadar24
 Project-URL: Homepage, https://github.com/JeanExtreme002/FlightRadarAPI
 Project-URL: Source Code, https://github.com/JeanExtreme002/FlightRadarAPI

flightradarapi-1.6.2/FlightRadarAPI/parsers.py

@@ -1,239 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import json
-import logging
-import math
-import re
-import unicodedata
-from enum import Enum
-from typing import Dict, Iterable, List, Optional, Union
-
-from bs4 import BeautifulSoup
-
-from .entities.airport import Airport
-
-_logger = logging.getLogger(__name__)
-
-# ASCII only: \d would otherwise match Unicode digits such as "٤٣".
-NUMERIC_PATTERN = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$", re.ASCII)
-
-# str.strip() and trim() disagree on U+001C-U+001F and U+FEFF: pick one set.
-SURROUNDING_SPACE = " \t\n\r\f\v"
-
-# Number.MAX_SAFE_INTEGER, the largest integer JavaScript holds exactly.
-MAX_EXACT_INTEGER = 2 ** 53 - 1
-
-
-def parse_airlines_html(html: bytes) -> List[Dict]:
-    """
-    Parse the airlines listing HTML page into a list of airline dicts.
-    """
-    soup = BeautifulSoup(html, "html.parser")
-    tbody = soup.find("tbody")
-
-    if not tbody:
-        _logger.warning(
-            "parse_airlines_html: no <tbody> in response — FR24 page layout may have changed."
-        )
-        return []
-
-    airlines = []
-
-    for tr in tbody.find_all("tr"):
-        td_notranslate = tr.find("td", class_="notranslate")
-
-        if not td_notranslate:
-            continue
-
-        a_element = td_notranslate.find("a", href=lambda href: href and href.startswith("/data/airlines"))
-
-        if not a_element:
-            continue
-
-        airline_name = a_element.get_text(strip=True)
-
-        if len(airline_name) < 2:
-            continue
-
-        td_elements = tr.find_all("td")
-        iata = None
-        icao = None
-
-        if len(td_elements) >= 4:
-            codes_text = td_elements[3].get_text(strip=True)
-
-            if " / " in codes_text:
-                parts = codes_text.split(" / ")
-                if len(parts) == 2:
-                    iata = parts[0].strip()
-                    icao = parts[1].strip()
-            elif len(codes_text) == 2:
-                iata = codes_text
-            elif len(codes_text) == 3:
-                icao = codes_text
-
-        n_aircrafts = None
-
-        if len(td_elements) >= 5:
-            aircrafts_text = td_elements[4].get_text(strip=True)
-            if aircrafts_text:
-                n_aircrafts = int(aircrafts_text.split(" ", maxsplit=1)[0].strip())
-
-        airlines.append({"Name": airline_name, "ICAO": icao, "IATA": iata, "n_aircrafts": n_aircrafts})
-
-    return airlines
-
-
-def country_to_slug(country: object) -> str:
-    """
-    Slugify a country name the way FR24 spells it in its data page URLs, so feed
-    rows can be matched against the Countries enum ("United States" -> "united-states").
-    """
-    # `.value` unwraps a Countries member, which str() would render as
-    # "Countries.BRAZIL". Diacritics are stripped so a future "Curaçao" still
-    # matches "curacao"; `is None` rather than truthiness because str(0 or "") is
-    # "" here while String(0 ?? "") is "0" in Node.
-    if isinstance(country, Enum):
-        country = country.value
-
-    decomposed = unicodedata.normalize("NFKD", "" if country is None else str(country))
-    ascii_only = "".join(char for char in decomposed if not unicodedata.combining(char))
-
-    # Punctuation becomes a hyphen rather than being deleted: FR24's own assets are
-    # named that way, e.g. flags-small/cote-d-ivoire.svg.
-    return re.sub(r"[^a-z0-9]+", "-", ascii_only.lower()).strip("-")
-
-
-def _to_text(value: object) -> str:
-    """
-    Keep a text field as a string, or "" when the feed sends anything else.
-
-    Airport documents these as strings and callers treat them as such:
-    get_country_flag(airport.country) would slugify a dict into "object-object"
-    and request that flag.
-    """
-    return value if isinstance(value, str) else ""
-
-
-def _to_number(value: object) -> Optional[Union[int, float]]:
-    """
-    Coerce a numeric feed field into a number, or None when it is unusable.
-
-    An unusable coordinate must not become 0.0: that would place the airport in
-    the Gulf of Guinea instead of marking its position as unknown.
-
-    Plain whole numbers stay ints, so an altitude reads 2436 in both ports rather
-    than 2436.0 here -- the feed sends `alt` as an int for most rows and as a
-    string ("-1") for the rest. Anything written with a decimal point or an
-    exponent stays a float: "2436.0" and "1e3" are floats here and integral
-    Numbers in Node, equal in value but not in type.
-    """
-    # bool is an int here, and str([43]) would invent a number.
-    if isinstance(value, bool) or not isinstance(value, (int, float, str)):
-        return None
-
-    if isinstance(value, str):
-        stripped = value.strip(SURROUNDING_SPACE)
-
-        if not NUMERIC_PATTERN.match(stripped):
-            return None
-
-        text: Union[int, float, str] = stripped
-    else:
-        text = value
-
-    # One conversion, so overflow is handled once: Python ints have no ceiling.
-    try:
-        number = float(text)
-    except OverflowError:
-        return None
-
-    if not math.isfinite(number):
-        return None
-
-    try:
-        exact = int(text)
-    except (ValueError, OverflowError):
-        return number
-
-    # int only when lossless (so -23.4 survives) and only where JavaScript is exact.
-    return exact if exact == number and abs(exact) <= MAX_EXACT_INTEGER else number
-
-
-def parse_airports_json(
-    payload: Union[bytes, str, Dict], countries: Optional[Iterable[object]] = None,
-) -> List[Airport]:
-    """
-    Parse the airports JSON feed into a list of Airport instances.
-
-    :param payload: Body of Core.airports_json_url.
-    :param countries: Country slugs, or Countries members. Every airport is kept when omitted.
-    """
-    if isinstance(payload, (bytes, str)):
-        try:
-            data = json.loads(payload)
-        except ValueError:
-            _logger.warning("parse_airports_json: response is not valid JSON — FR24 feed may have changed.")
-            return []
-    else:
-        data = payload
-
-    rows = data.get("rows") if isinstance(data, dict) else None
-
-    if not isinstance(rows, list):
-        _logger.warning('parse_airports_json: no "rows" array in response — FR24 feed may have changed.')
-        return []
-
-    wanted = {country_to_slug(country) for country in countries} if countries is not None else None
-    matched = set()
-    unpositioned = []
-    airports = []
-
-    for row in rows:
-        if not isinstance(row, dict):
-            continue
-
-        # Slugified only when filtering: it is otherwise unused, and this loop
-        # runs over every airport in the feed.
-        if wanted is not None:
-            slug = country_to_slug(row.get("country"))
-
-            if slug not in wanted:
-                continue
-            matched.add(slug)
-
-        latitude = _to_number(row.get("lat"))
-        longitude = _to_number(row.get("lon"))
-
-        # One bad coordinate drops both: half a position reads as located.
-        if latitude is None or longitude is None:
-            latitude = longitude = None
-            unpositioned.append(_to_text(row.get("name")))
-
-        airports.append(Airport(basic_info={
-            "name": _to_text(row.get("name")),
-            "icao": _to_text(row.get("icao")),
-            "iata": _to_text(row.get("iata")),
-            "lat": latitude,
-            "lon": longitude,
-            "alt": _to_number(row.get("alt")),
-            "country": _to_text(row.get("country")),
-        }))
-
-    # One line, not one per row: the feed carries every airport.
-    if unpositioned:
-        _logger.warning(
-            "parse_airports_json: %d airport(s) had unusable coordinates and carry no position (e.g. %s).",
-            len(unpositioned), ", ".join(repr(name) for name in unpositioned[:3]),
-        )
-
-    if wanted is not None:
-        missing = sorted(wanted - matched)
-
-        if missing:
-            _logger.warning(
-                "parse_airports_json: no airports found for %s — check the Countries enum.",
-                ", ".join(missing),
-            )
-
-    return airports