mobile-store-scraper 0.1.0__py3-none-any.whl

mobile_store_scraper/__init__.py

@@ -0,0 +1,31 @@
+"""mobile-store-scraper - async Google Play + Apple App Store scraper with reliable versions.
+
+>>> import mobile_store_scraper as mss
+>>> app = mss.scrape(store="gp", app="com.naturalmotion.customstreetracer2", country="DE")
+>>> app.version, app.last_updated, app.min_os_ver, app.downloads, app.released, app.publisher
+('6.8.0', '2026-10-14T07:10:38Z', 'Android 7.0', 82008679, '2016-06-29T15:01:05Z', 'Zynga')
+"""
+from .appstore import AppStore
+from .countries import ALL_COUNTRIES, APP_STORE_COUNTRIES, PLAY_COUNTRIES
+from .dates import to_iso
+from .errors import (AppNotFound, InvalidCountry, IpBlocked, ParseError, PlayApiError, PlayAuthError, RateLimited,
+                     StoreScraperError)
+from .http import HttpClient
+from .models import AppData, AppStoreApp, DeviceVersion, GooglePlayApp, Store, VersionSource
+from .play import (DEFAULT_DEVICE, REPRESENTATIVE_DEVICES, GooglePlay, PlayApiClient, PlaySession,
+                   available_devices)
+from .scrape import (Scraper, resolve_store, scrape, scrape_async, scrape_countries, scrape_countries_async,
+                     scrape_many, scrape_many_async)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "scrape", "scrape_async", "scrape_countries", "scrape_countries_async", "scrape_many", "scrape_many_async",
+    "Scraper", "resolve_store",
+    "GooglePlay", "AppStore", "PlayApiClient", "PlaySession", "HttpClient",
+    "AppData", "GooglePlayApp", "AppStoreApp", "DeviceVersion", "Store", "VersionSource",
+    "PLAY_COUNTRIES", "APP_STORE_COUNTRIES", "ALL_COUNTRIES",
+    "DEFAULT_DEVICE", "REPRESENTATIVE_DEVICES", "available_devices", "to_iso",
+    "StoreScraperError", "AppNotFound", "IpBlocked", "InvalidCountry", "ParseError", "RateLimited",
+    "PlayAuthError", "PlayApiError", "__version__",
+]

mobile_store_scraper/__main__.py

@@ -0,0 +1,63 @@
+"""Command line.
+
+    python -m mobile_store_scraper gp com.naturalmotion.customstreetracer2 -c DE
+    python -m mobile_store_scraper as 310633997 -c all --concurrency 20 --rate-limit 10
+    python -m mobile_store_scraper gp com.google.android.youtube -m all_devices --json
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+
+from .errors import StoreScraperError
+from .scrape import Scraper
+
+SHOW = ("version", "last_updated", "min_os_ver", "downloads", "released", "publisher")
+
+
+def _row(app) -> str:
+    return (f"{app.country:3} {str(app.version):22} {str(app.last_updated):21} {str(app.min_os_ver):13} "
+            f"{str(getattr(app, 'downloads', '-')):>13} {str(app.released):21} {app.publisher or ''}")
+
+
+async def _main(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser(prog="mobile-store-scraper")
+    ap.add_argument("store", help="gp / google_play / play  or  as / apple / app_store / ios")
+    ap.add_argument("app", help="package name (Play) or track id / bundle id (App Store)")
+    ap.add_argument("-c", "--country", default="US", help="ISO code, comma list, or 'all'")
+    ap.add_argument("-l", "--lang", default=None)
+    ap.add_argument("-m", "--mode", default="auto", choices=["auto", "web", "api", "all_devices"],
+                    help="Google Play version mode")
+    ap.add_argument("--concurrency", type=int, default=None, help="max simultaneous requests (default: unlimited)")
+    ap.add_argument("--rate-limit", type=float, default=None, help="requests/second per host (default: unlimited)")
+    ap.add_argument("--json", action="store_true", help="print every attribute as JSON")
+    a = ap.parse_args(argv)
+
+    async with Scraper(concurrency=a.concurrency, rate_limit=a.rate_limit, version_mode=a.mode, lang=a.lang) as s:
+        if a.country.lower() != "all" and "," not in a.country:
+            try:
+                res = {a.country.upper(): await s.scrape(a.store, a.app, a.country)}
+            except StoreScraperError as exc:
+                res = {a.country.upper(): exc}
+        else:
+            res = await s.scrape_countries(a.store, a.app, a.country)
+    if a.json:
+        print(json.dumps({k: (v.to_dict() if not isinstance(v, Exception) else {"error": f"{type(v).__name__}: {v}"})
+                          for k, v in res.items()}, indent=2, ensure_ascii=False, default=str))
+    else:
+        print(f"{'cc':3} {'version':22} {'last_updated':21} {'min_os_ver':13} {'downloads':>13} {'released':21} publisher")
+        for k, v in res.items():
+            print(_row(v) if not isinstance(v, Exception) else f"{k:3} {type(v).__name__}: {v}")
+    return 0
+
+
+def main() -> None:
+    if sys.platform == "win32":
+        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+    sys.exit(asyncio.run(_main(sys.argv[1:])))
+
+
+if __name__ == "__main__":
+    main()

mobile_store_scraper/appstore.py

@@ -0,0 +1,263 @@
+"""Async Apple App Store scraper.
+
+Primary source is the iTunes lookup API (fast, JSON, up to 150 ids per call). The public
+``apps.apple.com`` listing is used as a fallback when lookup has no result, and optionally
+as a cross-check (``verify_with_web=True``) because lookup's CDN cache sometimes lags a
+fresh release by a few hours.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+from typing import Any, Iterable, Optional, Union
+
+from .countries import APP_STORE_COUNTRIES, normalize_country
+from .dates import from_iso, from_js_date, to_iso
+from .errors import AppNotFound, InvalidCountry, ParseError, StoreScraperError
+from .http import HttpClient
+from .models import AppStoreApp, VersionSource
+
+LOOKUP_URL = "https://itunes.apple.com/lookup"
+WEB_URL = "https://apps.apple.com/{cc}/app/id{id}"
+_SSD_RE = re.compile(r'<script[^>]*id="serialized-server-data"[^>]*>(.*?)</script>', re.S)
+_VER_RE = re.compile(r"^\d+(\.\d+)*[\w.\-]*$")
+
+
+def _version_key(v: Optional[str]) -> tuple:
+    if not v:
+        return ()
+    return tuple(int(p) if p.isdigit() else 0 for p in re.split(r"[.\-]", v))
+
+
+def _is_numeric_id(app_id: Union[str, int]) -> bool:
+    return _clean_id(app_id).isdigit()
+
+
+def _clean_id(app_id: Union[str, int]) -> str:
+    s = str(app_id).strip()
+    return s[2:] if s.startswith("id") and s[2:].isdigit() else s
+
+
+def parse_lookup_result(r: dict[str, Any], country: str, lang: Optional[str] = None) -> AppStoreApp:
+    price = r.get("price")
+    min_os = r.get("minimumOsVersion")
+    return AppStoreApp(
+        app_id=str(r.get("trackId")),
+        country=country,
+        language=lang,
+        url=r.get("trackViewUrl"),
+        title=r.get("trackName"),
+        version=r.get("version"),
+        version_source=VersionSource.ITUNES_LOOKUP.value if r.get("version") else None,
+        last_updated=to_iso(from_iso(r.get("currentVersionReleaseDate"))),
+        released=to_iso(from_iso(r.get("releaseDate"))),
+        min_os_ver=f"iOS {min_os}" if min_os else None,
+        publisher=r.get("artistName") or r.get("sellerName"),
+        developer_id=str(r["artistId"]) if r.get("artistId") else None,
+        developer_website=r.get("sellerUrl"),
+        description=r.get("description"),
+        release_notes=r.get("releaseNotes"),
+        genre=r.get("primaryGenreName"),
+        genre_id=str(r["primaryGenreId"]) if r.get("primaryGenreId") else None,
+        icon=r.get("artworkUrl512") or r.get("artworkUrl100"),
+        screenshots=r.get("screenshotUrls") or [],
+        score=r.get("averageUserRating"),
+        ratings=r.get("userRatingCount"),
+        price=price,
+        currency=r.get("currency"),
+        free=(price == 0) if price is not None else None,
+        content_rating=r.get("trackContentRating") or r.get("contentAdvisoryRating"),
+        track_id=r.get("trackId"),
+        bundle_id=r.get("bundleId"),
+        seller=r.get("sellerName"),
+        seller_url=r.get("sellerUrl"),
+        developer_url=r.get("artistViewUrl"),
+        file_size_bytes=int(r["fileSizeBytes"]) if str(r.get("fileSizeBytes", "")).isdigit() else None,
+        supported_devices=r.get("supportedDevices") or [],
+        ipad_screenshots=r.get("ipadScreenshotUrls") or [],
+        appletv_screenshots=r.get("appletvScreenshotUrls") or [],
+        languages=r.get("languageCodesISO2A") or [],
+        genres=r.get("genres") or [],
+        genre_ids=r.get("genreIds") or [],
+        advisories=r.get("advisories") or [],
+        features=r.get("features") or [],
+        content_advisory_rating=r.get("contentAdvisoryRating"),
+        score_current_version=r.get("averageUserRatingForCurrentVersion"),
+        ratings_current_version=r.get("userRatingCountForCurrentVersion"),
+        formatted_price=r.get("formattedPrice"),
+        is_game_center_enabled=r.get("isGameCenterEnabled"),
+        kind=r.get("kind"),
+        raw={"lookup": r},
+    )
+
+
+def _walk(o: Any):
+    if isinstance(o, dict):
+        yield o
+        for v in o.values():
+            yield from _walk(v)
+    elif isinstance(o, list):
+        for v in o:
+            yield from _walk(v)
+
+
+def parse_web_html(html: str) -> dict[str, Any]:
+    """Extract newest version + its timestamp (and title) from an apps.apple.com page."""
+    m = _SSD_RE.search(html)
+    if not m:
+        raise ParseError("apps.apple.com page has no serialized-server-data block")
+    data = json.loads(m.group(1))
+    version = date = None
+    for node in _walk(data):
+        mrv = node.get("mostRecentVersion")
+        if mrv is None:
+            continue
+        for sub in _walk(mrv):
+            ps, ss = sub.get("primarySubtitle"), sub.get("secondarySubtitle")
+            if isinstance(ps, str) and _VER_RE.match(ps) and isinstance(ss, str):
+                version, date = ps, from_js_date(ss)
+                break
+        if version:
+            break
+    if not version:
+        for sub in _walk(data):
+            ps, ss = sub.get("primarySubtitle"), sub.get("secondarySubtitle")
+            if isinstance(ps, str) and _VER_RE.match(ps) and isinstance(ss, str) and from_js_date(ss):
+                version, date = ps, from_js_date(ss)
+                break
+    tm = (re.search(r'<meta[^>]*property="og:title"[^>]*content="([^"]+)"', html)
+          or re.search(r'<meta[^>]*content="([^"]+)"[^>]*property="og:title"', html))
+    return {"version": version, "updated": date, "title": tm.group(1) if tm else None}
+
+
+class AppStore:
+    """Async App Store scraper.
+
+    ``app_id`` may be the numeric track id (``310633997`` / ``"id310633997"``) or a bundle id
+    (``"net.whatsapp.WhatsApp"``). Pace: unlimited by default; set ``concurrency`` and/or
+    ``rate_limit`` (requests/second) to slow down.
+    """
+
+    def __init__(
+        self,
+        *,
+        verify_with_web: bool = False,
+        web_fallback: bool = True,
+        concurrency: Optional[int] = None,
+        rate_limit: Optional[float] = None,
+        http: Optional[HttpClient] = None,
+        proxy: Optional[str] = None,
+    ) -> None:
+        self.verify_with_web = verify_with_web
+        self.web_fallback = web_fallback
+        self._own_http = http is None
+        self.http = http or HttpClient(concurrency=concurrency, rate_limit=rate_limit, proxy=proxy)
+
+    async def aclose(self) -> None:
+        if self._own_http:
+            await self.http.aclose()
+
+    async def __aenter__(self) -> "AppStore":
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.aclose()
+
+    # ------------------------------------------------------------------ raw calls
+    async def lookup(self, ids: Iterable[Union[str, int]], country: str = "US",
+                     lang: Optional[str] = None) -> list[dict[str, Any]]:
+        """Raw iTunes lookup for numeric ids and/or bundle ids (batched, concurrent)."""
+        country = normalize_country(country, APP_STORE_COUNTRIES, "App Store")
+        ids = [_clean_id(i) for i in ids]
+        numeric = [i for i in ids if i.isdigit()]
+        bundles = [i for i in ids if not i.isdigit()]
+        params_list = [{"id": ",".join(numeric[i:i + 150])} for i in range(0, len(numeric), 150)]
+        params_list += [{"bundleId": ",".join(bundles[i:i + 150])} for i in range(0, len(bundles), 150)]
+
+        async def one(params: dict) -> list[dict[str, Any]]:
+            params.update({"country": country.lower(), "entity": "software"})
+            if lang:
+                params["lang"] = lang
+            resp = await self.http.get(LOOKUP_URL, params=params)
+            if resp.status_code == 400 and "country" in resp.text:
+                raise InvalidCountry(f"{country} is not an App Store storefront")
+            if resp.status_code != 200:
+                raise StoreScraperError(f"iTunes lookup -> HTTP {resp.status_code}")
+            try:
+                data = json.loads(resp.text)
+            except ValueError as exc:
+                raise ParseError(f"iTunes lookup sent non-JSON: {resp.text[:120]!r}") from exc
+            return [r for r in data.get("results", []) if r.get("wrapperType") == "software" or "trackId" in r]
+
+        out: list[dict[str, Any]] = []
+        for chunk in await asyncio.gather(*(one(p) for p in params_list)):
+            out += chunk
+        return out
+
+    async def web(self, track_id: Union[str, int], country: str = "US") -> dict[str, Any]:
+        country = normalize_country(country, APP_STORE_COUNTRIES, "App Store")
+        resp = await self.http.get(WEB_URL.format(cc=country.lower(), id=_clean_id(track_id)))
+        if resp.status_code == 404:
+            raise AppNotFound(str(track_id), country, "app_store")
+        if resp.status_code != 200:
+            raise StoreScraperError(f"apps.apple.com {track_id}/{country} -> HTTP {resp.status_code}")
+        return parse_web_html(resp.text)
+
+    # ------------------------------------------------------------------ public
+    async def app(self, app_id: Union[str, int], country: str = "US", *, lang: Optional[str] = None,
+                  verify_with_web: Optional[bool] = None) -> AppStoreApp:
+        country = normalize_country(country, APP_STORE_COUNTRIES, "App Store")
+        results = await self.lookup([app_id], country, lang)
+        verify = self.verify_with_web if verify_with_web is None else verify_with_web
+        if results:
+            info = parse_lookup_result(results[0], country, lang)
+            if verify:
+                try:
+                    await self._merge_web(info)
+                except StoreScraperError as exc:
+                    info.raw["web_error"] = str(exc)
+            return info
+        if self.web_fallback and _is_numeric_id(app_id):
+            w = await self.web(app_id, country)  # raises AppNotFound on 404
+            if w["version"]:
+                return AppStoreApp(
+                    app_id=_clean_id(app_id), country=country, language=lang, title=w["title"],
+                    version=w["version"], version_source=VersionSource.APP_STORE_WEB.value,
+                    last_updated=to_iso(w["updated"]), track_id=int(_clean_id(app_id)),
+                    url=WEB_URL.format(cc=country.lower(), id=_clean_id(app_id)), raw={"web": w},
+                )
+        raise AppNotFound(str(app_id), country, "app_store")
+
+    async def _merge_web(self, info: AppStoreApp) -> None:
+        w = await self.web(info.app_id, info.country)
+        info.raw["web"] = w
+        if w["version"] and _version_key(w["version"]) > _version_key(info.version):
+            info.version = w["version"]
+            info.version_source = VersionSource.APP_STORE_WEB.value
+            if w["updated"]:
+                info.last_updated = to_iso(w["updated"])
+
+    async def apps(self, app_ids: Iterable[Union[str, int]], country: str = "US", *,
+                   lang: Optional[str] = None) -> dict[str, Union[AppStoreApp, Exception]]:
+        """Many apps, one country, batched into as few lookup calls as possible."""
+        ids = [_clean_id(i) for i in app_ids]
+        country = normalize_country(country, APP_STORE_COUNTRIES, "App Store")
+        results = await self.lookup(ids, country, lang)
+        by_key: dict[str, dict] = {}
+        for r in results:
+            by_key[str(r.get("trackId"))] = r
+            if r.get("bundleId"):
+                by_key[r["bundleId"]] = r
+        return {i: (parse_lookup_result(by_key[i], country, lang) if i in by_key
+                    else AppNotFound(i, country, "app_store")) for i in ids}
+
+    async def app_all_countries(
+        self, app_id: Union[str, int], countries: Optional[Iterable[str]] = None, *,
+        lang: Optional[str] = None, verify_with_web: Optional[bool] = None,
+    ) -> dict[str, Union[AppStoreApp, Exception]]:
+        """``{country: AppStoreApp | AppNotFound | error}`` for every App Store storefront."""
+        codes = [normalize_country(c, APP_STORE_COUNTRIES, "App Store") for c in (countries or APP_STORE_COUNTRIES)]
+        res = await asyncio.gather(*(self.app(app_id, c, lang=lang, verify_with_web=verify_with_web) for c in codes),
+                                   return_exceptions=True)
+        return dict(zip(codes, res))

mobile_store_scraper/countries.py

@@ -0,0 +1,46 @@
+"""Storefront country lists.
+
+``PLAY_COUNTRIES``: every location on Google's "Supported locations for distribution to
+Google Play users" table (version 2026/01, published 2026-09-28), 225 codes.
+
+``APP_STORE_COUNTRIES``: every ISO 3166-1 code accepted by the iTunes lookup API, found by
+probing all 249 ISO codes on 2026-10-24 (invalid storefronts answer HTTP 400), 184 codes.
+
+Re-validate them any time with ``python bench/bench.py --validate-countries``.
+"""
+from __future__ import annotations
+
+from .errors import InvalidCountry
+
+PLAY_COUNTRIES: tuple[str, ...] = tuple("""
+AD AE AF AG AI AL AM AO AQ AR AT AU AW AZ BA BB BD BE BF BG BH BI BJ BM BN BO BQ BR BS BT BV BW
+BY BZ CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ DE DJ DK DM DO DZ EC EE EG EH ER ES
+ET FI FJ FK FM FR GA GB GD GE GG GH GI GM GN GQ GR GS GT GW GY HK HM HN HR HT HU ID IE IL IM IN IO
+IQ IR IS IT JE JM JO JP KE KG KH KI KM KN KR KW KY KZ LA LB LC LI LK LR LS LT LU LV LY MA MC MD ME
+MG MK ML MM MN MO MR MS MT MU MV MW MX MY MZ NA NE NF NG NI NL NO NP NR NU NZ OM PA PE PG PH PK PL
+PN PS PT PY QA RO RS RU RW SA SB SC SD SE SG SH SI SK SL SM SN SO SR SS ST SV SZ TC TD TF TG TH TJ
+TK TL TM TN TO TR TT TV TW TZ UA UG UM US UY UZ VA VC VE VG VN VU WS XK YE ZA ZM ZW
+""".split())
+
+APP_STORE_COUNTRIES: tuple[str, ...] = tuple("""
+AD AE AF AG AI AL AM AO AR AT AU AZ BA BB BD BE BF BG BH BJ BM BN BO BR BS BT BW BY BZ CA CD CF CG
+CH CI CL CM CN CO CR CV CY CZ DE DK DM DO DZ EC EE EG ES ET FI FJ FM FR GA GB GD GE GH GM GN GR GT
+GW GY HK HN HR HU ID IE IL IN IQ IS IT JM JO JP KE KG KH KN KR KW KY KZ LA LB LC LI LK LR LT LU LV
+LY MA MC MD ME MG MK ML MM MN MO MR MS MT MU MV MW MX MY MZ NA NE NG NI NL NO NP NR NZ OM PA PE PG
+PH PK PL PS PT PW PY QA RO RS RU RW SA SB SC SE SG SI SK SL SN SR ST SV SZ TC TD TH TJ TM TN TO TR
+TT TW TZ UA UG US UY UZ VC VE VG VN VU WS XK YE ZA ZM ZW
+""".split())
+
+ALL_COUNTRIES: tuple[str, ...] = tuple(sorted(set(PLAY_COUNTRIES) | set(APP_STORE_COUNTRIES)))
+
+
+def normalize_country(code: str, allowed: tuple[str, ...] | None = None, store: str = "") -> str:
+    """Upper-case and validate a two-letter country code (``"uk"`` is accepted for ``GB``)."""
+    if not isinstance(code, str) or len(code.strip()) != 2:
+        raise InvalidCountry(f"{code!r} is not a two-letter country code")
+    c = code.strip().upper()
+    if c == "UK":
+        c = "GB"
+    if allowed is not None and c not in allowed:
+        raise InvalidCountry(f"{c} is not a {store or 'store'} storefront")
+    return c