keepa 1.4.0__tar.gz → 1.4.2__tar.gz

{keepa-1.4.0 → keepa-1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: keepa
-Version: 1.4.0
+Version: 1.4.2
 Summary: Interfaces with keepa.com's API.
 Keywords: keepa
 Author-email: Alex Kaszynski <akascap@gmail.com>

{keepa-1.4.0 → keepa-1.4.2}/pyproject.toml

@@ -33,7 +33,7 @@ keywords = ["keepa"]
 name = "keepa"
 readme = "README.rst"
 requires-python = ">=3.10"
-version = "1.4.0"
+version = "1.4.2"
 
 [project.optional-dependencies]
 doc = [

{keepa-1.4.0 → keepa-1.4.2}/src/keepa/__init__.py

@@ -8,23 +8,20 @@ try:
 except PackageNotFoundError:
     __version__ = "unknown"
 
-from keepa.data_models import ProductParams
-from keepa.interface import (
-    DCODES,
-    KEEPA_ST_ORDINAL,
-    SCODES,
-    AsyncKeepa,
-    Domain,
-    Keepa,
+from keepa.constants import DCODES, KEEPA_ST_ORDINAL, SCODES, csv_indices
+from keepa.keepa_async import AsyncKeepa
+from keepa.keepa_sync import Keepa
+from keepa.models.domain import Domain
+from keepa.models.product_params import ProductParams
+from keepa.plotting import plot_product
+from keepa.utils import (
     convert_offer_history,
-    csv_indices,
     format_items,
     keepa_minutes_to_time,
     parse_csv,
     process_used_buybox,
     run_and_get,
 )
-from keepa.plotting import plot_product
 
 __all__ = [
     "AsyncKeepa",

keepa-1.4.2/src/keepa/constants.py

@@ -0,0 +1,78 @@
+"""Constants for Keepa API interactions."""
+
+import numpy as np
+
+# hardcoded ordinal time from
+KEEPA_ST_ORDINAL = np.datetime64("2011-01-01")
+
+# Request limit
+REQUEST_LIMIT = 100
+
+# Status code dictionary/key
+SCODES = {
+    "400": "REQUEST_REJECTED",
+    "402": "PAYMENT_REQUIRED",
+    "405": "METHOD_NOT_ALLOWED",
+    "429": "NOT_ENOUGH_TOKEN",
+}
+
+# domain codes
+# Valid values: [ 1: com | 2: co.uk | 3: de | 4: fr | 5:
+#                 co.jp | 6: ca | 7: cn | 8: it | 9: es | 10: in | 11: com.mx | 12: com.br ]
+DCODES = [
+    "RESERVED",
+    "US",
+    "GB",
+    "DE",
+    "FR",
+    "JP",
+    "CA",
+    "CN",
+    "IT",
+    "ES",
+    "IN",
+    "MX",
+    "BR",
+]
+# developer note: appears like CN (China) has changed to RESERVED2
+
+# csv indices. used when parsing csv and stats fields.
+# https://github.com/keepacom/api_backend
+# see api_backend/src/main/java/com/keepa/api/backend/structs/Product.java
+# [index in csv, key name, isfloat(is price or rating)]
+csv_indices: list[tuple[int, str, bool]] = [
+    (0, "AMAZON", True),
+    (1, "NEW", True),
+    (2, "USED", True),
+    (3, "SALES", False),
+    (4, "LISTPRICE", True),
+    (5, "COLLECTIBLE", True),
+    (6, "REFURBISHED", True),
+    (7, "NEW_FBM_SHIPPING", True),
+    (8, "LIGHTNING_DEAL", True),
+    (9, "WAREHOUSE", True),
+    (10, "NEW_FBA", True),
+    (11, "COUNT_NEW", False),
+    (12, "COUNT_USED", False),
+    (13, "COUNT_REFURBISHED", False),
+    (14, "CollectableOffers", False),
+    (15, "EXTRA_INFO_UPDATES", False),
+    (16, "RATING", True),
+    (17, "COUNT_REVIEWS", False),
+    (18, "BUY_BOX_SHIPPING", True),
+    (19, "USED_NEW_SHIPPING", True),
+    (20, "USED_VERY_GOOD_SHIPPING", True),
+    (21, "USED_GOOD_SHIPPING", True),
+    (22, "USED_ACCEPTABLE_SHIPPING", True),
+    (23, "COLLECTIBLE_NEW_SHIPPING", True),
+    (24, "COLLECTIBLE_VERY_GOOD_SHIPPING", True),
+    (25, "COLLECTIBLE_GOOD_SHIPPING", True),
+    (26, "COLLECTIBLE_ACCEPTABLE_SHIPPING", True),
+    (27, "REFURBISHED_SHIPPING", True),
+    (28, "EBAY_NEW_SHIPPING", True),
+    (29, "EBAY_USED_SHIPPING", True),
+    (30, "TRADE_IN", True),
+    (31, "RENT", False),
+]
+
+_SELLER_TIME_DATA_KEYS = ["trackedSince", "lastUpdate"]

keepa-1.4.2/src/keepa/keepa_async.py

@@ -0,0 +1,566 @@
+"""Interface module to download Amazon product and history data from keepa.com asynchronously."""
+
+import asyncio
+import json
+import logging
+import time
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any, Literal
+
+import aiohttp
+from tqdm import tqdm
+
+from keepa.constants import SCODES
+from keepa.keepa_sync import Keepa
+from keepa.models.domain import Domain
+from keepa.models.product_params import ProductParams
+from keepa.models.status import Status
+from keepa.query_keys import DEAL_REQUEST_KEYS
+from keepa.utils import (
+    _domain_to_dcode,
+    _parse_seller,
+    _parse_stats,
+    format_items,
+    is_documented_by,
+    parse_csv,
+)
+
+log = logging.getLogger(__name__)
+
+
+# Request limit
+REQUEST_LIMIT = 100
+
+
+class AsyncKeepa:
+    r"""
+    Asynchronous Python interface to keepa backend.
+
+    Initializes API with access key. Access key can be obtained by signing up
+    for a reoccurring or one time plan. To obtain a key, sign up for one at
+    `Keepa Data <https://get.keepa.com/d7vrq>`_.
+
+    Parameters
+    ----------
+    accesskey : str
+        64 character access key string.
+    timeout : float, default: 10.0
+        Default timeout when issuing any request.  This is not a time
+        limit on the entire response download; rather, an exception is
+        raised if the server has not issued a response for timeout
+        seconds.  Setting this to 0.0 disables the timeout, but will
+        cause any request to hang indefiantly should keepa.com be down
+
+    Examples
+    --------
+    Query for all of Jim Butcher's books using the asynchronous
+    ``keepa.AsyncKeepa`` class.
+
+    >>> import asyncio
+    >>> import keepa
+    >>> product_parms = {"author": "jim butcher"}
+    >>> async def main():
+    ...     key = "<REAL_KEEPA_KEY>"
+    ...     api = await keepa.AsyncKeepa().create(key)
+    ...     return await api.product_finder(product_parms)
+    ...
+    >>> asins = asyncio.run(main())
+    >>> asins
+    ['B000HRMAR2',
+     '0578799790',
+     'B07PW1SVHM',
+    ...
+     'B003MXM744',
+     '0133235750',
+     'B01MXXLJPZ']
+
+    Query for product with ASIN ``'B0088PUEPK'`` using the asynchronous
+    keepa interface.
+
+    >>> import asyncio
+    >>> import keepa
+    >>> async def main():
+    ...     key = "<REAL_KEEPA_KEY>"
+    ...     api = await keepa.AsyncKeepa().create(key)
+    ...     return await api.query("B0088PUEPK")
+    ...
+    >>> response = asyncio.run(main())
+    >>> response[0]["title"]
+    'Western Digital 1TB WD Blue PC Internal Hard Drive HDD - 7200 RPM,
+    SATA 6 Gb/s, 64 MB Cache, 3.5" - WD10EZEX'
+
+    """
+
+    accesskey: str
+    tokens_left: int
+    status: Status
+    _timeout: float
+
+    @classmethod
+    async def create(cls, accesskey: str, timeout: float = 10.0) -> "AsyncKeepa":
+        """Create the async object."""
+        self = AsyncKeepa()
+        self.accesskey = accesskey
+        self.tokens_left = 0
+        self._timeout = timeout
+
+        # don't update the user status on init
+        self.status = Status()
+        return self
+
+    @property
+    def time_to_refill(self) -> float:
+        """Return the time to refill in seconds."""
+        # Get current timestamp in milliseconds from UNIX epoch
+        now = int(time.time() * 1000)
+        time_at_refill = self.status.timestamp + self.status.refillIn
+
+        # wait plus one second fudge factor
+        time_to_refill = time_at_refill - now + 1000
+        if time_to_refill < 0:
+            time_to_refill = 0
+
+        # Account for negative tokens left
+        if self.tokens_left < 0:
+            time_to_refill += (abs(self.tokens_left) / self.status.refillRate) * 60000
+
+        # Return value in seconds
+        return time_to_refill / 1000.0
+
+    async def update_status(self) -> None:
+        """Update available tokens."""
+        self.status = await self._request("token", {"key": self.accesskey}, wait=False)
+
+    async def wait_for_tokens(self) -> None:
+        """Check if there are any remaining tokens and waits if none are available."""
+        await self.update_status()
+
+        # Wait if no tokens available
+        if self.tokens_left <= 0:
+            tdelay = self.time_to_refill
+            log.warning("Waiting %.0f seconds for additional tokens", tdelay)
+            await asyncio.sleep(tdelay)
+            await self.update_status()
+
+    @is_documented_by(Keepa.query)
+    async def query(
+        self,
+        items: str | Sequence[str],
+        stats: int | None = None,
+        domain: str = "US",
+        history: bool = True,
+        offers: int | None = None,
+        update: int | None = None,
+        to_datetime: bool = True,
+        rating: bool = False,
+        out_of_stock_as_nan: bool = True,
+        stock: bool = False,
+        product_code_is_asin: bool = True,
+        progress_bar: bool = True,
+        buybox: bool = False,
+        wait: bool = True,
+        days: int | None = None,
+        only_live_offers: bool | None = None,
+        raw: bool = False,
+        videos: bool = False,
+        aplus: bool = False,
+        extra_params: dict[str, Any] | None = None,
+    ):
+        """Documented in Keepa.query."""
+        if raw:
+            raise ValueError("Raw response is only available in the non-async class")
+
+        if extra_params is None:
+            extra_params = {}
+
+        # Format items into numpy array
+        try:
+            items = format_items(items)
+        except BaseException:
+            raise Exception("Invalid product codes input")
+        assert len(items), "No valid product codes"
+
+        nitems = len(items)
+        if nitems == 1:
+            log.debug("Executing single product query")
+        else:
+            log.debug("Executing %d item product query", nitems)
+
+        # check offer input
+        if offers:
+            if not isinstance(offers, int):
+                raise TypeError('Parameter "offers" must be an interger')
+
+            if offers > 100 or offers < 20:
+                raise ValueError('Parameter "offers" must be between 20 and 100')
+
+        # Report time to completion
+        if self.status.refillRate is not None and self.status.refillIn is not None:
+            tcomplete = (
+                float(nitems - self.tokens_left) / self.status.refillRate
+                - (60000 - self.status.refillIn) / 60000.0
+            )
+            if tcomplete < 0.0:
+                tcomplete = 0.5
+            log.debug(
+                "Estimated time to complete %d request(s) is %.2f minutes",
+                nitems,
+                tcomplete,
+            )
+            log.debug("\twith a refill rate of %d token(s) per minute", self.status.refillRate)
+
+        # product list
+        products = []
+
+        pbar = None
+        if progress_bar:
+            pbar = tqdm(total=nitems)
+
+        # Number of requests is dependent on the number of items and
+        # request limit.  Use available tokens first
+        idx = 0  # or number complete
+        while idx < nitems:
+            nrequest = nitems - idx
+
+            # cap request
+            if nrequest > REQUEST_LIMIT:
+                nrequest = REQUEST_LIMIT
+
+            # request from keepa and increment current position
+            item_request = items[idx : idx + nrequest]  # noqa: E203
+            response = await self._product_query(
+                item_request,
+                product_code_is_asin,
+                stats=stats,
+                domain=domain,
+                stock=stock,
+                offers=offers,
+                update=update,
+                history=history,
+                rating=rating,
+                to_datetime=to_datetime,
+                out_of_stock_as_nan=out_of_stock_as_nan,
+                buybox=buybox,
+                wait=wait,
+                days=days,
+                only_live_offers=only_live_offers,
+                videos=videos,
+                aplus=aplus,
+                **extra_params,
+            )
+            idx += nrequest
+            products.extend(response["products"])
+
+            if pbar is not None:
+                pbar.update(nrequest)
+
+        return products
+
+    @is_documented_by(Keepa._product_query)
+    async def _product_query(self, items, product_code_is_asin=True, **kwargs):
+        """Documented in Keepa._product_query."""
+        # ASINs convert to comma joined string
+        assert len(items) <= 100
+
+        if product_code_is_asin:
+            kwargs["asin"] = ",".join(items)
+        else:
+            kwargs["code"] = ",".join(items)
+
+        kwargs["key"] = self.accesskey
+        kwargs["domain"] = _domain_to_dcode(kwargs["domain"])
+
+        # Convert bool values to 0 and 1.
+        kwargs["stock"] = int(kwargs["stock"])
+        kwargs["history"] = int(kwargs["history"])
+        kwargs["rating"] = int(kwargs["rating"])
+        kwargs["buybox"] = int(kwargs["buybox"])
+
+        if kwargs["update"] is None:
+            del kwargs["update"]
+        else:
+            kwargs["update"] = int(kwargs["update"])
+
+        if kwargs["offers"] is None:
+            del kwargs["offers"]
+        else:
+            kwargs["offers"] = int(kwargs["offers"])
+
+        if kwargs["only_live_offers"] is None:
+            del kwargs["only_live_offers"]
+        else:
+            kwargs["only-live-offers"] = int(kwargs.pop("only_live_offers"))
+            # Keepa's param actually doesn't use snake_case.
+            # Keeping with snake case for consistency
+
+        if kwargs["days"] is None:
+            del kwargs["days"]
+        else:
+            assert kwargs["days"] > 0
+
+        if kwargs["stats"] is None:
+            del kwargs["stats"]
+
+        # videos and aplus must be ints
+        kwargs["videos"] = int(kwargs["videos"])
+        kwargs["aplus"] = int(kwargs["aplus"])
+
+        out_of_stock_as_nan = kwargs.pop("out_of_stock_as_nan", True)
+        to_datetime = kwargs.pop("to_datetime", True)
+
+        # Query and replace csv with parsed data if history enabled
+        wait = kwargs.get("wait")
+        kwargs.pop("wait", None)
+
+        raw_response = kwargs.pop("raw", False)
+        response = await self._request("product", kwargs, wait=wait, raw_response=raw_response)
+        if kwargs["history"]:
+            if "products" not in response:
+                raise RuntimeError("No products in response. Possibly invalid ASINs")
+
+            for product in response["products"]:
+                if product["csv"]:  # if data exists
+                    product["data"] = parse_csv(product["csv"], to_datetime, out_of_stock_as_nan)
+
+        if kwargs.get("stats", None):
+            for product in response["products"]:
+                stats = product.get("stats", None)
+                if stats:
+                    product["stats_parsed"] = _parse_stats(stats, to_datetime)
+
+        return response
+
+    @is_documented_by(Keepa.best_sellers_query)
+    async def best_sellers_query(
+        self,
+        category: str,
+        rank_avg_range: Literal[0, 30, 90, 180] = 0,
+        variations: bool = False,
+        sublist: bool = False,
+        domain: str | Domain = "US",
+        wait: bool = True,
+    ):
+        """Documented by Keepa.best_sellers_query."""
+        payload = {
+            "key": self.accesskey,
+            "domain": _domain_to_dcode(domain),
+            "variations": int(variations),
+            "sublist": int(sublist),
+            "category": category,
+            "range": rank_avg_range,
+        }
+
+        response = await self._request("bestsellers", payload, wait=wait)
+        if "bestSellersList" not in response:
+            raise RuntimeError(f"Best sellers search results for {category} not yet available")
+        return response["bestSellersList"]["asinList"]
+
+    @is_documented_by(Keepa.search_for_categories)
+    async def search_for_categories(
+        self, searchterm, domain: str | Domain = "US", wait: bool = True
+    ):
+        """Documented by Keepa.search_for_categories."""
+        payload = {
+            "key": self.accesskey,
+            "domain": _domain_to_dcode(domain),
+            "type": "category",
+            "term": searchterm,
+        }
+
+        response = await self._request("search", payload, wait=wait)
+        if response["categories"] == {}:  # pragma no cover
+            raise Exception(
+                "Categories search results not yet available " + "or no search terms found."
+            )
+        else:
+            return response["categories"]
+
+    @is_documented_by(Keepa.category_lookup)
+    async def category_lookup(
+        self,
+        category_id,
+        domain: str | Domain = "US",
+        include_parents=0,
+        wait: bool = True,
+    ):
+        """Documented by Keepa.category_lookup."""
+        payload = {
+            "key": self.accesskey,
+            "domain": _domain_to_dcode(domain),
+            "category": category_id,
+            "parents": include_parents,
+        }
+
+        response = await self._request("category", payload, wait=wait)
+        if response["categories"] == {}:  # pragma no cover
+            raise Exception("Category lookup results not yet available or no" + "match found.")
+        else:
+            return response["categories"]
+
+    @is_documented_by(Keepa.seller_query)
+    async def seller_query(
+        self,
+        seller_id,
+        domain: str | Domain = "US",
+        to_datetime=True,
+        storefront=False,
+        update=None,
+        wait: bool = True,
+    ):
+        """Documented by Keepa.sellerer_query."""
+        if isinstance(seller_id, list):
+            if len(seller_id) > 100:
+                err_str = "seller_id can contain at maximum 100 sellers"
+                raise RuntimeError(err_str)
+            seller = ",".join(seller_id)
+        else:
+            seller = seller_id
+
+        payload = {
+            "key": self.accesskey,
+            "domain": _domain_to_dcode(domain),
+            "seller": seller,
+        }
+
+        if storefront:
+            payload["storefront"] = int(storefront)
+        if update:
+            payload["update"] = update
+
+        response = await self._request("seller", payload, wait=wait)
+        return _parse_seller(response["sellers"], to_datetime)
+
+    @is_documented_by(Keepa.product_finder)
+    async def product_finder(
+        self,
+        product_parms: dict[str, Any] | ProductParams,
+        domain: str | Domain = "US",
+        wait: bool = True,
+        n_products: int = 50,
+    ) -> list[str]:
+        """Documented by Keepa.product_finder."""
+        if isinstance(product_parms, dict):
+            product_parms_valid = ProductParams(**product_parms)
+        else:
+            product_parms_valid = product_parms
+        product_parms_dict = product_parms_valid.model_dump(exclude_none=True)
+        product_parms_dict.setdefault("perPage", n_products)
+        payload = {
+            "key": self.accesskey,
+            "domain": _domain_to_dcode(domain),
+            "selection": json.dumps(product_parms_dict),
+        }
+
+        response = await self._request("query", payload, wait=wait)
+        return response["asinList"]
+
+    @is_documented_by(Keepa.deals)
+    async def deals(self, deal_parms, domain: str | Domain = "US", wait: bool = True):
+        """Documented in Keepa.deals."""
+        # verify valid keys
+        for key in deal_parms:
+            if key not in DEAL_REQUEST_KEYS:
+                raise ValueError(f'Invalid key "{key}"')
+
+            # verify json type
+            key_type = DEAL_REQUEST_KEYS[key]
+            deal_parms[key] = key_type(deal_parms[key])
+
+        deal_parms.setdefault("priceTypes", 0)
+
+        payload = {
+            "key": self.accesskey,
+            "domain": _domain_to_dcode(domain),
+            "selection": json.dumps(deal_parms),
+        }
+
+        deals = await self._request("deal", payload, wait=wait)
+        return deals["deals"]
+
+    @is_documented_by(Keepa.download_graph_image)
+    async def download_graph_image(
+        self,
+        asin: str,
+        filename: str | Path,
+        domain: str | Domain = "US",
+        wait: bool = True,
+        **graph_kwargs: dict[str, Any],
+    ) -> None:
+        """Documented in Keepa.download_graph_image."""
+        payload = {
+            "asin": asin,
+            "key": self.accesskey,
+            "domain": _domain_to_dcode(domain),
+        }
+        payload.update(graph_kwargs)
+
+        async with aiohttp.ClientSession() as session:
+            async with session.get(
+                "https://api.keepa.com/graphimage",
+                params=payload,
+                timeout=self._timeout,
+            ) as resp:
+                first_chunk = True
+                filename = Path(filename)
+                with open(filename, "wb") as f:
+                    async for chunk in resp.content.iter_chunked(8192):
+                        if first_chunk:
+                            if not chunk.startswith(b"\x89PNG\r\n\x1a\n"):
+                                raise ValueError(
+                                    "Response from api.keepa.com/graphimage is not a valid "
+                                    "PNG image"
+                                )
+                            first_chunk = False
+                        f.write(chunk)
+
+    async def _request(
+        self,
+        request_type: str,
+        payload: dict[str, Any],
+        wait: bool = True,
+        raw_response: bool = False,
+        is_json: bool = True,
+    ):
+        """Documented in Keepa._request."""
+        while True:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(
+                    f"https://api.keepa.com/{request_type}/?",
+                    params=payload,
+                    timeout=self._timeout,
+                ) as raw:
+                    status_code = str(raw.status)
+
+                    if not is_json:
+                        return raw
+
+                    try:
+                        response = await raw.json()
+                    except Exception:
+                        raise RuntimeError(f"Invalid JSON from Keepa API (status {status_code})")
+
+                    # user status is always returned
+                    if "tokensLeft" in response:
+                        self.tokens_left = response["tokensLeft"]
+                        self.status.tokensLeft = self.tokens_left
+                        log.info("%d tokens remain", self.tokens_left)
+                    for key in ["refillIn", "refillRate", "timestamp"]:
+                        if key in response:
+                            setattr(self.status, key, response[key])
+
+                    if status_code == "200":
+                        if raw_response:
+                            return raw
+                        return response
+
+                    if status_code == "429" and wait:
+                        tdelay = self.time_to_refill
+                        log.warning("Waiting %.0f seconds for additional tokens", tdelay)
+                        time.sleep(tdelay)
+                        continue
+
+                    # otherwise, it's an error code
+                    if status_code in SCODES:
+                        raise RuntimeError(SCODES[status_code])
+                    raise RuntimeError(f"REQUEST_FAILED. Status code: {status_code}")