tbr-deal-finder 0.2.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

tbr_deal_finder/retailer/chirp.py

@@ -3,13 +3,12 @@ import json
 import os
 from datetime import datetime, timedelta
 from textwrap import dedent
+from typing import Union
 
-import aiohttp
 import click
 
-from tbr_deal_finder import TBR_DEALS_PATH
 from tbr_deal_finder.config import Config
-from tbr_deal_finder.retailer.models import AioHttpSession, Retailer
+from tbr_deal_finder.retailer.models import AioHttpSession, Retailer, GuiAuthContext
 from tbr_deal_finder.book import Book, BookFormat, get_normalized_authors, is_matching_authors
 from tbr_deal_finder.utils import currency_to_float, echo_err
 
@@ -52,17 +51,21 @@ class Chirp(AioHttpSession, Retailer):
         else:
             return {}
 
-    async def set_auth(self):
-        auth_path = TBR_DEALS_PATH.joinpath("chirp.json")
-        if os.path.exists(auth_path):
-            with open(auth_path, "r") as f:
+    def user_is_authed(self) -> bool:
+        if os.path.exists(self.auth_path):
+            with open(self.auth_path, "r") as f:
                 auth_info = json.load(f)
                 if auth_info:
                     token_created_at = datetime.fromtimestamp(auth_info["created_at"])
-                    max_token_age = datetime.now() - timedelta(days=5)
+                    max_token_age = datetime.now() - timedelta(days=7)
                     if token_created_at > max_token_age:
                         self.auth_token = auth_info["data"]["signIn"]["user"]["token"]
-                        return
+                        return True
+        return False
+
+    async def set_auth(self):
+        if self.user_is_authed():
+            return
 
         response = await self.make_request(
             "POST",
@@ -82,12 +85,53 @@ class Chirp(AioHttpSession, Retailer):
         self.auth_token = response["data"]["signIn"]["user"]["token"]
 
         response["created_at"] = datetime.now().timestamp()
-        with open(auth_path, "w") as f:
+        with open(self.auth_path, "w") as f:
+            json.dump(response, f)
+
+    @property
+    def gui_auth_context(self) -> GuiAuthContext:
+        return GuiAuthContext(
+            title="Login to Chirp",
+            fields=[
+                {"name": "email", "label": "Email", "type": "email"},
+                {"name": "password", "label": "Password", "type": "password"}
+            ]
+        )
+
+    async def gui_auth(self, form_data: dict) -> bool:
+        response = await self.make_request(
+            "POST",
+            json={
+                "query": "mutation signIn($email: String!, $password: String!) { signIn(email: $email, password: $password) { user { id token webToken email } } }",
+                "variables": {
+                    "email": form_data["email"],
+                    "password": form_data["password"],
+                }
+            }
+        )
+        if not response:
+            return False
+
+        auth_token = response.get("data", {})
+        for key in ["signIn", "user", "token"]:
+            if key not in auth_token:
+                return False
+            auth_token = auth_token[key]
+
+            if not auth_token:
+                return False
+
+        # Set token for future requests during the current execution
+        self.auth_token = auth_token
+
+        response["created_at"] = datetime.now().timestamp()
+        with open(self.auth_path, "w") as f:
             json.dump(response, f)
+        return True
 
     async def get_book(
         self, target: Book, semaphore: asyncio.Semaphore
-    ) -> Book:
+    ) -> Union[Book, None]:
         title = target.title
         async with semaphore:
             session = await self._get_session()

tbr_deal_finder/retailer/kindle.py

@@ -1,13 +1,17 @@
 import asyncio
-import readline  # type: ignore
+import json
+from typing import Union
 
 from tbr_deal_finder.config import Config
-from tbr_deal_finder.retailer.amazon import Amazon
+from tbr_deal_finder.retailer.amazon import Amazon, AUTH_PATH
 from tbr_deal_finder.book import Book, BookFormat, get_normalized_title, get_normalized_authors, is_matching_authors
 
 
 class Kindle(Amazon):
 
+    def __init__(self):
+        self._headers = {}
+
     @property
     def name(self) -> str:
         return "Kindle"
@@ -16,9 +20,32 @@ class Kindle(Amazon):
     def format(self) -> BookFormat:
         return BookFormat.EBOOK
 
+    @property
+    def max_concurrency(self) -> int:
+        return 3
+
     def _get_base_url(self) -> str:
         return f"https://www.amazon.{self._auth.locale.domain}"
 
+    def _get_read_base_url(self) -> str:
+        return f"https://read.amazon.{self._auth.locale.domain}"
+
+    async def set_auth(self):
+        await super().set_auth()
+
+        with open(AUTH_PATH, "r") as f:
+            auth_info = json.load(f)
+
+            cookies = auth_info["website_cookies"]
+            cookies["x-access-token"] = auth_info["access_token"]
+
+        self._headers = {
+            "User-Agent": "Mozilla/5.0 (Linux; Android 10; Kindle) AppleWebKit/537.3",
+            "Accept": "application/json, */*",
+            "Cookie": "; ".join([f"{k}={v}" for k, v in cookies.items()])
+        }
+
+
     async def get_book_asin(
         self,
         target: Book,
@@ -50,7 +77,7 @@ class Kindle(Amazon):
         self,
         target: Book,
         semaphore: asyncio.Semaphore
-    ) -> Book:
+    ) -> Union[Book, None]:
         target.exists = False
 
         if not target.ebook_asin:
@@ -58,26 +85,34 @@ class Kindle(Amazon):
 
         asin = target.ebook_asin
         async with semaphore:
-            match = await self._client.get(
-                f"{self._get_base_url()}/api/bifrost/offers/batch/v1/{asin}?ref_=KindleDeepLinkOffers",
-                headers={"x-client-id": "kindle-android-deeplink"},
-            )
-            products = match.get("resources", [])
-            if not products:
-                return target
+            for i in range(10):
+                match = await self._client.get(
+                    f"{self._get_base_url()}/api/bifrost/offers/batch/v1/{asin}?ref_=KindleDeepLinkOffers",
+                    headers={"x-client-id": "kindle-android-deeplink"},
+                )
+                products = match.get("resources", [])
+                if not products:
+                    await asyncio.sleep(1)
+                    continue
 
-            actions = products[0].get("personalizedActionOutput", {}).get("personalizedActions", [])
-            if not actions:
-                return target
+                actions = products[0].get("personalizedActionOutput", {}).get("personalizedActions", [])
+                if not actions:
+                    await asyncio.sleep(1)
+                    continue
 
-            for action in actions:
-                if "printListPrice" in action["offer"]:
-                    target.list_price = action["offer"]["printListPrice"]["value"]
-                    target.current_price = action["offer"]["digitalPrice"]["value"]
-                    target.exists = True
-                    break
+                for action in actions:
+                    if "printListPrice" in action["offer"]:
+                        target.list_price = action["offer"]["printListPrice"]["value"]
+                        target.current_price = action["offer"]["digitalPrice"]["value"]
+                        target.exists = True
+                        break
 
-            return target
+                # The sleep is a pre-emptive backoff
+                # Concurrency is already low, but this endpoint loves to throttle
+                await asyncio.sleep(.25)
+                return target
+
+            return None
 
     async def get_wishlist(self, config: Config) -> list[Book]:
         """Not currently supported
@@ -90,44 +125,28 @@ class Kindle(Amazon):
         return []
 
     async def get_library(self, config: Config) -> list[Book]:
-        """Not currently supported
-
-        Getting this info is proving to be a nightmare
-
-        :param config:
-        :return:
-        """
-        return []
-
-    async def _get_library_attempt(self, config: Config) -> list[Book]:
-        """This should work, but it's returning a redirect
-
-        The user is already authenticated at this point, so I'm not sure what's happening
-        """
-        response = []
+        books = []
         pagination_token = 0
-        total_pages = 1
+        url = f"{self._get_read_base_url()}/kindle-library/search"
 
-        while pagination_token < total_pages:
+        while True:
             optional_params = {}
             if pagination_token:
                 optional_params["paginationToken"] = pagination_token
 
             response = await self._client.get(
-                "https://read.amazon.com/kindle-library/search",
+                url,
+                headers=self._headers,
                 query="",
                 libraryType="BOOKS",
                 sortType="recency",
                 resourceType="EBOOK",
-                querySize=5,
+                querySize=50,
                 **optional_params
             )
 
-            if "paginationToken" in response:
-                total_pages = int(response["paginationToken"])
-
             for book in response["itemsList"]:
-                response.append(
+                books.append(
                     Book(
                         retailer=self.name,
                         title = book["title"],
@@ -138,4 +157,9 @@ class Kindle(Amazon):
                     )
                 )
 
-        return response
+            if "paginationToken" in response:
+                pagination_token = int(response["paginationToken"])
+            else:
+                break
+
+        return books

tbr_deal_finder/retailer/librofm.py

@@ -3,14 +3,14 @@ import json
 import os
 import urllib.parse
 from datetime import datetime, timedelta
+from typing import Union
 
 import click
 
-from tbr_deal_finder import TBR_DEALS_PATH
 from tbr_deal_finder.config import Config
-from tbr_deal_finder.retailer.models import AioHttpSession, Retailer
+from tbr_deal_finder.retailer.models import AioHttpSession, Retailer, GuiAuthContext
 from tbr_deal_finder.book import Book, BookFormat, get_normalized_authors, is_matching_authors, get_normalized_title
-from tbr_deal_finder.utils import currency_to_float
+from tbr_deal_finder.utils import currency_to_float, echo_err
 
 
 class LibroFM(AioHttpSession, Retailer):
@@ -57,16 +57,21 @@ class LibroFM(AioHttpSession, Retailer):
         else:
             return {}
 
-    async def set_auth(self):
-        auth_path = TBR_DEALS_PATH.joinpath("libro_fm.json")
-        if os.path.exists(auth_path):
-            with open(auth_path, "r") as f:
+    def user_is_authed(self) -> bool:
+        if os.path.exists(self.auth_path):
+            with open(self.auth_path, "r") as f:
                 auth_info = json.load(f)
                 token_created_at = datetime.fromtimestamp(auth_info["created_at"])
-                max_token_age = datetime.now() - timedelta(days=5)
+                max_token_age = datetime.now() - timedelta(days=7)
                 if token_created_at > max_token_age:
                     self.auth_token = auth_info["access_token"]
-                    return
+                    return True
+
+        return False
+
+    async def set_auth(self):
+        if self.user_is_authed():
+            return
 
         response = await self.make_request(
             "/oauth/token",
@@ -77,10 +82,42 @@ class LibroFM(AioHttpSession, Retailer):
                 "password": click.prompt("Libro FM Password", hide_input=True),
             }
         )
+        if "access_token" not in response:
+            echo_err("Login failed. Try again.")
+            await self.set_auth()
+
         self.auth_token = response["access_token"]
-        with open(auth_path, "w") as f:
+        with open(self.auth_path, "w") as f:
             json.dump(response, f)
 
+    @property
+    def gui_auth_context(self) -> GuiAuthContext:
+        return GuiAuthContext(
+            title="Login to Libro.FM",
+            fields=[
+                {"name": "username", "label": "Username", "type": "text"},
+                {"name": "password", "label": "Password", "type": "password"}
+            ]
+        )
+
+    async def gui_auth(self, form_data: dict) -> bool:
+        response = await self.make_request(
+            "/oauth/token",
+            "POST",
+            json={
+                "grant_type": "password",
+                "username": form_data["username"],
+                "password": form_data["password"],
+            }
+        )
+        if "access_token" not in response:
+            return False
+
+        self.auth_token = response["access_token"]
+        with open(self.auth_path, "w") as f:
+            json.dump(response, f)
+        return True
+
     async def get_book_isbn(self, book: Book, semaphore: asyncio.Semaphore) -> Book:
         # runtime isn't used but get_book_isbn must follow the get_book method signature.
 
@@ -111,24 +148,24 @@ class LibroFM(AioHttpSession, Retailer):
 
     async def get_book(
         self, target: Book, semaphore: asyncio.Semaphore
-    ) -> Book:
+    ) -> Union[Book, None]:
         if not target.audiobook_isbn:
             target.exists = False
             return target
 
         async with semaphore:
-            response = await self.make_request(
-                f"api/v10/explore/audiobook_details/{target.audiobook_isbn}",
-                "GET"
-            )
+            for _ in range(10):
+                response = await self.make_request(
+                    f"api/v10/explore/audiobook_details/{target.audiobook_isbn}",
+                    "GET"
+                )
 
-        if response:
-            target.list_price = target.audiobook_list_price
-            target.current_price = currency_to_float(response["data"]["purchase_info"]["price"])
-            return target
+                if response:
+                    target.list_price = target.audiobook_list_price
+                    target.current_price = currency_to_float(response["data"]["purchase_info"]["price"])
+                    return target
 
-        target.exists = False
-        return target
+        return None
 
     async def get_wishlist(self, config: Config) -> list[Book]:
         wishlist_books = []

tbr_deal_finder/retailer/models.py

@@ -1,10 +1,23 @@
 import abc
 import asyncio
+import dataclasses
+from pathlib import Path
+from typing import Optional, Union
 
 import aiohttp
 
 from tbr_deal_finder.book import Book, BookFormat
 from tbr_deal_finder.config import Config
+from tbr_deal_finder.utils import get_data_dir
+
+
+@dataclasses.dataclass
+class GuiAuthContext:
+    title: str
+    fields: list[dict]
+    message: Optional[str] = None
+    user_copy_context: Optional[str] = None
+    pop_up_type: Optional[str] = "form"
 
 
 class Retailer(abc.ABC):
@@ -26,12 +39,29 @@ class Retailer(abc.ABC):
         """
         raise NotImplementedError
 
+    @property
+    def auth_path(self) -> Path:
+        name = self.name.replace(".", "").lower()
+        return get_data_dir().joinpath(f"{name}.json")
+
+    @property
+    def gui_auth_context(self) -> GuiAuthContext:
+        raise NotImplementedError
+
+    @property
+    def max_concurrency(self) -> int:
+        # The max number of simultaneous requests to send to this retailer
+        return 10
+
     async def set_auth(self):
         raise NotImplementedError
 
+    async def gui_auth(self, form_data: dict) -> bool:
+        raise NotImplementedError
+
     async def get_book(
             self, target: Book, semaphore: asyncio.Semaphore
-    ) -> Book:
+    ) -> Union[Book, None]:
         """Get book information from the retailer.
 
         - Uses Audible's product API to fetch book details

tbr_deal_finder/retailer_deal.py

@@ -2,16 +2,14 @@ import asyncio
 import copy
 from collections import defaultdict
 
-import click
 import pandas as pd
-from tqdm.asyncio import tqdm_asyncio
 
 from tbr_deal_finder.book import Book, get_active_deals, BookFormat
 from tbr_deal_finder.config import Config
-from tbr_deal_finder.tracked_books import get_tbr_books
+from tbr_deal_finder.tracked_books import get_tbr_books, get_unknown_books, set_unknown_books
 from tbr_deal_finder.retailer import RETAILER_MAP
 from tbr_deal_finder.retailer.models import Retailer
-from tbr_deal_finder.utils import get_duckdb_conn, echo_warning, echo_info
+from tbr_deal_finder.utils import get_duckdb_conn, echo_info, echo_err
 
 
 def update_retailer_deal_table(config: Config, new_deals: list[Book]):
@@ -38,14 +36,6 @@ def update_retailer_deal_table(config: Config, new_deals: list[Book]):
         else:
             df_data.append(deal.dict())
 
-    # Any remaining values in active_deal_map mean that
-    # it wasn't found and should be marked for deletion
-    for deal in active_deal_map.values():
-        echo_warning(f"{str(deal)} is no longer active\n")
-        deal.timepoint = config.run_time
-        deal.deleted = True
-        df_data.append(deal.dict())
-
     if df_data:
         df = pd.DataFrame(df_data)
 
@@ -55,7 +45,12 @@ def update_retailer_deal_table(config: Config, new_deals: list[Book]):
         db_conn.unregister("_df")
 
 
-async def _get_books(config, retailer: Retailer, books: list[Book]) -> list[Book]:
+async def _get_books(
+    config, 
+    retailer: Retailer, 
+    books: list[Book],
+    ignored_deal_ids: set[str],
+) -> tuple[list[Book], list[Book]]:
     """Get Books with limited concurrency.
 
     - Creates semaphore to limit concurrent requests.
@@ -70,9 +65,16 @@ async def _get_books(config, retailer: Retailer, books: list[Book]) -> list[Book
     Returns:
         List of Book objects with updated pricing and availability
     """
-    semaphore = asyncio.Semaphore(10)
+
+    echo_info(f"Getting deals from {retailer.name}")
+    books = _get_retailer_relevant_tbr_books(
+        retailer,
+        books,
+    )
+
+    semaphore = asyncio.Semaphore(retailer.max_concurrency)
     response = []
-    unresolved_books = []
+    unknown_books = []
     books = [copy.deepcopy(book) for book in books]
     for book in books:
         book.retailer = retailer.name
@@ -81,19 +83,26 @@ async def _get_books(config, retailer: Retailer, books: list[Book]) -> list[Book
     tasks = [
         retailer.get_book(book, semaphore)
         for book in books
+        if book.deal_id not in ignored_deal_ids
     ]
-    results = await tqdm_asyncio.gather(*tasks, desc=f"Getting latest prices from {retailer.name}")
+
+    results = await asyncio.gather(*tasks)
     for book in results:
-        if book.exists:
+        if not book:
+            """Cases where we know the retailer has the book but it's not coming back.
+            We don't want to mark it as unknown it's more like we just got rate limited.
+
+            Kindle has been particularly bad about this. 
+            """
+            continue
+        elif book.exists:
             response.append(book)
         elif not book.exists:
-            unresolved_books.append(book)
+            unknown_books.append(book)
 
-    click.echo()
-    for book in unresolved_books:
-        echo_info(f"{book.title} by {book.authors} not found")
+    echo_info(f"Finished getting deals from {retailer.name}")
 
-    return response
+    return response, unknown_books
 
 
 def _apply_proper_list_prices(books: list[Book]):
@@ -151,7 +160,7 @@ def _get_retailer_relevant_tbr_books(
     return response
 
 
-async def get_latest_deals(config: Config):
+async def _get_latest_deals(config: Config):
     """
     Fetches the latest book deals from all tracked retailers for the user's TBR list.
 
@@ -169,28 +178,51 @@ async def get_latest_deals(config: Config):
     """
 
     books: list[Book] = []
+    unknown_books: list[Book] = []
     tbr_books = await get_tbr_books(config)
+    ignore_books: list[Book] = get_unknown_books(config)
+    ignored_deal_ids: set[str] = {book.deal_id for book in ignore_books}
 
+    tasks = []
     for retailer_str in config.tracked_retailers:
         retailer = RETAILER_MAP[retailer_str]()
         await retailer.set_auth()
 
-        relevant_tbr_books = _get_retailer_relevant_tbr_books(
-            retailer,
-            tbr_books,
+        tasks.append(
+            _get_books(
+                config,
+                retailer,
+                tbr_books,
+                ignored_deal_ids
+            )
         )
 
-        echo_info(f"Getting deals from {retailer.name}")
-        click.echo("\n---------------")
-        books.extend(await _get_books(config, retailer, relevant_tbr_books))
-        click.echo("---------------\n")
+    results = await asyncio.gather(*tasks)
+    for retailer_books, u_books in results:
+        books.extend(retailer_books)
+        unknown_books.extend(u_books)
 
     _apply_proper_list_prices(books)
-
-    books = [
-        book
-        for book in books
-        if book.current_price <= config.max_price and book.discount() >= config.min_discount
-    ]
-
     update_retailer_deal_table(config, books)
+    set_unknown_books(config, unknown_books)
+
+
+async def get_latest_deals(config: Config) -> bool:
+    try:
+        await _get_latest_deals(config)
+    except Exception as e:
+        ran_successfully = False
+        details = f"Error getting deals: {e}"
+        echo_err(details)
+    else:
+        ran_successfully = True
+        details = ""
+
+    # Save execution results
+    db_conn = get_duckdb_conn()
+    db_conn.execute(
+        "INSERT INTO latest_deal_run_history (timepoint, ran_successfully, details) VALUES (?, ?, ?)",
+        [config.run_time, ran_successfully, details]
+    )
+
+    return ran_successfully