fmslist 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl

fmslist/scraper.py

@@ -0,0 +1,151 @@
+import json
+import re
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Mapping
+
+import requests
+
+
+@dataclass
+class Variant:
+    """A class to hold details of a variant of an item."""
+
+    id: int
+    name: str
+    price: str
+    available: bool
+    quantity: int
+
+
+@dataclass(frozen=True)
+class ItemDetails:
+    """A class to hold details of an item."""
+
+    id: int
+    title: str
+    vendor: str
+    image_urls: list[str]
+    link: str
+    published_at: datetime
+    updated_at: datetime
+    variants: list[Variant]
+
+
+class FindMeStoreList:
+    """A class to scrape the Find Me Store (FMS) list from the specified URL."""
+
+    items: list[ItemDetails] = []
+
+    def __init__(self):
+        self._session = requests.Session()
+        self._session.headers.update({"User-Agent": "MJ12bot"})
+        self._base_url = "https://findmestore.thinkr.jp"
+
+    def fetch_items(self) -> None:
+        """Fetches all items from the FMS list."""
+        page = 1
+        while True:
+            try:
+                # Fetch the first page of products
+                items = self._fetch_products(page)
+                self.items.extend(items)
+                if not items:
+                    break  # No products found, exit the loop
+                page += 1
+            except ValueError as e:
+                print(f"Error fetching products: {e}")
+                break  # Exit the loop on error
+
+        # Sort items by publish time
+        self.items.sort(key=lambda item: item.published_at, reverse=True)
+
+    def fill_quantities(self) -> None:
+        """Fills the quantities for each variant in the items."""
+        quantities: Mapping[int, int] = {}
+        page = 1
+        while True:
+            try:
+                # Fetch quantities from the search API
+                q = self._fetch_quantities(page)
+                quantities.update(q)
+                if not q:
+                    break
+                page += 1
+            except ValueError as e:
+                print(f"Error fetching quantities: {e}")
+                break
+
+        for item in self.items:
+            for variant in item.variants:
+                variant.quantity = max(quantities.get(variant.id, 0), -1)
+
+    def _fetch_quantities(self, page: int) -> Mapping[int, int]:
+        """Fetches the quantities from search API. Returns a mapping of variant IDs to quantities."""
+        while True:
+            res = self._session.get(
+                f"{self._base_url}/search?view=preorderjson&q=*&page={page}"
+            )
+            if res.status_code == 200:
+                break
+            elif res.status_code == 429:
+                print(f"Rate limit exceeded, waiting 5s before retrying page {page}...")
+                time.sleep(5)
+            else:
+                raise ValueError(
+                    f"Failed to fetch search result at page {page}: [{res.status_code}] {res.text}"
+                )
+        # A hacky fix for the API returning an empty "id" field
+        json_fixed = re.sub(r":\s*(,|\})", f": null\\1", res.text)
+        return {
+            variant["id"]: variant["inventory_quantity"]
+            for product in json.loads(json_fixed)
+            for variant in product.get("variants", [])
+            if variant["available"]
+        }
+
+    def _parse_timestamp(self, timestamp: str) -> datetime:
+        """Parses a timestamp string into a datetime object."""
+        return datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S%z")
+
+    def _parse_product(self, product: dict) -> ItemDetails:
+        """Parses a product dictionary into an ItemDetails object."""
+        id = product["id"]
+        title = product["title"]
+        vendor = product.get("vendor", "Unknown Vendor")
+        image_urls = [image["src"] for image in product.get("images", [])]
+        link = f"{self._base_url}/products/{product['handle']}"
+        published_at = self._parse_timestamp(product["published_at"])
+        updated_at = self._parse_timestamp(product["updated_at"])
+        variants = [
+            Variant(
+                id=variant["id"],
+                name=variant["title"] if variant["title"] != "Default Title" else "",
+                price=variant["price"],
+                available=variant["available"],
+                quantity=0,
+            )
+            for variant in product.get("variants", [])
+        ]
+        return ItemDetails(
+            id, title, vendor, image_urls, link, published_at, updated_at, variants
+        )
+
+    def _fetch_products(self, page: int) -> list[ItemDetails]:
+        """Fetches the products from the FMS list."""
+        while True:
+            res = self._session.get(
+                f"{self._base_url}/products.json?limit=250&page={page}"
+            )
+            if res.status_code == 200:
+                break
+            elif res.status_code == 429:
+                print(f"Rate limit exceeded, waiting 5s before retrying page {page}...")
+                time.sleep(5)
+            else:
+                raise ValueError(
+                    f"Failed to fetch products at page {page}: [{res.status_code}] {res.text}"
+                )
+        products = res.json().get("products", [])
+        return [self._parse_product(product) for product in products]

{fmslist-0.0.1.dist-info → fmslist-0.0.2.dist-info}/METADATA

@@ -1,12 +1,14 @@
 Metadata-Version: 2.4
 Name: fmslist
-Version: 0.0.1
+Version: 0.0.2
 Summary: A web scraper that lists items from Findmestore.
 Project-URL: Homepage, https://github.com/d4n1elchen/fmslist
 Project-URL: Issues, https://github.com/d4n1elchen/fmslist/issues
 License-Expression: MIT
 License-File: LICENSE
 Requires-Python: >=3.9
+Requires-Dist: bs4
+Requires-Dist: requests
 Description-Content-Type: text/markdown
 
 # fmslist

fmslist-0.0.2.dist-info/RECORD

@@ -0,0 +1,6 @@
+fmslist/__init__.py,sha256=LvCLbFusopU82bfaLvjmc_TnJJereamdRlo0diCSOlc,23
+fmslist/scraper.py,sha256=0M3DgfGI9NpaJb7GsWWvLsL2bU-SIyUe_SMs7eAxYQc,5257
+fmslist-0.0.2.dist-info/METADATA,sha256=5rvdDVw2USlkhi5jLVENdGCITrea1a72a2WPjksM2ig,413
+fmslist-0.0.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+fmslist-0.0.2.dist-info/licenses/LICENSE,sha256=ACwmltkrXIz5VsEQcrqljq-fat6ZXAMepjXGoe40KtE,1069
+fmslist-0.0.2.dist-info/RECORD,,

fmslist-0.0.1.dist-info/RECORD

@@ -1,6 +0,0 @@
-fmslist/__init__.py,sha256=LvCLbFusopU82bfaLvjmc_TnJJereamdRlo0diCSOlc,23
-fmslist/scraper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fmslist-0.0.1.dist-info/METADATA,sha256=6a0CJ6Y2UVotIePJcV5d3GhxfdJqfykpL0DMo8-eDSg,370
-fmslist-0.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-fmslist-0.0.1.dist-info/licenses/LICENSE,sha256=ACwmltkrXIz5VsEQcrqljq-fat6ZXAMepjXGoe40KtE,1069
-fmslist-0.0.1.dist-info/RECORD,,