thordata-sdk 1.4.0__py3-none-any.whl → 1.6.0__py3-none-any.whl

thordata/retry.py

@@ -186,7 +186,7 @@ def with_retry(
                     if isinstance(e, ThordataRateLimitError) and e.retry_after:
                         delay = max(delay, e.retry_after)
 
-                    logger.warning(
+                    logger.info(
                         f"Retry attempt {attempt + 1}/{config.max_retries} "
                         f"after {delay:.2f}s due to: {e}"
                     )

thordata/tools/__init__.py

@@ -0,0 +1,38 @@
+"""
+Web Scraper Tool Registry.
+High-level abstractions for specific scraping targets.
+"""
+
+from .base import ToolRequest, VideoToolRequest
+from .code import GitHub
+from .ecommerce import Amazon, Walmart, eBay
+from .professional import Crunchbase, Glassdoor, Indeed
+from .search import GoogleMaps, GooglePlay, GoogleShopping
+from .social import Facebook, Instagram, LinkedIn, Reddit, TikTok, Twitter
+from .travel import Airbnb, Booking, Zillow
+from .video import YouTube
+
+__all__ = [
+    "ToolRequest",
+    "VideoToolRequest",
+    "Amazon",
+    "eBay",
+    "Walmart",
+    "GoogleMaps",
+    "GoogleShopping",
+    "GooglePlay",
+    "TikTok",
+    "Facebook",
+    "Instagram",
+    "Twitter",
+    "LinkedIn",
+    "Reddit",
+    "YouTube",
+    "GitHub",
+    "Indeed",
+    "Glassdoor",
+    "Crunchbase",
+    "Booking",
+    "Zillow",
+    "Airbnb",
+]

thordata/tools/base.py

@@ -0,0 +1,42 @@
+"""
+Base classes for Web Scraper Tools.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any, ClassVar
+
+
+@dataclass
+class ToolRequest:
+    """Base class for standard scraping tools."""
+
+    # These must be defined in subclasses
+    SPIDER_ID: ClassVar[str]
+    SPIDER_NAME: ClassVar[str]
+
+    def to_task_parameters(self) -> dict[str, Any]:
+        """Convert dataclass fields to API parameters dict."""
+        # Filter out internal fields and None values
+        return {
+            k: v
+            for k, v in asdict(self).items()
+            if v is not None and k != "common_settings"
+        }
+
+    def get_spider_id(self) -> str:
+        return self.SPIDER_ID
+
+    def get_spider_name(self) -> str:
+        return self.SPIDER_NAME
+
+
+@dataclass
+class VideoToolRequest(ToolRequest):
+    """
+    Marker class for Video/Audio tools that use the /video_builder endpoint.
+    Concrete classes must define a 'common_settings' field.
+    """
+
+    pass

thordata/tools/code.py

@@ -0,0 +1,39 @@
+"""
+Code Repository Scraper Tools (GitHub, etc.)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .base import ToolRequest
+
+
+class GitHub:
+    """Namespace for GitHub tools."""
+
+    @dataclass
+    class Repository(ToolRequest):
+        """Github Repository Scraper by Repo URL"""
+
+        SPIDER_ID = "github_repository_by-repo-url"
+        SPIDER_NAME = "github.com"
+        repo_url: str
+
+    @dataclass
+    class RepositoryBySearchUrl(ToolRequest):
+        """Github Repository Scraper by Search URL"""
+
+        SPIDER_ID = "github_repository_by-search-url"
+        SPIDER_NAME = "github.com"
+        search_url: str
+        page_turning: int | None = None
+        max_num: int | None = None
+
+    @dataclass
+    class RepositoryByUrl(ToolRequest):
+        """Github Repository Scraper by URL"""
+
+        SPIDER_ID = "github_repository_by-url"
+        SPIDER_NAME = "github.com"
+        url: str

thordata/tools/ecommerce.py

@@ -0,0 +1,251 @@
+"""
+E-Commerce Scraper Tools (Amazon, etc.)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .base import ToolRequest
+
+
+class Amazon:
+    """Namespace for Amazon tools."""
+
+    # --- Product Details (5 methods) ---
+    @dataclass
+    class ProductByAsin(ToolRequest):
+        """Amazon Product Details Scraper by ASIN."""
+
+        SPIDER_ID = "amazon_product_by-asin"
+        SPIDER_NAME = "amazon.com"
+
+        asin: str
+        domain: str = "amazon.com"
+
+    # Backward compatible alias
+    Product = ProductByAsin
+
+    @dataclass
+    class ProductByUrl(ToolRequest):
+        """Amazon Product Details Scraper by URL."""
+
+        SPIDER_ID = "amazon_product_by-url"
+        SPIDER_NAME = "amazon.com"
+
+        url: str
+        zip_code: str | None = None
+
+    @dataclass
+    class ProductByKeywords(ToolRequest):
+        """Amazon Product Details Scraper by Keywords."""
+
+        SPIDER_ID = "amazon_product_by-keywords"
+        SPIDER_NAME = "amazon.com"
+
+        keyword: str
+        page_turning: int | None = None
+        lowest_price: float | None = None
+        highest_price: float | None = None
+
+    @dataclass
+    class ProductByCategoryUrl(ToolRequest):
+        """Amazon Product Details Scraper by Category URL."""
+
+        SPIDER_ID = "amazon_product_by-category-url"
+        SPIDER_NAME = "amazon.com"
+
+        url: str
+        sort_by: str | None = None
+        page_turning: int | None = None
+
+    @dataclass
+    class ProductByBestSellers(ToolRequest):
+        """Amazon Product Details Scraper by Best Sellers URL."""
+
+        SPIDER_ID = "amazon_product_by-best-sellers"
+        SPIDER_NAME = "amazon.com"
+
+        url: str
+        page_turning: int | None = None
+
+    # --- Other Amazon Tools ---
+
+    @dataclass
+    class GlobalProductByUrl(ToolRequest):
+        """Amazon Global Product Details Scraper by URL"""
+
+        SPIDER_ID = "amazon_global-product_by-url"
+        SPIDER_NAME = "amazon.com"
+
+        url: str
+
+    # Backward compatible alias
+    GlobalProduct = GlobalProductByUrl
+
+    @dataclass
+    class GlobalProductByCategoryUrl(ToolRequest):
+        """Amazon Global Product Details Scraper by Category URL"""
+
+        SPIDER_ID = "amazon_global-product_by-category-url"
+        SPIDER_NAME = "amazon.com"
+
+        url: str
+        sort_by: str | None = None
+        get_sponsored: str | None = None
+        maximum: int | None = None
+
+    @dataclass
+    class GlobalProductBySellerUrl(ToolRequest):
+        """Amazon Global Product Details Scraper by Seller URL"""
+
+        SPIDER_ID = "amazon_global-product_by-seller-url"
+        SPIDER_NAME = "amazon.com"
+
+        url: str
+        maximum: int | None = None
+
+    @dataclass
+    class GlobalProductByKeywords(ToolRequest):
+        """Amazon Global Product Details Scraper by Keywords"""
+
+        SPIDER_ID = "amazon_global-product_by-keywords"
+        SPIDER_NAME = "amazon.com"
+
+        keyword: str
+        domain: str = "https://www.amazon.com"
+        lowest_price: str | None = None
+        highest_price: str | None = None
+        page_turning: int | None = None
+
+    @dataclass
+    class GlobalProductByKeywordsBrand(ToolRequest):
+        """Amazon Global Product Details Scraper by Keywords and Brand"""
+
+        SPIDER_ID = "amazon_global-product_by-keywords-brand"
+        SPIDER_NAME = "amazon.com"
+
+        keyword: str
+        brands: str
+        page_turning: int | None = None
+
+    @dataclass
+    class Review(ToolRequest):
+        """Amazon Product Review Scraper"""
+
+        SPIDER_ID = "amazon_comment_by-url"
+        SPIDER_NAME = "amazon.com"
+
+        url: str
+        page_turning: int = 1
+
+    @dataclass
+    class Seller(ToolRequest):
+        """Amazon Seller Information Scraper"""
+
+        SPIDER_ID = "amazon_seller_by-url"
+        SPIDER_NAME = "amazon.com"
+
+        url: str
+
+    @dataclass
+    class Search(ToolRequest):
+        """Amazon Product Listing Scraper"""
+
+        SPIDER_ID = "amazon_product-list_by-keywords-domain"
+        SPIDER_NAME = "amazon.com"
+
+        keyword: str
+        domain: str = "https://www.amazon.com/"
+        page_turning: int = 1
+
+
+class eBay:
+    """Namespace for eBay tools."""
+
+    @dataclass
+    class ProductByUrl(ToolRequest):
+        """eBay Information Scraper by URL"""
+
+        SPIDER_ID = "ebay_ebay_by-url"
+        SPIDER_NAME = "ebay.com"
+        url: str
+
+    @dataclass
+    class ProductByCategoryUrl(ToolRequest):
+        """eBay Information Scraper by Category URL"""
+
+        SPIDER_ID = "ebay_ebay_by-category-url"
+        SPIDER_NAME = "ebay.com"
+        url: str
+        count: str | None = None
+
+    @dataclass
+    class ProductByKeywords(ToolRequest):
+        """eBay Information Scraper by Keywords"""
+
+        SPIDER_ID = "ebay_ebay_by-keywords"
+        SPIDER_NAME = "ebay.com"
+        keywords: str
+        count: str | None = None
+
+    @dataclass
+    class ProductByListUrl(ToolRequest):
+        """eBay Information Scraper by List URL"""
+
+        SPIDER_ID = "ebay_ebay_by-listurl"
+        SPIDER_NAME = "ebay.com"
+        url: str
+        count: str | None = None
+
+
+class Walmart:
+    """Namespace for Walmart tools."""
+
+    @dataclass
+    class ProductByUrl(ToolRequest):
+        """Walmart Product Information Scraper by URL"""
+
+        SPIDER_ID = "walmart_product_by-url"
+        SPIDER_NAME = "walmart.com"
+        url: str
+        all_variations: str | None = None
+
+    @dataclass
+    class ProductByCategoryUrl(ToolRequest):
+        """Walmart Product Information Scraper by Category URL"""
+
+        SPIDER_ID = "walmart_product_by-category-url"
+        SPIDER_NAME = "walmart.com"
+        category_url: str
+        all_variations: str | None = None
+        page_turning: int | None = None
+
+    @dataclass
+    class ProductBySku(ToolRequest):
+        """Walmart Product Information Scraper by SKU"""
+
+        SPIDER_ID = "walmart_product_by-sku"
+        SPIDER_NAME = "walmart.com"
+        sku: str
+        all_variations: str | None = None
+
+    @dataclass
+    class ProductByKeywords(ToolRequest):
+        """Walmart Product Information Scraper by Keywords"""
+
+        SPIDER_ID = "walmart_product_by-keywords"
+        SPIDER_NAME = "walmart.com"
+        keyword: str
+        domain: str = "https://www.walmart.com/"
+        all_variations: str | None = None
+        page_turning: int | None = None
+
+    @dataclass
+    class ProductByZipcodes(ToolRequest):
+        """Walmart Product Information Scraper by Zipcodes"""
+
+        SPIDER_ID = "walmart_product_by-zipcodes"
+        SPIDER_NAME = "walmart.com"
+        url: str
+        zip_code: str | None = None

thordata/tools/professional.py

@@ -0,0 +1,155 @@
+"""
+Professional Platform Scraper Tools (Indeed, Glassdoor, Crunchbase, etc.)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .base import ToolRequest
+
+
+class Indeed:
+    """Namespace for Indeed tools."""
+
+    @dataclass
+    class JobByUrl(ToolRequest):
+        """Indeed Job Listings Scraper by Job URL"""
+
+        SPIDER_ID = "indeed_job-listings_by-job-url"
+        SPIDER_NAME = "indeed.com"
+        job_url: str
+
+    @dataclass
+    class JobByKeyword(ToolRequest):
+        """Indeed Job Listings Scraper by Keyword"""
+
+        SPIDER_ID = "indeed_job-listings_by-keyword"
+        SPIDER_NAME = "indeed.com"
+        keyword: str
+        location: str
+        country: str | None = None
+        domain: str | None = None
+        date_posted: str | None = None
+        posted_by: str | None = None
+        pay: str | None = None
+        location_radius: str | None = None
+
+    @dataclass
+    class CompanyByListUrl(ToolRequest):
+        """Indeed Companies Info Scraper by Company List URL"""
+
+        SPIDER_ID = "indeed_companies-info_by-company-list-url"
+        SPIDER_NAME = "indeed.com"
+        company_list_url: str
+
+    @dataclass
+    class CompanyByKeyword(ToolRequest):
+        """Indeed Companies Info Scraper by Keyword"""
+
+        SPIDER_ID = "indeed_companies-info_by-keyword"
+        SPIDER_NAME = "indeed.com"
+        keyword: str
+
+    @dataclass
+    class CompanyByIndustryAndState(ToolRequest):
+        """Indeed Companies Info Scraper by Industry and State"""
+
+        SPIDER_ID = "indeed_companies-info_by-industry-and-state"
+        SPIDER_NAME = "indeed.com"
+        industry: str
+        state: str | None = None
+
+    @dataclass
+    class CompanyByUrl(ToolRequest):
+        """Indeed Companies Info Scraper by Company URL"""
+
+        SPIDER_ID = "indeed_companies-info_by-company-url"
+        SPIDER_NAME = "indeed.com"
+        company_url: str
+
+
+class Glassdoor:
+    """Namespace for Glassdoor tools."""
+
+    @dataclass
+    class CompanyByUrl(ToolRequest):
+        """Glassdoor Company Overview Information Scraper by URL"""
+
+        SPIDER_ID = "glassdoor_company_by-url"
+        SPIDER_NAME = "glassdoor.com"
+        url: str
+
+    @dataclass
+    class CompanyByInputFilter(ToolRequest):
+        """Glassdoor Company Overview Information Scraper by Input Filter"""
+
+        SPIDER_ID = "glassdoor_company_by-inputfilter"
+        SPIDER_NAME = "glassdoor.com"
+        company_name: str
+        location: str | None = None
+        industries: str | None = None
+        Job_title: str | None = None  # Note: capital J in API
+
+    @dataclass
+    class CompanyByKeywords(ToolRequest):
+        """Glassdoor Company Overview Information Scraper by Keywords"""
+
+        SPIDER_ID = "glassdoor_company_by-keywords"
+        SPIDER_NAME = "glassdoor.com"
+        search_url: str
+        max_search_results: int | None = None
+
+    @dataclass
+    class CompanyByListUrl(ToolRequest):
+        """Glassdoor Company Overview Information Scraper by List URL"""
+
+        SPIDER_ID = "glassdoor_company_by-listurl"
+        SPIDER_NAME = "glassdoor.com"
+        url: str
+
+    @dataclass
+    class JobByUrl(ToolRequest):
+        """Glassdoor Job Information Scraper by URL"""
+
+        SPIDER_ID = "glassdoor_joblistings_by-url"
+        SPIDER_NAME = "glassdoor.com"
+        url: str
+
+    @dataclass
+    class JobByKeywords(ToolRequest):
+        """Glassdoor Job Information Scraper by Keywords"""
+
+        SPIDER_ID = "glassdoor_joblistings_by-keywords"
+        SPIDER_NAME = "glassdoor.com"
+        keyword: str
+        location: str
+        country: str | None = None
+
+    @dataclass
+    class JobByListUrl(ToolRequest):
+        """Glassdoor Job Information Scraper by List URL"""
+
+        SPIDER_ID = "glassdoor_joblistings_by-listurl"
+        SPIDER_NAME = "glassdoor.com"
+        url: str
+
+
+class Crunchbase:
+    """Namespace for Crunchbase tools."""
+
+    @dataclass
+    class CompanyByUrl(ToolRequest):
+        """Crunchbase Company Information Scraper by URL"""
+
+        SPIDER_ID = "crunchbase_company_by-url"
+        SPIDER_NAME = "crunchbase.com"
+        url: str
+
+    @dataclass
+    class CompanyByKeywords(ToolRequest):
+        """Crunchbase Company Information Scraper by Keywords"""
+
+        SPIDER_ID = "crunchbase_company_by-keywords"
+        SPIDER_NAME = "crunchbase.com"
+        keyword: str

thordata/tools/search.py

@@ -0,0 +1,115 @@
+"""
+Search Engine & Map Scraper Tools (Google, etc.)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .base import ToolRequest
+
+
+class GoogleMaps:
+    """Namespace for Google Maps tools."""
+
+    @dataclass
+    class DetailsByUrl(ToolRequest):
+        """Google Maps Details Scraper by URL."""
+
+        SPIDER_ID = "google_map-details_by-url"
+        SPIDER_NAME = "google.com"
+
+        url: str
+
+    @dataclass
+    class DetailsByCid(ToolRequest):
+        """Google Maps Details Scraper by CID."""
+
+        SPIDER_ID = "google_map-details_by-cid"
+        SPIDER_NAME = "google.com"
+
+        CID: str
+
+    @dataclass
+    class DetailsByLocation(ToolRequest):
+        """Google Maps Details Scraper by Location keyword + country (+ optional lat/long/zoom)."""  # noqa: E501
+
+        SPIDER_ID = "google_map-details_by-location"
+        SPIDER_NAME = "google.com"
+
+        country: str
+        keyword: str
+        lat: str | None = None
+        long: str | None = None
+        zoom_level: str | None = None
+
+    @dataclass
+    class DetailsByPlaceId(ToolRequest):
+        """Google Maps Details Scraper by Place ID."""
+
+        SPIDER_ID = "google_map-details_by-placeid"
+        SPIDER_NAME = "google.com"
+
+        place_id: str
+
+    # Backward compatible alias: keep old name working
+    Details = DetailsByUrl
+
+    @dataclass
+    class Reviews(ToolRequest):
+        """Google Maps Review Information Scraper"""
+
+        SPIDER_ID = "google_comment_by-url"
+        SPIDER_NAME = "google.com"
+
+        url: str
+        days_limit: int | None = None  # Crawl reviews within X days
+
+
+class GoogleShopping:
+    """Namespace for Google Shopping tools."""
+
+    @dataclass
+    class Product(ToolRequest):
+        """Google Shopping Information Scraper by URL"""
+
+        SPIDER_ID = "google_shopping_by-url"
+        SPIDER_NAME = "google.com"
+        url: str
+        country: str | None = None  # e.g. "US"
+
+    @dataclass
+    class ProductByKeywords(ToolRequest):
+        """Google Shopping Information Scraper by Keywords"""
+
+        SPIDER_ID = "google_shopping_by-keywords"
+        SPIDER_NAME = "google.com"
+        keyword: str
+        country: str | None = None  # e.g. "US"
+
+
+class GooglePlay:
+    """Namespace for Google Play Store tools."""
+
+    @dataclass
+    class AppInfo(ToolRequest):
+        """Google Play Store Information Scraper"""
+
+        SPIDER_ID = "google-play-store_information_by-url"
+        SPIDER_NAME = "google.com"
+
+        app_url: str
+        country: str | None = None
+
+    @dataclass
+    class Reviews(ToolRequest):
+        """Google Play Store Reviews Scraper"""
+
+        SPIDER_ID = "google-play-store_reviews_by-url"
+        SPIDER_NAME = "google.com"
+
+        app_url: str
+        num_of_reviews: int | None = None
+        start_date: str | None = None  # yyyy-mm-dd
+        end_date: str | None = None
+        country: str | None = None