thordata-sdk 1.4.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

thordata/tools/__init__.py

@@ -0,0 +1,28 @@
+"""
+Web Scraper Tool Registry.
+High-level abstractions for specific scraping targets.
+"""
+
+from .base import ToolRequest, VideoToolRequest
+from .code import GitHub
+from .ecommerce import Amazon
+from .search import GoogleMaps, GooglePlay, GoogleShopping
+from .social import Facebook, Instagram, LinkedIn, Reddit, TikTok, Twitter
+from .video import YouTube
+
+__all__ = [
+    "ToolRequest",
+    "VideoToolRequest",
+    "Amazon",
+    "GoogleMaps",
+    "GoogleShopping",
+    "GooglePlay",
+    "TikTok",
+    "Facebook",
+    "Instagram",
+    "Twitter",
+    "LinkedIn",
+    "Reddit",
+    "YouTube",
+    "GitHub",
+]

thordata/tools/base.py

@@ -0,0 +1,42 @@
+"""
+Base classes for Web Scraper Tools.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any, ClassVar
+
+
+@dataclass
+class ToolRequest:
+    """Base class for standard scraping tools."""
+
+    # These must be defined in subclasses
+    SPIDER_ID: ClassVar[str]
+    SPIDER_NAME: ClassVar[str]
+
+    def to_task_parameters(self) -> dict[str, Any]:
+        """Convert dataclass fields to API parameters dict."""
+        # Filter out internal fields and None values
+        return {
+            k: v
+            for k, v in asdict(self).items()
+            if v is not None and k != "common_settings"
+        }
+
+    def get_spider_id(self) -> str:
+        return self.SPIDER_ID
+
+    def get_spider_name(self) -> str:
+        return self.SPIDER_NAME
+
+
+@dataclass
+class VideoToolRequest(ToolRequest):
+    """
+    Marker class for Video/Audio tools that use the /video_builder endpoint.
+    Concrete classes must define a 'common_settings' field.
+    """
+
+    pass

thordata/tools/code.py

@@ -0,0 +1,26 @@
+"""
+Code Repository Scraper Tools (GitHub, etc.)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .base import ToolRequest
+
+
+class GitHub:
+    """Namespace for GitHub tools."""
+
+    @dataclass
+    class Repository(ToolRequest):
+        """Github Repository Scraper"""
+
+        SPIDER_ID = "github_repository_by-repo-url"
+        SPIDER_NAME = "github.com"
+
+        repo_url: str
+        search_url: str | None = None
+        url: str | None = None  # The generic URL param
+        page_turning: int | None = None
+        max_num: int | None = None

thordata/tools/ecommerce.py

@@ -0,0 +1,67 @@
+"""
+E-Commerce Scraper Tools (Amazon, etc.)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .base import ToolRequest
+
+
+class Amazon:
+    """Namespace for Amazon tools."""
+
+    @dataclass
+    class Product(ToolRequest):
+        """Amazon Product Details Scraper"""
+
+        SPIDER_ID = "amazon_product_by-asin"
+        SPIDER_NAME = "amazon.com"
+
+        asin: str
+        domain: str = "amazon.com"
+
+    @dataclass
+    class GlobalProduct(ToolRequest):
+        """Amazon Global Product Details Scraper"""
+
+        SPIDER_ID = "amazon_global-product_by-url"
+        SPIDER_NAME = "amazon.com"
+
+        url: str
+        zip_code: str | None = None
+
+    @dataclass
+    class Review(ToolRequest):
+        """Amazon Product Review Scraper"""
+
+        SPIDER_ID = "amazon_comment_by-url"
+        SPIDER_NAME = "amazon.com"
+
+        url: str
+        page_turning: int = 1
+
+    @dataclass
+    class Seller(ToolRequest):
+        """Amazon Seller Information Scraper"""
+
+        SPIDER_ID = "amazon_seller_by-url"
+        SPIDER_NAME = "amazon.com"
+
+        url: str
+
+    @dataclass
+    class Search(ToolRequest):
+        """Amazon Product Listing Scraper"""
+
+        SPIDER_ID = "amazon_product-list_by-keywords-domain"
+        SPIDER_NAME = "amazon.com"
+
+        keyword: str
+        domain: str = "amazon.com"
+        page_turning: int = 1
+        sort_by: str | None = None  # Best Sellers, Newest Arrivals, etc.
+        min_price: float | None = None
+        max_price: float | None = None
+        get_sponsored: bool | None = None

thordata/tools/search.py

@@ -0,0 +1,73 @@
+"""
+Search Engine & Map Scraper Tools (Google, etc.)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .base import ToolRequest
+
+
+class GoogleMaps:
+    """Namespace for Google Maps tools."""
+
+    @dataclass
+    class Details(ToolRequest):
+        """Google Maps Details Information Scraper"""
+
+        SPIDER_ID = "google_map-details_by-url"
+        SPIDER_NAME = "google.com"
+
+        url: str  # Google Maps URL
+
+    @dataclass
+    class Reviews(ToolRequest):
+        """Google Maps Review Information Scraper"""
+
+        SPIDER_ID = "google_comment_by-url"
+        SPIDER_NAME = "google.com"
+
+        url: str
+        days_limit: int | None = None  # Crawl reviews within X days
+
+
+class GoogleShopping:
+    """Namespace for Google Shopping tools."""
+
+    @dataclass
+    class Product(ToolRequest):
+        """Google Shopping Information Scraper"""
+
+        SPIDER_ID = "google_shopping_by-url"
+        SPIDER_NAME = "google.com"
+
+        url: str
+        country: str | None = None  # e.g. "US"
+
+
+class GooglePlay:
+    """Namespace for Google Play Store tools."""
+
+    @dataclass
+    class AppInfo(ToolRequest):
+        """Google Play Store Information Scraper"""
+
+        SPIDER_ID = "google-play-store_information_by-url"
+        SPIDER_NAME = "google.com"
+
+        app_url: str
+        country: str | None = None
+
+    @dataclass
+    class Reviews(ToolRequest):
+        """Google Play Store Reviews Scraper"""
+
+        SPIDER_ID = "google-play-store_reviews_by-url"
+        SPIDER_NAME = "google.com"
+
+        app_url: str
+        num_of_reviews: int | None = None
+        start_date: str | None = None  # yyyy-mm-dd
+        end_date: str | None = None
+        country: str | None = None

thordata/tools/social.py

@@ -0,0 +1,190 @@
+"""
+Social Media Scraper Tools.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .base import ToolRequest
+
+
+class TikTok:
+    @dataclass
+    class Post(ToolRequest):
+        """TikTok Post Information Scraper"""
+
+        SPIDER_ID = "tiktok_posts_by-url"
+        SPIDER_NAME = "tiktok.com"
+        url: str
+        page_turning: int | None = None
+
+    @dataclass
+    class Comment(ToolRequest):
+        """TikTok Comment Scraper"""
+
+        SPIDER_ID = "tiktok_comment_by-url"
+        SPIDER_NAME = "tiktok.com"
+        url: str
+        page_turning: int | None = None
+
+    @dataclass
+    class Profile(ToolRequest):
+        """TikTok Profile Information Scraper"""
+
+        SPIDER_ID = "tiktok_profiles_by-url"
+        SPIDER_NAME = "tiktok.com"
+
+        url: str  # Profile URL (e.g. https://www.tiktok.com/@user)
+        search_url: str | None = None
+
+        country: str | None = None
+        page_turning: int | None = None
+
+    @dataclass
+    class Shop(ToolRequest):
+        """TikTok Shop Information Scraper"""
+
+        SPIDER_ID = "tiktok_shop_by-url"
+        SPIDER_NAME = "tiktok.com"
+        url: str
+        category_url: str | None = None
+        keyword: str | None = None
+        page_turning: int | None = None
+
+
+class Facebook:
+    @dataclass
+    class Posts(ToolRequest):
+        """Facebook Posts Scraper"""
+
+        SPIDER_ID = "facebook_post_by-keywords"
+        SPIDER_NAME = "facebook.com"
+        keyword: str
+        recent_posts: bool | None = None
+        date: str | None = None  # Year 2025 etc.
+        number: int = 10
+
+    @dataclass
+    class PostDetails(ToolRequest):
+        """Facebook Post Details Scraper"""
+
+        SPIDER_ID = "facebook_post_by-posts-url"
+        SPIDER_NAME = "facebook.com"
+        url: str
+
+
+class Instagram:
+    @dataclass
+    class Profile(ToolRequest):
+        """Instagram Profile Scraper"""
+
+        SPIDER_ID = "ins_profiles_by-username"
+        SPIDER_NAME = "instagram.com"
+        username: str
+        profileurl: str | None = None
+
+    @dataclass
+    class Post(ToolRequest):
+        """Instagram Post Information Scraper"""
+
+        SPIDER_ID = "ins_posts_by-profileurl"
+        SPIDER_NAME = "instagram.com"
+        profileurl: str
+        resultsLimit: int = 10
+        start_date: str | None = None
+        end_date: str | None = None
+        post_type: str | None = None  # Post or Reel
+
+    @dataclass
+    class Reel(ToolRequest):
+        """Instagram Reel Information Scraper"""
+
+        SPIDER_ID = "ins_reel_by-url"
+        SPIDER_NAME = "instagram.com"
+        url: str
+        num_of_posts: int | None = None
+
+    @dataclass
+    class Comment(ToolRequest):
+        """Instagram Post Comment Scraper"""
+
+        SPIDER_ID = "ins_comment_by-posturl"
+        SPIDER_NAME = "instagram.com"
+        posturl: str
+
+
+class Twitter:
+    @dataclass
+    class Profile(ToolRequest):
+        """Twitter(X) Profile Scraper"""
+
+        SPIDER_ID = "twitter_profiles_by-url"
+        SPIDER_NAME = "twitter.com"
+        url: str
+        max_number_of_posts: int | None = None
+        user_name: str | None = None
+
+    @dataclass
+    class Post(ToolRequest):
+        """
+        Twitter(X) Post Information Scraper
+        Updates based on integration snippet:
+        - SPIDER_NAME is 'x.com'
+        - Only 'url' is required.
+        """
+
+        SPIDER_ID = "twitter_by-posturl_by-url"
+        SPIDER_NAME = "x.com"  # Updated from snippet
+
+        url: str  # Post URL (e.g. https://x.com/user/status/123)
+
+        start_date: str | None = None
+        end_date: str | None = None
+
+
+class LinkedIn:
+    @dataclass
+    class Company(ToolRequest):
+        """LinkedIn Company Information Scraper"""
+
+        SPIDER_ID = "linkedin_company_information_by-url"
+        SPIDER_NAME = "linkedin.com"
+        url: str
+
+    @dataclass
+    class Jobs(ToolRequest):
+        """LinkedIn Job Listing Scraper"""
+
+        SPIDER_ID = "linkedin_job_listings_information_by-job-listing-url"
+        SPIDER_NAME = "linkedin.com"
+        job_listing_url: str
+        location: str
+        job_url: str | None = None
+        page_turning: int | None = None
+        keyword: str | None = None
+        remote: str | None = None  # On_site, Remote, Hybrid
+
+
+class Reddit:
+    @dataclass
+    class Posts(ToolRequest):
+        """Reddit Post Information Scraper"""
+
+        SPIDER_ID = "reddit_posts_by-url"
+        SPIDER_NAME = "reddit.com"
+        url: str
+        keyword: str | None = None
+        subreddit_url: str | None = None
+        num_of_posts: int | None = None
+        sort_by: str | None = None  # Relevance, Hot, Top, New
+
+    @dataclass
+    class Comment(ToolRequest):
+        """Reddit Post Comment Scraper"""
+
+        SPIDER_ID = "reddit_comment_by-url"
+        SPIDER_NAME = "reddit.com"
+        url: str
+        days_back: int | None = None
+        load_all_replies: bool | None = None

thordata/tools/video.py

@@ -0,0 +1,81 @@
+"""
+Video & Audio Scraper Tools (YouTube, etc.)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from ..types.common import CommonSettings
+from .base import ToolRequest, VideoToolRequest
+
+
+class YouTube:
+    """Namespace for YouTube tools."""
+
+    @dataclass
+    class VideoDownload(VideoToolRequest):
+        """YouTube Video File Scraper (Download). Uses video_builder."""
+
+        SPIDER_ID = "youtube_video_by-url"
+        SPIDER_NAME = "youtube.com"
+
+        url: str  # Video URL
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class AudioDownload(VideoToolRequest):
+        """YouTube Audio File Scraper (Download). Uses video_builder."""
+
+        SPIDER_ID = "youtube_audio_by-url"
+        SPIDER_NAME = "youtube.com"
+
+        url: str
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class SubtitleDownload(VideoToolRequest):
+        """YouTube Subtitle File Scraper. Uses video_builder."""
+
+        SPIDER_ID = "youtube_transcript_by-id"
+        SPIDER_NAME = "youtube.com"
+
+        video_id: str
+        subtitles_type: str | None = None  # Auto generated / user uploaded
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class Profile(VideoToolRequest):
+        """YouTube Profile Scraper. Uses video_builder."""
+
+        SPIDER_ID = "youtube_profiles_by-keyword"
+        SPIDER_NAME = "youtube.com"
+
+        url: str  # Channel URL
+        page_turning: int = 1
+        keyword: str | None = None
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class Comments(VideoToolRequest):
+        """YouTube Comment Information Scraper. Uses video_builder."""
+
+        SPIDER_ID = "youtube_comment_by-id"
+        SPIDER_NAME = "youtube.com"
+
+        video_id: str
+        num_of_comments: int | None = None
+        sort_by: str | None = None  # Top comments / Newest first
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class VideoInfo(ToolRequest):
+        """YouTube Video Post Scraper (Metadata only). Standard builder."""
+
+        # Note: This one does NOT inherit from VideoToolRequest because it uses the standard builder
+        # and doesn't support common_settings in the same way.
+        SPIDER_ID = "youtube_video-post_by-url"
+        SPIDER_NAME = "youtube.com"
+
+        url: str  # Channel Video URL
+        num_of_posts: str | None = None

thordata/types/__init__.py

@@ -0,0 +1,77 @@
+"""
+Thordata Data Types and Models.
+"""
+
+from .common import (
+    CommonSettings,
+    Continent,
+    Country,
+    Device,
+    OutputFormat,
+    ThordataBaseConfig,
+    normalize_enum_value,
+)
+from .proxy import (
+    ProxyConfig,
+    ProxyHost,
+    ProxyPort,
+    ProxyProduct,
+    ProxyServer,
+    ProxyType,
+    ProxyUser,
+    ProxyUserList,
+    SessionType,
+    StaticISPProxy,
+    StickySession,
+)
+from .serp import (
+    BingSearchType,
+    Engine,
+    GoogleSearchType,
+    GoogleTbm,
+    SerpRequest,
+    TimeRange,
+)
+from .task import (
+    DataFormat,
+    ScraperTaskConfig,
+    TaskStatus,
+    TaskStatusResponse,
+    UsageStatistics,
+    VideoTaskConfig,
+)
+from .universal import UniversalScrapeRequest
+
+__all__ = [
+    "CommonSettings",
+    "Device",
+    "OutputFormat",
+    "ThordataBaseConfig",
+    "Continent",
+    "Country",
+    "normalize_enum_value",
+    "ProxyConfig",
+    "ProxyProduct",
+    "ProxyServer",
+    "ProxyType",
+    "ProxyUser",
+    "ProxyUserList",
+    "SessionType",
+    "StaticISPProxy",
+    "StickySession",
+    "ProxyHost",
+    "ProxyPort",
+    "BingSearchType",
+    "Engine",
+    "GoogleSearchType",
+    "GoogleTbm",
+    "SerpRequest",
+    "TimeRange",
+    "DataFormat",
+    "ScraperTaskConfig",
+    "TaskStatus",
+    "TaskStatusResponse",
+    "UsageStatistics",
+    "VideoTaskConfig",
+    "UniversalScrapeRequest",
+]