thordata-sdk 1.4.0__py3-none-any.whl → 1.6.0__py3-none-any.whl

thordata/tools/social.py

@@ -0,0 +1,374 @@
+"""
+Social Media Scraper Tools.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .base import ToolRequest
+
+
+class TikTok:
+    @dataclass
+    class Post(ToolRequest):
+        """TikTok Post Information Scraper by URL"""
+
+        SPIDER_ID = "tiktok_posts_by-url"
+        SPIDER_NAME = "tiktok.com"
+        url: str
+        country: str | None = None
+
+    @dataclass
+    class PostsByKeywords(ToolRequest):
+        """TikTok Post Information Scraper by Keywords"""
+
+        SPIDER_ID = "tiktok_posts_by-keywords"
+        SPIDER_NAME = "tiktok.com"
+        search_keyword: str
+        num_of_posts: int | None = None
+        posts_to_not_include: str | None = None
+        country: str | None = None
+
+    @dataclass
+    class PostsByProfileUrl(ToolRequest):
+        """TikTok Post Information Scraper by Profile URL"""
+
+        SPIDER_ID = "tiktok_posts_by-profileurl"
+        SPIDER_NAME = "tiktok.com"
+        url: str
+        start_date: str | None = None
+        end_date: str | None = None
+        num_of_posts: int | None = None
+        what_to_collect: str | None = None
+        post_type: str | None = None
+        posts_to_not_include: str | None = None
+        country: str | None = None
+
+    @dataclass
+    class PostsByListUrl(ToolRequest):
+        """TikTok Post Information Scraper by List URL"""
+
+        SPIDER_ID = "tiktok_posts_by-listurl"
+        SPIDER_NAME = "tiktok.com"
+        url: str
+        num_of_posts: int | None = None
+
+    @dataclass
+    class Comment(ToolRequest):
+        """TikTok Comment Scraper"""
+
+        SPIDER_ID = "tiktok_comment_by-url"
+        SPIDER_NAME = "tiktok.com"
+        url: str
+        page_turning: int | None = None
+
+    @dataclass
+    class Profile(ToolRequest):
+        """TikTok Profile Information Scraper by URL"""
+
+        SPIDER_ID = "tiktok_profiles_by-url"
+        SPIDER_NAME = "tiktok.com"
+        url: str  # Profile URL (e.g. https://www.tiktok.com/@user)
+        country: str | None = None
+
+    @dataclass
+    class ProfilesByListUrl(ToolRequest):
+        """TikTok Profile Information Scraper by List URL"""
+
+        SPIDER_ID = "tiktok_profiles_by-listurl"
+        SPIDER_NAME = "tiktok.com"
+        search_url: str
+        country: str | None = None
+        page_turning: int | None = None
+
+    @dataclass
+    class Shop(ToolRequest):
+        """TikTok Shop Information Scraper by URL"""
+
+        SPIDER_ID = "tiktok_shop_by-url"
+        SPIDER_NAME = "tiktok.com"
+        url: str
+
+    @dataclass
+    class ShopByCategoryUrl(ToolRequest):
+        """TikTok Shop Information Scraper by Category URL"""
+
+        SPIDER_ID = "tiktok_shop_by-category-url"
+        SPIDER_NAME = "tiktok.com"
+        category_url: str
+
+    @dataclass
+    class ShopByKeywords(ToolRequest):
+        """TikTok Shop Information Scraper by Keywords"""
+
+        SPIDER_ID = "tiktok_shop_by-keywords"
+        SPIDER_NAME = "tiktok.com"
+        keyword: str
+        domain: str = "https://www.tiktok.com/shop"
+        page_turning: int | None = None
+
+
+class Facebook:
+    @dataclass
+    class PostDetails(ToolRequest):
+        """Facebook Post Details Scraper"""
+
+        SPIDER_ID = "facebook_post_by-posts-url"
+        SPIDER_NAME = "facebook.com"
+        url: str
+
+    @dataclass
+    class Posts(ToolRequest):
+        """Facebook Posts Scraper by Keywords"""
+
+        SPIDER_ID = "facebook_post_by-keywords"
+        SPIDER_NAME = "facebook.com"
+        keyword: str
+        recent_posts: bool | None = None
+        date: str | None = None  # Year 2025 etc.
+        number: int = 10
+
+    @dataclass
+    class EventByEventListUrl(ToolRequest):
+        """Facebook Events Scraper by Event List URL"""
+
+        SPIDER_ID = "facebook_event_by-eventlist-url"
+        SPIDER_NAME = "facebook.com"
+        url: str
+        upcoming_events_only: str | None = None
+
+    @dataclass
+    class EventBySearchUrl(ToolRequest):
+        """Facebook Events Scraper by Search URL"""
+
+        SPIDER_ID = "facebook_event_by-search-url"
+        SPIDER_NAME = "facebook.com"
+        url: str
+
+    @dataclass
+    class EventByEventsUrl(ToolRequest):
+        """Facebook Events Scraper by Events URL"""
+
+        SPIDER_ID = "facebook_event_by-events-url"
+        SPIDER_NAME = "facebook.com"
+        url: str
+
+    @dataclass
+    class Profile(ToolRequest):
+        """Facebook Profile Scraper"""
+
+        SPIDER_ID = "facebook_profile_by-profiles-url"
+        SPIDER_NAME = "facebook.com"
+        url: str
+
+    @dataclass
+    class Comment(ToolRequest):
+        """Facebook Post Comments Scraper"""
+
+        SPIDER_ID = "facebook_comment_by-comments-url"
+        SPIDER_NAME = "facebook.com"
+        url: str
+        get_all_replies: str | None = None
+        limit_records: str | None = None
+        comments_sort: str | None = None  # All comments
+
+
+class Instagram:
+    @dataclass
+    class Profile(ToolRequest):
+        """Instagram Profile Scraper by Username"""
+
+        SPIDER_ID = "ins_profiles_by-username"
+        SPIDER_NAME = "instagram.com"
+        username: str
+
+    @dataclass
+    class ProfileByUrl(ToolRequest):
+        """Instagram Profile Scraper by Profile URL"""
+
+        SPIDER_ID = "ins_profiles_by-profileurl"
+        SPIDER_NAME = "instagram.com"
+        profileurl: str
+
+    @dataclass
+    class Post(ToolRequest):
+        """Instagram Post Information Scraper by Profile URL"""
+
+        SPIDER_ID = "ins_posts_by-profileurl"
+        SPIDER_NAME = "instagram.com"
+        profileurl: str
+        resultsLimit: int = 10
+        start_date: str | None = None
+        end_date: str | None = None
+        post_type: str | None = None  # Post or Reel
+
+    @dataclass
+    class PostByUrl(ToolRequest):
+        """Instagram Post Information Scraper by Post URL"""
+
+        SPIDER_ID = "ins_posts_by-posturl"
+        SPIDER_NAME = "instagram.com"
+        posturl: str
+
+    @dataclass
+    class Reel(ToolRequest):
+        """Instagram Reel Information Scraper by URL"""
+
+        SPIDER_ID = "ins_reel_by-url"
+        SPIDER_NAME = "instagram.com"
+        url: str
+
+    @dataclass
+    class AllReel(ToolRequest):
+        """Instagram All Reel Information Scraper by URL"""
+
+        SPIDER_ID = "ins_allreel_by-url"
+        SPIDER_NAME = "instagram.com"
+        url: str
+        num_of_posts: int | None = None
+        posts_to_not_include: str | None = None
+        start_date: str | None = None
+        end_date: str | None = None
+
+    @dataclass
+    class ReelByListUrl(ToolRequest):
+        """Instagram Reel Information Scraper by List URL"""
+
+        SPIDER_ID = "ins_reel_by-listurl"
+        SPIDER_NAME = "instagram.com"
+        url: str
+        num_of_posts: int | None = None
+        posts_to_not_include: str | None = None
+        start_date: str | None = None
+        end_date: str | None = None
+
+    @dataclass
+    class Comment(ToolRequest):
+        """Instagram Post Comment Scraper"""
+
+        SPIDER_ID = "ins_comment_by-posturl"
+        SPIDER_NAME = "instagram.com"
+        posturl: str
+
+
+class Twitter:
+    @dataclass
+    class Profile(ToolRequest):
+        """Twitter(X) Profile Scraper by Profile URL"""
+
+        SPIDER_ID = "twitter_profile_by-profileurl"
+        SPIDER_NAME = "x.com"
+        url: str
+
+    @dataclass
+    class ProfileByUsername(ToolRequest):
+        """Twitter(X) Profile Scraper by Username"""
+
+        SPIDER_ID = "twitter_profile_by-username"
+        SPIDER_NAME = "x.com"
+        user_name: str
+
+    @dataclass
+    class Post(ToolRequest):
+        """Twitter(X) Post Information Scraper by Post URL"""
+
+        SPIDER_ID = "twitter_post_by-posturl"
+        SPIDER_NAME = "x.com"
+        url: str  # Post URL (e.g. https://x.com/user/status/123)
+
+    @dataclass
+    class PostByProfileUrl(ToolRequest):
+        """Twitter(X) Post Information Scraper by Profile URL"""
+
+        SPIDER_ID = "twitter_post_by-profileurl"
+        SPIDER_NAME = "x.com"
+        url: str  # Profile URL
+
+
+class LinkedIn:
+    @dataclass
+    class Company(ToolRequest):
+        """LinkedIn Company Information Scraper"""
+
+        SPIDER_ID = "linkedin_company_information_by-url"
+        SPIDER_NAME = "linkedin.com"
+        url: str
+
+    @dataclass
+    class Jobs(ToolRequest):
+        """LinkedIn Job Listing Scraper by Job Listing URL"""
+
+        SPIDER_ID = "linkedin_job_listings_information_by-job-listing-url"
+        SPIDER_NAME = "linkedin.com"
+        job_listing_url: str
+        page_turning: int | None = None
+
+    @dataclass
+    class JobByUrl(ToolRequest):
+        """LinkedIn Job Listing Scraper by Job URL"""
+
+        SPIDER_ID = "linkedin_job_listings_information_by-job-url"
+        SPIDER_NAME = "linkedin.com"
+        job_url: str
+
+    @dataclass
+    class JobByKeyword(ToolRequest):
+        """LinkedIn Job Listing Scraper by Keyword"""
+
+        SPIDER_ID = "linkedin_job_listings_information_by-keyword"
+        SPIDER_NAME = "linkedin.com"
+        location: str
+        keyword: str
+        time_range: str | None = None
+        experience_level: str | None = None
+        job_type: str | None = None
+        remote: str | None = None
+        company: str | None = None
+        selective_search: str | None = None
+        jobs_to_not_include: str | None = None
+        location_radius: str | None = None
+        page_turning: int | None = None
+
+
+class Reddit:
+    @dataclass
+    class Posts(ToolRequest):
+        """Reddit Post Information Scraper by URL"""
+
+        SPIDER_ID = "reddit_posts_by-url"
+        SPIDER_NAME = "reddit.com"
+        url: str
+
+    @dataclass
+    class PostsByKeywords(ToolRequest):
+        """Reddit Post Information Scraper by Keywords"""
+
+        SPIDER_ID = "reddit_posts_by-keywords"
+        SPIDER_NAME = "reddit.com"
+        keyword: str
+        date: str | None = None  # All time
+        num_of_posts: int | None = None
+        sort_by: str | None = None
+
+    @dataclass
+    class PostsBySubredditUrl(ToolRequest):
+        """Reddit Post Information Scraper by Subreddit URL"""
+
+        SPIDER_ID = "reddit_posts_by-subredditurl"
+        SPIDER_NAME = "reddit.com"
+        url: str
+        sort_by: str | None = None
+        num_of_posts: int | None = None
+        sort_by_time: str | None = None  # All Time
+
+    @dataclass
+    class Comment(ToolRequest):
+        """Reddit Post Comment Scraper"""
+
+        SPIDER_ID = "reddit_comment_by-url"
+        SPIDER_NAME = "reddit.com"
+        url: str
+        days_back: int | None = None
+        load_all_replies: str | None = None
+        comment_limit: int | None = None

thordata/tools/travel.py

@@ -0,0 +1,100 @@
+"""
+Travel & Real Estate Scraper Tools (Booking, Zillow, Airbnb)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .base import ToolRequest
+
+
+class Booking:
+    """Namespace for Booking.com tools."""
+
+    @dataclass
+    class HotelByUrl(ToolRequest):
+        """Booking Hotel Information Scraper by URL"""
+
+        SPIDER_ID = "booking_hotellist_by-url"
+        SPIDER_NAME = "booking.com"
+        url: str
+
+
+class Zillow:
+    """Namespace for Zillow tools."""
+
+    @dataclass
+    class PriceByUrl(ToolRequest):
+        """Zillow Property Price History Information Scraper by URL"""
+
+        SPIDER_ID = "zillow_price_by-url"
+        SPIDER_NAME = "zillow.com"
+        url: str
+
+    @dataclass
+    class ProductByUrl(ToolRequest):
+        """Zillow Property Details Information Scraper by URL"""
+
+        SPIDER_ID = "zillow_product_by-url"
+        SPIDER_NAME = "zillow.com"
+        url: str
+
+    @dataclass
+    class ProductByFilter(ToolRequest):
+        """Zillow Property Details Information Scraper by Filter"""
+
+        SPIDER_ID = "zillow_product_by-filter"
+        SPIDER_NAME = "zillow.com"
+        keywords_location: str
+        listingCategory: str | None = None  # For Rent, For Sale
+        HomeType: str | None = None  # Houses
+        days_on_zillow: str | None = None  # Any
+        maximum: int | None = None
+
+    @dataclass
+    class ProductByListUrl(ToolRequest):
+        """Zillow Property Details Information Scraper by List URL"""
+
+        SPIDER_ID = "zillow_product_by-listurl"
+        SPIDER_NAME = "zillow.com"
+        url: str
+        maximum: int | None = None
+
+
+class Airbnb:
+    """Namespace for Airbnb tools."""
+
+    @dataclass
+    class ProductBySearchUrl(ToolRequest):
+        """Airbnb Properties Information Scraper by Search URL"""
+
+        SPIDER_ID = "airbnb_product_by-searchurl"
+        SPIDER_NAME = "airbnb.com"
+        searchurl: str
+        country: str | None = None
+
+    @dataclass
+    class ProductByLocation(ToolRequest):
+        """Airbnb Properties Information Scraper by Location"""
+
+        SPIDER_ID = "airbnb_product_by-location"
+        SPIDER_NAME = "airbnb.com"
+        location: str
+        check_in: str | None = None
+        check_out: str | None = None
+        num_of_adults: str | None = None
+        num_of_children: str | None = None
+        num_of_infants: str | None = None
+        num_of_pets: str | None = None
+        country: str | None = None
+        currency: str | None = None
+
+    @dataclass
+    class ProductByUrl(ToolRequest):
+        """Airbnb Properties Information Scraper by URL"""
+
+        SPIDER_ID = "airbnb_product_by-url"
+        SPIDER_NAME = "airbnb.com"
+        url: str
+        country: str | None = None

thordata/tools/video.py

@@ -0,0 +1,154 @@
+"""
+Video & Audio Scraper Tools (YouTube, etc.)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from ..types.common import CommonSettings
+from .base import ToolRequest, VideoToolRequest
+
+
+class YouTube:
+    """Namespace for YouTube tools."""
+
+    @dataclass
+    class VideoDownload(VideoToolRequest):
+        """YouTube Video File Scraper (Download). Uses video_builder."""
+
+        SPIDER_ID = "youtube_video_by-url"
+        SPIDER_NAME = "youtube.com"
+
+        url: str  # Video URL
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class AudioDownload(VideoToolRequest):
+        """YouTube Audio File Scraper (Download). Uses video_builder."""
+
+        SPIDER_ID = "youtube_audio_by-url"
+        SPIDER_NAME = "youtube.com"
+
+        url: str
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class SubtitleDownload(VideoToolRequest):
+        """YouTube Subtitle File Scraper. Uses video_builder."""
+
+        SPIDER_ID = "youtube_transcript_by-id"
+        SPIDER_NAME = "youtube.com"
+
+        video_id: str
+        subtitles_type: str | None = None  # Auto generated / user uploaded
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class Profile(VideoToolRequest):
+        """YouTube Profile Scraper by Keyword. Uses video_builder."""
+
+        SPIDER_ID = "youtube_profiles_by-keyword"
+        SPIDER_NAME = "youtube.com"
+
+        keyword: str
+        page_turning: int = 1
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class ProfileByUrl(VideoToolRequest):
+        """YouTube Profile Scraper by URL. Uses video_builder."""
+
+        SPIDER_ID = "youtube_profiles_by-url"
+        SPIDER_NAME = "youtube.com"
+
+        url: str  # Channel URL
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class Comments(VideoToolRequest):
+        """YouTube Comment Information Scraper. Uses video_builder."""
+
+        SPIDER_ID = "youtube_comment_by-id"
+        SPIDER_NAME = "youtube.com"
+
+        video_id: str
+        num_of_comments: int | None = None
+        sort_by: str | None = None  # Top comments / Newest first
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class VideoInfo(VideoToolRequest):
+        """YouTube Video Basic Information Scraper. Uses video_builder."""
+
+        SPIDER_ID = "youtube_product_by-id"
+        SPIDER_NAME = "youtube.com"
+
+        video_id: str
+        common_settings: CommonSettings = field(default_factory=CommonSettings)
+
+    @dataclass
+    class VideoPostByUrl(ToolRequest):
+        """YouTube Video Post Scraper by URL. Uses standard builder."""
+
+        SPIDER_ID = "youtube_video-post_by-url"
+        SPIDER_NAME = "youtube.com"
+
+        url: str  # Channel Video URL
+        order_by: str | None = None
+        start_index: str | None = None
+        num_of_posts: str | None = None
+
+    @dataclass
+    class VideoPostBySearchFilters(ToolRequest):
+        """YouTube Video Post Scraper by Search Filters. Uses standard builder."""
+
+        SPIDER_ID = "youtube_video-post_by-search-filters"
+        SPIDER_NAME = "youtube.com"
+
+        keyword_search: str
+        features: str | None = None
+        type: str | None = None  # Videos
+        duration: str | None = None
+        upload_date: str | None = None
+        num_of_posts: str | None = None
+
+    @dataclass
+    class VideoPostByHashtag(ToolRequest):
+        """YouTube Video Post Scraper by Hashtag. Uses standard builder."""
+
+        SPIDER_ID = "youtube_video-post_by-hashtag"
+        SPIDER_NAME = "youtube.com"
+
+        hashtag: str
+        num_of_posts: str | None = None
+
+    @dataclass
+    class VideoPostByPodcastUrl(ToolRequest):
+        """YouTube Video Post Scraper by Podcast URL. Uses standard builder."""
+
+        SPIDER_ID = "youtube_video-post_by-podcast-url"
+        SPIDER_NAME = "youtube.com"
+
+        url: str  # Playlist URL
+        num_of_posts: str | None = None
+
+    @dataclass
+    class VideoPostByKeyword(ToolRequest):
+        """YouTube Video Post Scraper by Keyword. Uses standard builder."""
+
+        SPIDER_ID = "youtube_video-post_by-keyword"
+        SPIDER_NAME = "youtube.com"
+
+        keyword: str
+        num_of_posts: str | None = None
+
+    @dataclass
+    class VideoPostByExplore(ToolRequest):
+        """YouTube Video Post Scraper by Explore URL. Uses standard builder."""
+
+        SPIDER_ID = "youtube_video-post_by-explore"
+        SPIDER_NAME = "youtube.com"
+
+        url: str
+        all_tabs: str | None = None

thordata/types/__init__.py

@@ -0,0 +1,77 @@
+"""
+Thordata Data Types and Models.
+"""
+
+from .common import (
+    CommonSettings,
+    Continent,
+    Country,
+    Device,
+    OutputFormat,
+    ThordataBaseConfig,
+    normalize_enum_value,
+)
+from .proxy import (
+    ProxyConfig,
+    ProxyHost,
+    ProxyPort,
+    ProxyProduct,
+    ProxyServer,
+    ProxyType,
+    ProxyUser,
+    ProxyUserList,
+    SessionType,
+    StaticISPProxy,
+    StickySession,
+)
+from .serp import (
+    BingSearchType,
+    Engine,
+    GoogleSearchType,
+    GoogleTbm,
+    SerpRequest,
+    TimeRange,
+)
+from .task import (
+    DataFormat,
+    ScraperTaskConfig,
+    TaskStatus,
+    TaskStatusResponse,
+    UsageStatistics,
+    VideoTaskConfig,
+)
+from .universal import UniversalScrapeRequest
+
+__all__ = [
+    "CommonSettings",
+    "Device",
+    "OutputFormat",
+    "ThordataBaseConfig",
+    "Continent",
+    "Country",
+    "normalize_enum_value",
+    "ProxyConfig",
+    "ProxyProduct",
+    "ProxyServer",
+    "ProxyType",
+    "ProxyUser",
+    "ProxyUserList",
+    "SessionType",
+    "StaticISPProxy",
+    "StickySession",
+    "ProxyHost",
+    "ProxyPort",
+    "BingSearchType",
+    "Engine",
+    "GoogleSearchType",
+    "GoogleTbm",
+    "SerpRequest",
+    "TimeRange",
+    "DataFormat",
+    "ScraperTaskConfig",
+    "TaskStatus",
+    "TaskStatusResponse",
+    "UsageStatistics",
+    "VideoTaskConfig",
+    "UniversalScrapeRequest",
+]