avito-sdk 0.1.0__py3-none-any.whl

avito_sdk/__init__.py

@@ -0,0 +1,77 @@
+"""
+avito-sdk: High-performance Avito scraping & data extraction SDK for Python.
+
+Includes enhancements from Duff89/parser_avito:
+- PR #334: Real-time price change tracking and seller name extraction
+- PR #337: Deep parameters and characteristics parsing («О помещении», авто, etc.)
+- PR #329 / Issue #305: Full description parsing from microdata & mobile API
+- Asynchronous and Synchronous clients
+- Headless execution (Zero GUI / Zero Tkinter dependencies)
+- Universal runtime support (Python 3.8-3.16, Free-Threaded No-GIL, PyPy)
+"""
+
+from __future__ import annotations
+
+from avito_sdk.async_client import AsyncAvitoClient
+from avito_sdk.client import AvitoClient
+from avito_sdk.export import (
+    to_csv,
+    to_dataframe,
+    to_excel,
+    to_json,
+    to_jsonl,
+)
+from avito_sdk.extractors import (
+    extract_catalog_items,
+    extract_description,
+    extract_params,
+    extract_seller_id,
+    extract_seller_name,
+    extract_views,
+    parse_raw_item,
+)
+from avito_sdk.models import (
+    Item,
+    PriceRecord,
+    SearchFilter,
+    SearchPage,
+)
+from avito_sdk.tracker import PriceTracker
+from avito_sdk.url import (
+    build_item_api_url,
+    build_item_web_url,
+    build_page_url,
+    build_search_url,
+    normalize_region,
+)
+
+__version__ = "0.1.0"
+__author__ = "eminsk"
+
+__all__ = [
+    "__version__",
+    "AvitoClient",
+    "AsyncAvitoClient",
+    "Item",
+    "SearchFilter",
+    "SearchPage",
+    "PriceRecord",
+    "PriceTracker",
+    "extract_params",
+    "extract_seller_name",
+    "extract_seller_id",
+    "extract_description",
+    "extract_views",
+    "parse_raw_item",
+    "extract_catalog_items",
+    "build_search_url",
+    "build_item_api_url",
+    "build_item_web_url",
+    "build_page_url",
+    "normalize_region",
+    "to_excel",
+    "to_json",
+    "to_jsonl",
+    "to_csv",
+    "to_dataframe",
+]

avito_sdk/async_client.py

@@ -0,0 +1,213 @@
+"""
+Asynchronous high-level client for Avito (AsyncAvitoClient).
+Designed for modern asyncio backends (FastAPI, aiohttp, Aiogram, Celery).
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import AsyncGenerator, List, Optional, Union
+
+from bs4 import BeautifulSoup
+
+from avito_sdk.extractors import (
+    extract_catalog_items,
+    extract_description,
+    extract_embedded_state,
+    extract_params,
+    extract_seller_id,
+    extract_seller_name,
+    extract_views,
+)
+from avito_sdk.http import AsyncHttpTransport
+from avito_sdk.models import Item, SearchFilter
+from avito_sdk.tracker import PriceTracker
+from avito_sdk.url import (
+    build_item_web_url,
+    build_page_url,
+    build_search_url,
+)
+
+logger = logging.getLogger("avito_sdk")
+
+
+class AsyncAvitoClient:
+    """
+    High-level asynchronous Avito SDK client.
+
+    Example:
+        >>> async with AsyncAvitoClient() as client:
+        ...     async for item in client.search("rtx 4070", limit=10):
+        ...         print(item.title, item.price)
+    """
+
+    def __init__(
+        self,
+        proxy: Optional[str] = None,
+        timeout: int = 25,
+        max_retries: int = 3,
+        tracker_db: Optional[Union[str, Path]] = "avito_prices.db",
+        enable_tracking: bool = True,
+    ):
+        self.transport = AsyncHttpTransport(
+            proxy=proxy,
+            timeout=timeout,
+            max_retries=max_retries,
+        )
+        self.tracker = PriceTracker(db_path=tracker_db) if (enable_tracking and tracker_db) else None
+
+    async def search(
+        self,
+        query: str = "",
+        region: Optional[str] = "rossiya",
+        min_price: Optional[int] = None,
+        max_price: Optional[int] = None,
+        sort: str = "date",
+        category: Optional[str] = None,
+        with_delivery: bool = False,
+        enrich_details: bool = False,
+        limit: Optional[int] = None,
+        max_pages: int = 5,
+    ) -> AsyncGenerator[Item, None]:
+        """
+        Asynchronously search Avito and stream items.
+        """
+        search_filter = SearchFilter(
+            query=query,
+            region=region,
+            min_price=min_price,
+            max_price=max_price,
+            sort=sort,
+            category=category,
+            with_delivery=with_delivery,
+            page=1,
+        )
+
+        yielded_count = 0
+        for page_num in range(1, max_pages + 1):
+            search_filter.page = page_num
+            url = build_search_url(search_filter)
+
+            items = await self.scrape_page_items(url)
+            if not items:
+                break
+
+            for item in items:
+                if self.tracker:
+                    self.tracker.check_and_update(item)
+
+                if enrich_details:
+                    await self.enrich_item(item)
+
+                yield item
+                yielded_count += 1
+                if limit and yielded_count >= limit:
+                    return
+
+    async def scrape_page_items(self, url: str) -> List[Item]:
+        """Fetch and extract items from a single page URL asynchronously."""
+        html_text = await self.transport.fetch_html(url)
+        soup = BeautifulSoup(html_text, "html.parser")
+        embedded = extract_embedded_state(soup)
+
+        if embedded:
+            items = extract_catalog_items(embedded)
+            if items:
+                return items
+
+        # Fallback to HTML elements
+        items: List[Item] = []
+        for el in soup.select('[data-marker="item"]'):
+            item_id_str = el.get("data-item-id") or el.get("id")
+            if not item_id_str:
+                continue
+            digits = "".join(filter(str.isdigit, str(item_id_str)))
+            if not digits:
+                continue
+            item_id = int(digits)
+
+            title_el = el.select_one('[data-marker="item-title"]') or el.select_one("h3")
+            title = title_el.get_text(strip=True) if title_el else ""
+
+            price_el = el.select_one('[data-marker="item-price"]') or el.select_one('[itemprop="price"]')
+            price = 0
+            if price_el:
+                price_digits = "".join(filter(str.isdigit, price_el.get_text()))
+                price = int(price_digits) if price_digits else 0
+
+            url_el = el.select_one('a[data-marker="item-title"]') or el.select_one('a[itemprop="url"]')
+            link = url_el.get("href") if url_el else ""
+            if link and link.startswith("/"):
+                link = f"https://www.avito.ru{link}"
+
+            seller_name = extract_seller_name({}, html_text=str(el))
+
+            items.append(
+                Item(
+                    id=item_id,
+                    title=title,
+                    price=price,
+                    url=link,
+                    seller_name=seller_name,
+                )
+            )
+
+        return items
+
+    async def get_item(self, item_id: int) -> Item:
+        """Fetch full details for an item by ID asynchronously."""
+        item = Item(id=item_id, url=build_item_web_url(item_id))
+        await self.enrich_item(item)
+        return item
+
+    async def enrich_item(self, item: Item) -> Item:
+        """Enrich an item with card parameters, seller, description, and views asynchronously."""
+        try:
+            payload = await self.transport.fetch_item_card(item.id)
+            if payload:
+                params = extract_params(payload)
+                if params:
+                    item.params = params
+
+                if not item.seller_name:
+                    item.seller_name = extract_seller_name(payload)
+                if not item.seller_id:
+                    item.seller_id = extract_seller_id(payload)
+
+                desc = extract_description(payload)
+                if desc:
+                    item.description = desc
+
+                total_views, today_views = extract_views(payload)
+                if total_views is not None:
+                    item.total_views = total_views
+                if today_views is not None:
+                    item.today_views = today_views
+        except Exception as err:
+            logger.debug(f"Async API card fetch failed for item {item.id}: {err}")
+            try:
+                web_url = item.url or build_item_web_url(item.id)
+                html_text = await self.transport.fetch_html(web_url)
+                if not item.seller_name:
+                    item.seller_name = extract_seller_name({}, html_text=html_text)
+                if not item.description:
+                    item.description = extract_description({}, html_text=html_text)
+                total_views, today_views = extract_views({}, html_text=html_text)
+                if total_views is not None:
+                    item.total_views = total_views
+                if today_views is not None:
+                    item.today_views = today_views
+            except Exception as html_err:
+                logger.warning(f"Could not enrich item {item.id} asynchronously: {html_err}")
+
+        return item
+
+    async def close(self) -> None:
+        await self.transport.close()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()

avito_sdk/cli.py

@@ -0,0 +1,123 @@
+"""
+Command-line interface (CLI) for avito-sdk.
+Run queries, inspect items, monitor prices, and export datasets directly from terminal.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from avito_sdk.client import AvitoClient
+from avito_sdk.export import to_csv, to_excel, to_json
+from avito_sdk.tracker import PriceTracker
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="avito-sdk",
+        description="High-performance Avito Scraper & SDK with price tracking and parameters extraction",
+    )
+    subparsers = parser.add_subparsers(dest="command", help="Available subcommands")
+
+    # Search command
+    search_p = subparsers.add_parser("search", help="Search Avito listings")
+    search_p.add_argument("query", help="Search query (e.g. 'ноутбук lenovo')")
+    search_p.add_argument("-r", "--region", default="rossiya", help="Region slug or city (default: rossiya)")
+    search_p.add_argument("--min-price", type=int, default=None, help="Minimum price filter in RUB")
+    search_p.add_argument("--max-price", type=int, default=None, help="Maximum price filter in RUB")
+    search_p.add_argument("-n", "--limit", type=int, default=20, help="Maximum number of items to fetch")
+    search_p.add_argument("-p", "--pages", type=int, default=3, help="Max search result pages to scan")
+    search_p.add_argument("-o", "--output", help="Save output to file (.json, .csv, .xlsx)")
+    search_p.add_argument("--enrich", action="store_true", help="Fetch detailed parameters and full descriptions")
+
+    # Item command
+    item_p = subparsers.add_parser("item", help="Fetch detailed info for a single item")
+    item_p.add_argument("item_id", type=int, help="Avito numeric item ID")
+    item_p.add_argument("-o", "--output", help="Save output to JSON file")
+
+    # Drops command
+    drops_p = subparsers.add_parser("drops", help="Show all tracked items whose price has dropped")
+    drops_p.add_argument("--db", default="avito_prices.db", help="Path to SQLite price database")
+
+    args = parser.parse_args(argv)
+
+    if not args.command:
+        parser.print_help()
+        return 0
+
+    if args.command == "search":
+        print(f"[*] Searching for '{args.query}' in '{args.region}' (limit={args.limit})...")
+        client = AvitoClient()
+        items = list(
+            client.search(
+                query=args.query,
+                region=args.region,
+                min_price=args.min_price,
+                max_price=args.max_price,
+                enrich_details=args.enrich,
+                limit=args.limit,
+                max_pages=args.pages,
+            )
+        )
+        print(f"[+] Found {len(items)} items:")
+        for idx, item in enumerate(items, 1):
+            price_str = f"{item.price:,} ₽"
+            if item.old_price:
+                price_str += f" (было: {item.old_price:,} ₽, снижение на {item.price_drop:,} ₽)"
+            seller_str = f" | {item.seller_name}" if item.seller_name else ""
+            print(f" {idx:2d}. [{item.id}] {item.title[:45]} - {price_str}{seller_str}")
+
+        if args.output:
+            out_path = Path(args.output)
+            suffix = out_path.suffix.lower()
+            if suffix == ".xlsx":
+                to_excel(items, out_path)
+            elif suffix == ".csv":
+                to_csv(items, out_path)
+            else:
+                to_json(items, out_path)
+            print(f"[✓] Saved {len(items)} items to {out_path}")
+
+    elif args.command == "item":
+        print(f"[*] Fetching item {args.item_id}...")
+        client = AvitoClient()
+        item = client.get_item(args.item_id)
+        print(f"\nID: {item.id}")
+        print(f"Title: {item.title}")
+        print(f"Price: {item.price:,} ₽")
+        if item.seller_name:
+            print(f"Seller: {item.seller_name} (ID: {item.seller_id})")
+        if item.total_views is not None:
+            print(f"Views: {item.total_views} (today: {item.today_views})")
+        if item.params:
+            print("Parameters:")
+            for k, v in item.params.items():
+                print(f"  - {k}: {v}")
+        if item.description:
+            print(f"\nDescription:\n{item.description[:300]}...")
+
+        if args.output:
+            to_json([item], args.output)
+            print(f"[✓] Saved to {args.output}")
+
+    elif args.command == "drops":
+        tracker = PriceTracker(db_path=args.db)
+        drops = tracker.get_price_drops()
+        if not drops:
+            print("[-] No price drops recorded yet in database.")
+        else:
+            print(f"[+] Found {len(drops)} items with price drops:")
+            for d in drops:
+                diff = d['initial_price'] - d['current_price']
+                print(
+                    f" - [{d['item_id']}] {d.get('title', '')[:40]}: "
+                    f"{d['initial_price']:,} -> {d['current_price']:,} ₽ (-{diff:,} ₽)"
+                )
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

avito_sdk/client.py

@@ -0,0 +1,264 @@
+"""
+Synchronous high-level client for Avito (AvitoClient).
+Provides intuitive methods for search, item enrichment, price tracking, and export.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Generator, Iterable, List, Optional, Union
+
+from bs4 import BeautifulSoup
+
+from avito_sdk.export import to_csv, to_dataframe, to_excel, to_json, to_jsonl
+from avito_sdk.extractors import (
+    extract_catalog_items,
+    extract_description,
+    extract_embedded_state,
+    extract_params,
+    extract_seller_id,
+    extract_seller_name,
+    extract_views,
+    parse_raw_item,
+)
+from avito_sdk.http import SyncHttpTransport
+from avito_sdk.models import Item, SearchFilter, SearchPage
+from avito_sdk.tracker import PriceTracker
+from avito_sdk.url import (
+    build_item_api_url,
+    build_item_web_url,
+    build_page_url,
+    build_search_url,
+)
+
+logger = logging.getLogger("avito_sdk")
+
+
+class AvitoClient:
+    """
+    High-level synchronous Avito SDK client.
+
+    Example:
+        >>> from avito_sdk import AvitoClient
+        >>> client = AvitoClient()
+        >>> for item in client.search("ThinkPad", region="moskva", limit=10):
+        ...     print(item.title, item.price, item.seller_name)
+    """
+
+    def __init__(
+        self,
+        proxy: Optional[str] = None,
+        timeout: int = 25,
+        max_retries: int = 3,
+        tracker_db: Optional[Union[str, Path]] = "avito_prices.db",
+        enable_tracking: bool = True,
+    ):
+        self.transport = SyncHttpTransport(
+            proxy=proxy,
+            timeout=timeout,
+            max_retries=max_retries,
+        )
+        self.tracker = PriceTracker(db_path=tracker_db) if (enable_tracking and tracker_db) else None
+
+    def search(
+        self,
+        query: str = "",
+        region: Optional[str] = "rossiya",
+        min_price: Optional[int] = None,
+        max_price: Optional[int] = None,
+        sort: str = "date",
+        category: Optional[str] = None,
+        with_delivery: bool = False,
+        enrich_details: bool = False,
+        limit: Optional[int] = None,
+        max_pages: int = 5,
+    ) -> Generator[Item, None, None]:
+        """
+        Search Avito for items matching specified criteria.
+        Yields Item objects one by one.
+        """
+        search_filter = SearchFilter(
+            query=query,
+            region=region,
+            min_price=min_price,
+            max_price=max_price,
+            sort=sort,
+            category=category,
+            with_delivery=with_delivery,
+            page=1,
+        )
+
+        yielded_count = 0
+        for page_num in range(1, max_pages + 1):
+            search_filter.page = page_num
+            url = build_search_url(search_filter)
+            logger.debug(f"Fetching search page {page_num}: {url}")
+
+            items = self.scrape_page_items(url)
+            if not items:
+                break
+
+            for item in items:
+                if self.tracker:
+                    self.tracker.check_and_update(item)
+
+                if enrich_details:
+                    self.enrich_item(item)
+
+                yield item
+                yielded_count += 1
+                if limit and yielded_count >= limit:
+                    return
+
+    def scrape_url(
+        self,
+        url: str,
+        max_pages: int = 1,
+        enrich_details: bool = False,
+    ) -> List[Item]:
+        """Scrape items from an existing Avito search or catalog URL across multiple pages."""
+        results: List[Item] = []
+        for page in range(1, max_pages + 1):
+            page_url = build_page_url(url, page)
+            page_items = self.scrape_page_items(page_url)
+            if not page_items:
+                break
+
+            for item in page_items:
+                if self.tracker:
+                    self.tracker.check_and_update(item)
+                if enrich_details:
+                    self.enrich_item(item)
+                results.append(item)
+
+        return results
+
+    def scrape_page_items(self, url: str) -> List[Item]:
+        """Fetch and extract items from a single page URL."""
+        html_text = self.transport.fetch_html(url)
+        soup = BeautifulSoup(html_text, "html.parser")
+        embedded = extract_embedded_state(soup)
+
+        if embedded:
+            items = extract_catalog_items(embedded)
+            if items:
+                return items
+
+        # Fallback to HTML elements
+        items: List[Item] = []
+        for el in soup.select('[data-marker="item"]'):
+            item_id_str = el.get("data-item-id") or el.get("id")
+            if not item_id_str:
+                continue
+            digits = "".join(filter(str.isdigit, str(item_id_str)))
+            if not digits:
+                continue
+            item_id = int(digits)
+
+            title_el = el.select_one('[data-marker="item-title"]') or el.select_one("h3")
+            title = title_el.get_text(strip=True) if title_el else ""
+
+            price_el = el.select_one('[data-marker="item-price"]') or el.select_one('[itemprop="price"]')
+            price = 0
+            if price_el:
+                price_digits = "".join(filter(str.isdigit, price_el.get_text()))
+                price = int(price_digits) if price_digits else 0
+
+            url_el = el.select_one('a[data-marker="item-title"]') or el.select_one('a[itemprop="url"]')
+            link = url_el.get("href") if url_el else ""
+            if link and link.startswith("/"):
+                link = f"https://www.avito.ru{link}"
+
+            seller_name = extract_seller_name({}, html_text=str(el))
+
+            items.append(
+                Item(
+                    id=item_id,
+                    title=title,
+                    price=price,
+                    url=link,
+                    seller_name=seller_name,
+                )
+            )
+
+        return items
+
+    def get_item(self, item_id: int) -> Item:
+        """
+        Fetch full details for an item by ID, including:
+        - Full description (Issue #305)
+        - Parameters and specs (PR #337)
+        - Seller name and ID (PR #334)
+        - View counts (total and today)
+        """
+        item = Item(id=item_id, url=build_item_web_url(item_id))
+        self.enrich_item(item)
+        return item
+
+    def enrich_item(self, item: Item) -> Item:
+        """Enrich an existing Item instance with deep card parameters and views."""
+        try:
+            payload = self.transport.fetch_item_card(item.id)
+            if payload:
+                # Parameters (PR #337)
+                params = extract_params(payload)
+                if params:
+                    item.params = params
+
+                # Seller info (PR #334)
+                if not item.seller_name:
+                    item.seller_name = extract_seller_name(payload)
+                if not item.seller_id:
+                    item.seller_id = extract_seller_id(payload)
+
+                # Description and views
+                desc = extract_description(payload)
+                if desc:
+                    item.description = desc
+
+                total_views, today_views = extract_views(payload)
+                if total_views is not None:
+                    item.total_views = total_views
+                if today_views is not None:
+                    item.today_views = today_views
+        except Exception as err:
+            logger.debug(f"API card fetch failed for item {item.id}, attempting HTML fallback: {err}")
+            try:
+                web_url = item.url or build_item_web_url(item.id)
+                html_text = self.transport.fetch_html(web_url)
+                if not item.seller_name:
+                    item.seller_name = extract_seller_name({}, html_text=html_text)
+                if not item.description:
+                    item.description = extract_description({}, html_text=html_text)
+                total_views, today_views = extract_views({}, html_text=html_text)
+                if total_views is not None:
+                    item.total_views = total_views
+                if today_views is not None:
+                    item.today_views = today_views
+            except Exception as html_err:
+                logger.warning(f"Could not enrich item {item.id}: {html_err}")
+
+        return item
+
+    # Export helper shortcuts
+    def export_excel(self, items: List[Item], filepath: Union[str, Path]) -> None:
+        to_excel(items, filepath)
+
+    def export_json(self, items: List[Item], filepath: Union[str, Path], indent: int = 2) -> None:
+        to_json(items, filepath, indent=indent)
+
+    def export_csv(self, items: List[Item], filepath: Union[str, Path]) -> None:
+        to_csv(items, filepath)
+
+    def export_dataframe(self, items: List[Item]) -> Any:
+        return to_dataframe(items)
+
+    def close(self) -> None:
+        self.transport.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()