cian-playwright 2.0.0__tar.gz

cian_playwright-2.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mikhail
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cian_playwright-2.0.0/PKG-INFO

@@ -0,0 +1,103 @@
+Metadata-Version: 2.4
+Name: cian-playwright
+Version: 2.0.0
+Summary: Parser for Cian.ru real estate listings (Playwright-based)
+Author-email: Mikhail <tarminik@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/tarminik/cian-parser
+Project-URL: Repository, https://github.com/tarminik/cian-parser
+Project-URL: Issues, https://github.com/tarminik/cian-parser/issues
+Keywords: cian,parser,real-estate,scraper,playwright
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: playwright>=1.40
+Requires-Dist: playwright-stealth>=2.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+# cian-parser
+
+Playwright-based parser for [Cian.ru](https://cian.ru) real estate listings.
+
+## Installation
+
+```bash
+pip install cian-parser
+playwright install chromium
+```
+
+## Quick start
+
+```python
+from cian_parser import CianParser
+
+with CianParser("Москва") as parser:
+    # Sale
+    flats = parser.get_flats(deal_type="sale", rooms=(1, 2))
+
+    # Long-term rent
+    rent = parser.get_flats(deal_type="rent_long", rooms="all")
+
+    # Short-term rent
+    daily = parser.get_flats(deal_type="rent_short")
+
+    # Suburban
+    houses = parser.get_suburban("house", deal_type="sale")
+
+    # New construction
+    newbuilds = parser.get_newobjects()
+```
+
+## API
+
+### `CianParser(location, proxies=None)`
+
+- **location** — city name (e.g. `"Москва"`). See `cian_parser.list_locations()` for all options.
+- **proxies** — optional proxy URL string for Playwright.
+
+### `get_flats(deal_type, rooms="all", with_saving_csv=False, with_extra_data=False, additional_settings=None)`
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `deal_type` | `str` | `"sale"`, `"rent_long"`, or `"rent_short"` |
+| `rooms` | `int \| str \| tuple` | `1`–`5`, `"studio"`, `"all"`, or tuple like `(1, 2, "studio")` |
+| `with_saving_csv` | `bool` | Save results to CSV |
+| `with_extra_data` | `bool` | Parse detail pages for extra fields (slower) |
+| `additional_settings` | `dict` | Filters: `min_price`, `max_price`, `metro`, `district`, `sort_by`, etc. |
+
+### `get_suburban(suburban_type, deal_type, ...)`
+
+Same parameters as `get_flats`, plus:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `suburban_type` | `str` | `"house"`, `"house-part"`, `"land-plot"`, or `"townhouse"` |
+
+### `get_newobjects(with_saving_csv=False)`
+
+Parse new construction listings.
+
+### Helpers
+
+```python
+from cian_parser import list_locations, list_metro_stations, list_districts
+
+list_locations()              # [("Москва", "1"), ("Санкт-Петербург", "2"), ...]
+list_metro_stations()         # {"Москва": [("Арбатская", "1"), ...], ...}
+list_districts("Москва")      # [("Арбат", "13"), ...]
+```
+
+## License
+
+[MIT](LICENSE)

cian_playwright-2.0.0/README.md

@@ -0,0 +1,75 @@
+# cian-parser
+
+Playwright-based parser for [Cian.ru](https://cian.ru) real estate listings.
+
+## Installation
+
+```bash
+pip install cian-parser
+playwright install chromium
+```
+
+## Quick start
+
+```python
+from cian_parser import CianParser
+
+with CianParser("Москва") as parser:
+    # Sale
+    flats = parser.get_flats(deal_type="sale", rooms=(1, 2))
+
+    # Long-term rent
+    rent = parser.get_flats(deal_type="rent_long", rooms="all")
+
+    # Short-term rent
+    daily = parser.get_flats(deal_type="rent_short")
+
+    # Suburban
+    houses = parser.get_suburban("house", deal_type="sale")
+
+    # New construction
+    newbuilds = parser.get_newobjects()
+```
+
+## API
+
+### `CianParser(location, proxies=None)`
+
+- **location** — city name (e.g. `"Москва"`). See `cian_parser.list_locations()` for all options.
+- **proxies** — optional proxy URL string for Playwright.
+
+### `get_flats(deal_type, rooms="all", with_saving_csv=False, with_extra_data=False, additional_settings=None)`
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `deal_type` | `str` | `"sale"`, `"rent_long"`, or `"rent_short"` |
+| `rooms` | `int \| str \| tuple` | `1`–`5`, `"studio"`, `"all"`, or tuple like `(1, 2, "studio")` |
+| `with_saving_csv` | `bool` | Save results to CSV |
+| `with_extra_data` | `bool` | Parse detail pages for extra fields (slower) |
+| `additional_settings` | `dict` | Filters: `min_price`, `max_price`, `metro`, `district`, `sort_by`, etc. |
+
+### `get_suburban(suburban_type, deal_type, ...)`
+
+Same parameters as `get_flats`, plus:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `suburban_type` | `str` | `"house"`, `"house-part"`, `"land-plot"`, or `"townhouse"` |
+
+### `get_newobjects(with_saving_csv=False)`
+
+Parse new construction listings.
+
+### Helpers
+
+```python
+from cian_parser import list_locations, list_metro_stations, list_districts
+
+list_locations()              # [("Москва", "1"), ("Санкт-Петербург", "2"), ...]
+list_metro_stations()         # {"Москва": [("Арбатская", "1"), ...], ...}
+list_districts("Москва")      # [("Арбат", "13"), ...]
+```
+
+## License
+
+[MIT](LICENSE)

cian_playwright-2.0.0/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "cian-playwright"
+version = "2.0.0"
+description = "Parser for Cian.ru real estate listings (Playwright-based)"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.9"
+authors = [
+    { name = "Mikhail", email = "tarminik@gmail.com" },
+]
+keywords = ["cian", "parser", "real-estate", "scraper", "playwright"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+]
+dependencies = [
+    "playwright>=1.40",
+    "playwright-stealth>=2.0",
+]
+
+[project.optional-dependencies]
+dev = ["pytest"]
+
+[project.urls]
+Homepage = "https://github.com/tarminik/cian-parser"
+Repository = "https://github.com/tarminik/cian-parser"
+Issues = "https://github.com/tarminik/cian-parser/issues"
+
+[tool.pytest.ini_options]
+markers = [
+    "live: end-to-end tests that hit real Cian.ru (require network + chromium)",
+    "slow: slow tests (newobject, extra_data with detail pages)",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]

cian_playwright-2.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

cian_playwright-2.0.0/src/cian_parser/__init__.py

@@ -0,0 +1,15 @@
+"""cian_parser — Playwright-based parser for Cian.ru real estate listings."""
+
+from cian_parser.parser import CianParser, list_locations, list_metro_stations, list_districts
+from cian_parser.exceptions import CianParserError, CaptchaError, LocationNotFoundError, BrowserNotInstalledError
+
+__all__ = [
+    "CianParser",
+    "list_locations",
+    "list_metro_stations",
+    "list_districts",
+    "CianParserError",
+    "CaptchaError",
+    "LocationNotFoundError",
+    "BrowserNotInstalledError",
+]

cian_playwright-2.0.0/src/cian_parser/base_list.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import logging
+import random
+import time
+from typing import Any
+
+from playwright.sync_api import Page
+
+from cian_parser.browser import BrowserManager
+from cian_parser.csv_saver import save_to_csv
+from cian_parser.constants import SPECIFIC_FIELDS_FOR_RENT_LONG, SPECIFIC_FIELDS_FOR_RENT_SHORT, SPECIFIC_FIELDS_FOR_SALE
+from cian_parser.exceptions import CaptchaError
+from cian_parser.helpers import define_deal_url_id
+
+logger = logging.getLogger("cian_parser")
+
+
+class BaseListParser:
+    def __init__(
+        self,
+        browser: BrowserManager,
+        accommodation_type: str,
+        deal_type: str,
+        rent_period_type: int | None,
+        location_name: str,
+        with_saving_csv: bool = False,
+        with_extra_data: bool = False,
+        additional_settings: dict[str, Any] | None = None,
+        object_type: str | None = None,
+    ):
+        self.browser = browser
+        self.accommodation_type = accommodation_type
+        self.deal_type = deal_type
+        self.rent_period_type = rent_period_type
+        self.location_name = location_name
+        self.with_saving_csv = with_saving_csv
+        self.with_extra_data = with_extra_data
+        self.additional_settings = additional_settings
+        self.object_type = object_type
+
+        self.result: list[dict[str, Any]] = []
+        self.result_set: set[str] = set()
+        self.count_parsed_offers = 0
+        self.start_page = 1 if (additional_settings is None or "start_page" not in additional_settings) else additional_settings["start_page"]
+        self.end_page = 100 if (additional_settings is None or "end_page" not in additional_settings) else additional_settings["end_page"]
+        self.file_path = self.build_file_path()
+
+    def is_sale(self) -> bool:
+        return self.deal_type == "sale"
+
+    def is_rent_long(self) -> bool:
+        return self.deal_type == "rent" and self.rent_period_type == 4
+
+    def is_rent_short(self) -> bool:
+        return self.deal_type == "rent" and self.rent_period_type == 2
+
+    def build_file_path(self) -> str:
+        raise NotImplementedError
+
+    def parse_card(self, card: Any) -> dict[str, Any]:
+        raise NotImplementedError
+
+    def parse_detail_page(self, url: str) -> dict[str, Any]:
+        raise NotImplementedError
+
+    def remove_unnecessary_fields(self, record: dict[str, Any]) -> dict[str, Any]:
+        if self.is_sale():
+            for field in SPECIFIC_FIELDS_FOR_RENT_LONG | SPECIFIC_FIELDS_FOR_RENT_SHORT:
+                record.pop(field, None)
+        elif self.is_rent_long():
+            for field in SPECIFIC_FIELDS_FOR_RENT_SHORT | SPECIFIC_FIELDS_FOR_SALE:
+                record.pop(field, None)
+        elif self.is_rent_short():
+            for field in SPECIFIC_FIELDS_FOR_RENT_LONG | SPECIFIC_FIELDS_FOR_SALE:
+                record.pop(field, None)
+        return record
+
+    def run(self, url_format: str) -> list[dict[str, Any]]:
+        """Run pagination loop. Returns collected results."""
+        page = self.browser.new_page()
+
+        try:
+            self._paginate(page, url_format)
+        finally:
+            page.close()
+
+        return self.result
+
+    def _paginate(self, page: Page, url_format: str) -> None:
+        logger.info("Starting collection from page %d to %d", self.start_page, self.end_page)
+
+        if self.with_saving_csv:
+            logger.info("CSV will be saved to: %s", self.file_path)
+
+        page_number = self.start_page - 1
+
+        while page_number < self.end_page:
+            page_number += 1
+            attempt = 0
+            page_parsed = False
+
+            while attempt < 3 and not page_parsed:
+                try:
+                    page_parsed, should_stop = self._parse_page(page, url_format, page_number, attempt)
+                    if should_stop:
+                        return
+                except CaptchaError:
+                    raise
+                except Exception as exc:
+                    attempt += 1
+                    delay = attempt * 5
+                    logger.warning("Error on page %d (attempt %d/3): %s. Retrying in %ds...", page_number, attempt, exc, delay)
+                    if attempt >= 3:
+                        logger.error("Failed to parse page %d after 3 attempts: %s", page_number, exc)
+                        return
+                    time.sleep(delay)
+
+        logger.info("Collection complete. Total parsed: %d", self.count_parsed_offers)
+
+    def _parse_page(self, page: Page, url_format: str, page_number: int, attempt: int) -> tuple[bool, bool]:
+        """Parse a single list page. Returns (page_parsed, should_stop)."""
+        url = url_format.format(page_number)
+
+        if page_number == self.start_page and attempt == 0:
+            logger.info("Starting URL: %s", url)
+
+        self.browser.navigate(page, url)
+        page.wait_for_timeout(random.randint(3000, 5000))
+
+        # Scroll to load lazy content
+        page.evaluate("window.scrollTo(0, document.body.scrollHeight / 2)")
+        page.wait_for_timeout(1000)
+        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        page.wait_for_timeout(1000)
+
+        # Check for CAPTCHA
+        page_text = page.inner_text("body")
+        if "Captcha" in page_text or "captcha" in page_text:
+            logger.warning("CAPTCHA detected on page %d", page_number)
+            raise CaptchaError(
+                f"CAPTCHA detected on page {page_number}. Collected {len(self.result)} results before CAPTCHA.",
+                partial_results=list(self.result),
+            )
+
+        cards = self._get_cards(page)
+        if not cards:
+            logger.info("No cards found on page %d, stopping.", page_number)
+            return True, True
+
+        logger.info("Page %d: %d cards found", page_number, len(cards))
+
+        for card in cards:
+            self._process_card(card)
+
+        logger.info("Total parsed so far: %d", self.count_parsed_offers)
+
+        # Check pagination
+        has_next = self._has_next_page(page, page_number)
+        if not has_next:
+            logger.info("Last page reached (%d).", page_number)
+            return True, True
+
+        time.sleep(random.uniform(2, 4))
+        return True, False
+
+    def _get_cards(self, page: Page) -> list:
+        return page.query_selector_all("article[data-name='CardComponent']")
+
+    def _has_next_page(self, page: Page, current_page: int) -> bool:
+        next_btn = page.query_selector("[data-name='Pagination'] [class*='--next--']")
+        if next_btn:
+            return True
+
+        pag = page.query_selector("[data-name='Pagination']")
+        if not pag:
+            return False
+
+        page_links = page.query_selector_all("[data-name='Pagination'] li")
+        last_page = current_page
+        for pl in page_links:
+            txt = pl.inner_text().strip()
+            if txt.isdigit():
+                last_page = max(last_page, int(txt))
+
+        return current_page < last_page
+
+    def _process_card(self, card: Any) -> None:
+        card_data = self.parse_card(card)
+        url = card_data.get("url", "")
+
+        if not url:
+            return
+
+        url_id = define_deal_url_id(url)
+        if url_id in self.result_set:
+            return
+
+        page_data: dict[str, Any] = {}
+        if self.with_extra_data:
+            page_data = self.parse_detail_page(url)
+            time.sleep(4)
+
+        self.result_set.add(url_id)
+        # Merge detail data without overwriting valid card-level values with sentinels
+        for key, value in page_data.items():
+            if key in card_data and card_data[key] not in (-1, ""):
+                # Only overwrite if detail value is also non-sentinel
+                if value in (-1, ""):
+                    continue
+            card_data[key] = value
+        card_data = self.remove_unnecessary_fields(card_data)
+        self.result.append(card_data)
+        self.count_parsed_offers += 1
+
+        if self.with_saving_csv:
+            save_to_csv(self.result, str(self.file_path))

cian_playwright-2.0.0/src/cian_parser/browser.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from playwright.sync_api import sync_playwright, Browser, BrowserContext, Page, Playwright, Error as PlaywrightError
+from playwright_stealth import Stealth
+
+from cian_parser.exceptions import BrowserNotInstalledError
+
+logger = logging.getLogger("cian_parser")
+
+USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+)
+
+
+class BrowserManager:
+    """Manages a single Playwright browser instance with stealth settings."""
+
+    def __init__(self, *, headless: bool = True, proxy: str | None = None):
+        self._headless = headless
+        self._proxy = proxy
+        self._playwright: Playwright | None = None
+        self._browser: Browser | None = None
+        self._context: BrowserContext | None = None
+        self._stealth = Stealth()
+        self._started = False
+
+    def start(self) -> None:
+        if self._started:
+            return
+
+        self._playwright = sync_playwright().start()
+
+        try:
+            self._browser = self._playwright.chromium.launch(
+                headless=self._headless,
+                args=["--disable-blink-features=AutomationControlled", "--no-sandbox"],
+            )
+        except PlaywrightError as exc:
+            self._playwright.stop()
+            self._playwright = None
+            if "Executable doesn't exist" in str(exc) or "browserType.launch" in str(exc):
+                raise BrowserNotInstalledError(
+                    "Chromium browser is not installed. "
+                    "Run `playwright install chromium` to install it."
+                ) from exc
+            raise
+
+        context_kwargs: dict[str, Any] = {
+            "user_agent": USER_AGENT,
+            "viewport": {"width": 1920, "height": 1080},
+            "locale": "ru-RU",
+            "timezone_id": "Europe/Moscow",
+        }
+        if self._proxy:
+            context_kwargs["proxy"] = {"server": self._proxy}
+
+        self._context = self._browser.new_context(**context_kwargs)
+        self._started = True
+        logger.info("Browser started (headless=%s)", self._headless)
+
+    def new_page(self) -> Page:
+        self._ensure_started()
+        page = self._context.new_page()
+        self._stealth.apply_stealth_sync(page)
+        return page
+
+    def navigate(self, page: Page, url: str) -> None:
+        page.goto(url, wait_until="domcontentloaded", timeout=30000)
+
+    def close(self) -> None:
+        if self._browser:
+            self._browser.close()
+            self._browser = None
+        if self._playwright:
+            self._playwright.stop()
+            self._playwright = None
+        self._context = None
+        self._started = False
+        logger.info("Browser closed")
+
+    def _ensure_started(self) -> None:
+        if not self._started:
+            self.start()
+
+    def __enter__(self) -> "BrowserManager":
+        self.start()
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()