lacentrale-pros-scraper 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. lacentrale_pros_scraper-0.1.0/LICENSE +21 -0
  2. lacentrale_pros_scraper-0.1.0/MANIFEST.in +3 -0
  3. lacentrale_pros_scraper-0.1.0/PKG-INFO +102 -0
  4. lacentrale_pros_scraper-0.1.0/README.md +81 -0
  5. lacentrale_pros_scraper-0.1.0/examples/good_deals.py +28 -0
  6. lacentrale_pros_scraper-0.1.0/pyproject.toml +33 -0
  7. lacentrale_pros_scraper-0.1.0/setup.cfg +4 -0
  8. lacentrale_pros_scraper-0.1.0/src/lacentrale_pros/__init__.py +25 -0
  9. lacentrale_pros_scraper-0.1.0/src/lacentrale_pros/cli.py +120 -0
  10. lacentrale_pros_scraper-0.1.0/src/lacentrale_pros/client.py +164 -0
  11. lacentrale_pros_scraper-0.1.0/src/lacentrale_pros/parser.py +367 -0
  12. lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/PKG-INFO +102 -0
  13. lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/SOURCES.txt +20 -0
  14. lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/dependency_links.txt +1 -0
  15. lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/entry_points.txt +2 -0
  16. lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/requires.txt +1 -0
  17. lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/top_level.txt +1 -0
  18. lacentrale_pros_scraper-0.1.0/tests/fixtures/blocked.html +1 -0
  19. lacentrale_pros_scraper-0.1.0/tests/fixtures/dealer.html +1 -0
  20. lacentrale_pros_scraper-0.1.0/tests/fixtures/directory_lyon.html +1 -0
  21. lacentrale_pros_scraper-0.1.0/tests/fixtures/stock_page2.html +1 -0
  22. lacentrale_pros_scraper-0.1.0/tests/test_lacentrale.py +166 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Buğrahan Saka
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,3 @@
1
+ include README.md LICENSE
2
+ recursive-include tests *.py *.html
3
+ recursive-include examples *.py
@@ -0,0 +1,102 @@
1
+ Metadata-Version: 2.4
2
+ Name: lacentrale-pros-scraper
3
+ Version: 0.1.0
4
+ Summary: Scrape La Centrale's dealer directory (pros.lacentrale.fr): French car dealers, SIREN/SIRET, opening hours and every vehicle in stock with Python.
5
+ Author: Buğrahan Saka
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/bugrahansss/lacentrale-pros-scraper
8
+ Project-URL: Source, https://github.com/bugrahansss/lacentrale-pros-scraper
9
+ Project-URL: Issues, https://github.com/bugrahansss/lacentrale-pros-scraper/issues
10
+ Keywords: lacentrale,la centrale,pros.lacentrale.fr,car dealers,used cars,france,web scraping,scraper,scrapedo
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
16
+ Requires-Python: >=3.8
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: requests>=2.25
20
+ Dynamic: license-file
21
+
22
+ # La Centrale Pros Scraper
23
+
24
+ [![PyPI](https://img.shields.io/pypi/v/lacentrale-pros-scraper)](https://pypi.org/project/lacentrale-pros-scraper/)
25
+ [![Python](https://img.shields.io/pypi/pyversions/lacentrale-pros-scraper)](https://pypi.org/project/lacentrale-pros-scraper/)
26
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
27
+
28
+ Scrape [La Centrale's](https://www.lacentrale.fr) dealer directory and dealer pages ([pros.lacentrale.fr](https://pros.lacentrale.fr)) into structured data. List every car dealer in a **city, department, region or brand network**, open a dealer for its **SIREN and SIRET, founding year, verified badge, address with coordinates, opening hours, website and stock size**, and pull **every vehicle in its stock**: make, model, version, year, mileage, fuel, gearbox, price, monthly price, La Centrale's **deal rating** (very good, good, fair, bad), warranty and listing dates. JSON or CSV.
29
+
30
+ ```bash
31
+ pip install lacentrale-pros-scraper
32
+ export SCRAPEDO_TOKEN=your_token
33
+ lacentrale-pros --city lyon -f csv -o lyon_dealers.csv # directory cards
34
+ lacentrale-pros --city lyon --details -o lyon_dealers.json # + SIREN/SIRET, hours, website
35
+ lacentrale-pros --city lyon --stock -f csv -o lyon_cars.csv # every car of every dealer
36
+ lacentrale-pros C045088 --stock -o stock.json # one dealer
37
+ ```
38
+
39
+ ## Why plain requests fail
40
+
41
+ ```python
42
+ import requests
43
+
44
+ r = requests.get("https://pros.lacentrale.fr/C045088")
45
+ print(r.status_code) # 403
46
+ print(r.text[:200]) # ... Please enable JS and disable any ad blocker ... dd={'rt':'c' ...
47
+ ```
48
+
49
+ lacentrale.fr is protected by DataDome, which answers a script with `403` and a JavaScript challenge. This package sends each request through [Scrape.do](https://scrape.do/?utm_source=github&utm_medium=readme&utm_campaign=lacentrale-pros-scraper), which returns the real page. A plain request is enough, no extra parameters: **1 credit per page**.
50
+
51
+ ## Where the data is
52
+
53
+ * **Directory** (`www.lacentrale.fr/annuaire/garages-villes/<city>`, `garages-departements/<dept>`, `garages-regions/<region>`, `marques/<brand>`, `?page=N&perPage=20`): server-rendered cards with the dealer name, postcode and city, number of ads, a "Marques" badge for franchised dealers, and the dealer reference (`C` + 6 digits) in `data-tracking-meta`. Some dealers link to their own website instead of their La Centrale page, but the reference is still there.
54
+ * **Dealer page** (`pros.lacentrale.fr/C045088`):
55
+ * an `AutoDealer` JSON-LD block with name, address and coordinates. Its `openingHoursSpecification` is a **JSON string inside the JSON**: decode it a second time.
56
+ * an identity card ("Vendeur vérifié", "Date de création", "Stock"; big groups show "Stock national" instead).
57
+ * a footer with **SIREN / SIRET** and the dealer's website.
58
+ * `window.__PRELOADED_STATE_LISTING__`, whose `search.hits` is the first page of stock (9 vehicles) and `search.total` the stock size.
59
+ * **More stock**: `pros.lacentrale.fr/C045088/index?options=&page=2&vertical=car`, same state object.
60
+
61
+ In a live test, Lyon had 17 dealers in the directory and 228 cars in stock, scraped for 37 credits. 42 of those cars carried La Centrale's "very good deal" badge and 52 "good deal"; the median price was €28,940.
62
+
63
+ ## Python
64
+
65
+ ```python
66
+ from lacentrale_pros import LaCentralePros
67
+
68
+ lc = LaCentralePros(token="YOUR_SCRAPEDO_TOKEN") # or set SCRAPEDO_TOKEN
69
+
70
+ for card in lc.directory("city", "Lyon"):
71
+ print(card.ref, card.name, card.postcode, card.ads)
72
+
73
+ dealer, cars = lc.dealer_with_stock("C045088")
74
+ print(dealer.name, dealer.siret, dealer.founded, dealer.opening_hours)
75
+ for car in cars:
76
+ print(car.make, car.model, car.year, car.mileage, car.price, car.deal)
77
+ print(lc.credits_used)
78
+ ```
79
+
80
+ ## Fields
81
+
82
+ **Directory card:** `ref`, `name`, `postcode`, `city`, `ads`, `franchise`, `url`, `own_site`.
83
+
84
+ **Dealer:** `ref`, `url`, `name`, `dealer_type` (`CONCESSIONNAIRE`, `CENTRE_MULTIMARQUES`, ...), `verified`, `founded`, `stock`, `stock_national`, `street`, `postcode`, `city`, `country`, `latitude`, `longitude`, `opening_hours`, `siren`, `siret`, `website`, `phone`.
85
+
86
+ **Vehicle:** `reference`, `url`, `make`, `model`, `version`, `year`, `mileage`, `energy`, `gearbox`, `category`, `color`, `doors`, `price`, `monthly_from`, `deal` (`VERY_GOOD_DEAL`, `GOOD_DEAL`, `EQUITABLE_DEAL`, `BAD_DEAL`, `NOT_COMPUTED`), `warranty_months`, `first_online`, `last_update`, `department`, `pictures`, `photo`, `is_new`, `dealer_ref`, `dealer_name`.
87
+
88
+ ## Use it responsibly
89
+
90
+ This is business data published by professional sellers. Some dealers are sole traders, so treat names and phone numbers as personal data under the GDPR: use them for legitimate purposes, honour objection requests, and respect La Centrale's terms. Don't republish vehicle photos. This is not legal advice.
91
+
92
+ ## Tests
93
+
94
+ ```bash
95
+ PYTHONPATH=src python -m unittest discover -s tests
96
+ ```
97
+
98
+ The fixtures are fictional ("GARAGE EXEMPLE LYON", example.com).
99
+
100
+ ## License
101
+
102
+ MIT
@@ -0,0 +1,81 @@
1
+ # La Centrale Pros Scraper
2
+
3
+ [![PyPI](https://img.shields.io/pypi/v/lacentrale-pros-scraper)](https://pypi.org/project/lacentrale-pros-scraper/)
4
+ [![Python](https://img.shields.io/pypi/pyversions/lacentrale-pros-scraper)](https://pypi.org/project/lacentrale-pros-scraper/)
5
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
6
+
7
+ Scrape [La Centrale's](https://www.lacentrale.fr) dealer directory and dealer pages ([pros.lacentrale.fr](https://pros.lacentrale.fr)) into structured data. List every car dealer in a **city, department, region or brand network**, open a dealer for its **SIREN and SIRET, founding year, verified badge, address with coordinates, opening hours, website and stock size**, and pull **every vehicle in its stock**: make, model, version, year, mileage, fuel, gearbox, price, monthly price, La Centrale's **deal rating** (very good, good, fair, bad), warranty and listing dates. JSON or CSV.
8
+
9
+ ```bash
10
+ pip install lacentrale-pros-scraper
11
+ export SCRAPEDO_TOKEN=your_token
12
+ lacentrale-pros --city lyon -f csv -o lyon_dealers.csv # directory cards
13
+ lacentrale-pros --city lyon --details -o lyon_dealers.json # + SIREN/SIRET, hours, website
14
+ lacentrale-pros --city lyon --stock -f csv -o lyon_cars.csv # every car of every dealer
15
+ lacentrale-pros C045088 --stock -o stock.json # one dealer
16
+ ```
17
+
18
+ ## Why plain requests fail
19
+
20
+ ```python
21
+ import requests
22
+
23
+ r = requests.get("https://pros.lacentrale.fr/C045088")
24
+ print(r.status_code) # 403
25
+ print(r.text[:200]) # ... Please enable JS and disable any ad blocker ... dd={'rt':'c' ...
26
+ ```
27
+
28
+ lacentrale.fr is protected by DataDome, which answers a script with `403` and a JavaScript challenge. This package sends each request through [Scrape.do](https://scrape.do/?utm_source=github&utm_medium=readme&utm_campaign=lacentrale-pros-scraper), which returns the real page. A plain request is enough, no extra parameters: **1 credit per page**.
29
+
30
+ ## Where the data is
31
+
32
+ * **Directory** (`www.lacentrale.fr/annuaire/garages-villes/<city>`, `garages-departements/<dept>`, `garages-regions/<region>`, `marques/<brand>`, `?page=N&perPage=20`): server-rendered cards with the dealer name, postcode and city, number of ads, a "Marques" badge for franchised dealers, and the dealer reference (`C` + 6 digits) in `data-tracking-meta`. Some dealers link to their own website instead of their La Centrale page, but the reference is still there.
33
+ * **Dealer page** (`pros.lacentrale.fr/C045088`):
34
+ * an `AutoDealer` JSON-LD block with name, address and coordinates. Its `openingHoursSpecification` is a **JSON string inside the JSON**: decode it a second time.
35
+ * an identity card ("Vendeur vérifié", "Date de création", "Stock"; big groups show "Stock national" instead).
36
+ * a footer with **SIREN / SIRET** and the dealer's website.
37
+ * `window.__PRELOADED_STATE_LISTING__`, whose `search.hits` is the first page of stock (9 vehicles) and `search.total` the stock size.
38
+ * **More stock**: `pros.lacentrale.fr/C045088/index?options=&page=2&vertical=car`, same state object.
39
+
40
+ In a live test, Lyon had 17 dealers in the directory and 228 cars in stock, scraped for 37 credits. 42 of those cars carried La Centrale's "very good deal" badge and 52 "good deal"; the median price was €28,940.
41
+
42
+ ## Python
43
+
44
+ ```python
45
+ from lacentrale_pros import LaCentralePros
46
+
47
+ lc = LaCentralePros(token="YOUR_SCRAPEDO_TOKEN") # or set SCRAPEDO_TOKEN
48
+
49
+ for card in lc.directory("city", "Lyon"):
50
+ print(card.ref, card.name, card.postcode, card.ads)
51
+
52
+ dealer, cars = lc.dealer_with_stock("C045088")
53
+ print(dealer.name, dealer.siret, dealer.founded, dealer.opening_hours)
54
+ for car in cars:
55
+ print(car.make, car.model, car.year, car.mileage, car.price, car.deal)
56
+ print(lc.credits_used)
57
+ ```
58
+
59
+ ## Fields
60
+
61
+ **Directory card:** `ref`, `name`, `postcode`, `city`, `ads`, `franchise`, `url`, `own_site`.
62
+
63
+ **Dealer:** `ref`, `url`, `name`, `dealer_type` (`CONCESSIONNAIRE`, `CENTRE_MULTIMARQUES`, ...), `verified`, `founded`, `stock`, `stock_national`, `street`, `postcode`, `city`, `country`, `latitude`, `longitude`, `opening_hours`, `siren`, `siret`, `website`, `phone`.
64
+
65
+ **Vehicle:** `reference`, `url`, `make`, `model`, `version`, `year`, `mileage`, `energy`, `gearbox`, `category`, `color`, `doors`, `price`, `monthly_from`, `deal` (`VERY_GOOD_DEAL`, `GOOD_DEAL`, `EQUITABLE_DEAL`, `BAD_DEAL`, `NOT_COMPUTED`), `warranty_months`, `first_online`, `last_update`, `department`, `pictures`, `photo`, `is_new`, `dealer_ref`, `dealer_name`.
66
+
67
+ ## Use it responsibly
68
+
69
+ This is business data published by professional sellers. Some dealers are sole traders, so treat names and phone numbers as personal data under the GDPR: use them for legitimate purposes, honour objection requests, and respect La Centrale's terms. Don't republish vehicle photos. This is not legal advice.
70
+
71
+ ## Tests
72
+
73
+ ```bash
74
+ PYTHONPATH=src python -m unittest discover -s tests
75
+ ```
76
+
77
+ The fixtures are fictional ("GARAGE EXEMPLE LYON", example.com).
78
+
79
+ ## License
80
+
81
+ MIT
@@ -0,0 +1,28 @@
1
+ """Every car rated "very good deal" by La Centrale among a city's dealers, cheapest first.
2
+
3
+ SCRAPEDO_TOKEN=... python examples/good_deals.py lyon
4
+ """
5
+
6
+ import csv
7
+ import sys
8
+
9
+ from lacentrale_pros import LaCentralePros
10
+
11
+ city = sys.argv[1] if len(sys.argv) > 1 else "lyon"
12
+ lc = LaCentralePros()
13
+
14
+ deals = []
15
+ for card in lc.directory("city", city):
16
+ if not card.ads:
17
+ continue
18
+ dealer, cars = lc.dealer_with_stock(card.ref)
19
+ deals += [c for c in cars if c.deal == "VERY_GOOD_DEAL"]
20
+ print(f"{dealer.name}: {len(cars)} cars", file=sys.stderr)
21
+
22
+ deals.sort(key=lambda c: c.price or 0)
23
+ with open(f"{city}_very_good_deals.csv", "w", newline="", encoding="utf-8") as fh:
24
+ writer = csv.DictWriter(fh, fieldnames=["make", "model", "version", "year", "mileage", "price", "dealer_name", "url"],
25
+ extrasaction="ignore")
26
+ writer.writeheader()
27
+ writer.writerows(c.to_dict() for c in deals)
28
+ print(f"{len(deals)} very good deals, {lc.credits_used} credits", file=sys.stderr)
@@ -0,0 +1,33 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "lacentrale-pros-scraper"
7
+ version = "0.1.0"
8
+ description = "Scrape La Centrale's dealer directory (pros.lacentrale.fr): French car dealers, SIREN/SIRET, opening hours and every vehicle in stock with Python."
9
+ readme = "README.md"
10
+ requires-python = ">=3.8"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Buğrahan Saka" }]
14
+ keywords = ["lacentrale", "la centrale", "pros.lacentrale.fr", "car dealers", "used cars", "france", "web scraping", "scraper", "scrapedo"]
15
+ classifiers = [
16
+ "Development Status :: 4 - Beta",
17
+ "Intended Audience :: Developers",
18
+ "Operating System :: OS Independent",
19
+ "Programming Language :: Python :: 3",
20
+ "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
21
+ ]
22
+ dependencies = ["requests>=2.25"]
23
+
24
+ [project.urls]
25
+ Homepage = "https://github.com/bugrahansss/lacentrale-pros-scraper"
26
+ Source = "https://github.com/bugrahansss/lacentrale-pros-scraper"
27
+ Issues = "https://github.com/bugrahansss/lacentrale-pros-scraper/issues"
28
+
29
+ [project.scripts]
30
+ lacentrale-pros = "lacentrale_pros.cli:main"
31
+
32
+ [tool.setuptools.packages.find]
33
+ where = ["src"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,25 @@
1
+ """Scrape La Centrale's dealer directory (pros.lacentrale.fr): dealers, profiles and stock."""
2
+
3
+ from .client import LaCentralePros, ScrapeError
4
+ from .parser import (
5
+ Dealer,
6
+ DealerCard,
7
+ DirectoryPage,
8
+ StockPage,
9
+ Vehicle,
10
+ dealer_ref,
11
+ dealer_url,
12
+ directory_url,
13
+ opening_hours,
14
+ parse_dealer,
15
+ parse_directory,
16
+ parse_stock,
17
+ stock_url,
18
+ )
19
+
20
+ __version__ = "0.1.0"
21
+ __all__ = [
22
+ "LaCentralePros", "ScrapeError", "Dealer", "DealerCard", "DirectoryPage", "StockPage", "Vehicle",
23
+ "dealer_ref", "dealer_url", "directory_url", "opening_hours", "parse_dealer", "parse_directory",
24
+ "parse_stock", "stock_url", "__version__",
25
+ ]
@@ -0,0 +1,120 @@
1
+ """Command line entry point.
2
+
3
+ lacentrale-pros --city lyon -f csv -o lyon_dealers.csv
4
+ lacentrale-pros --city lyon --details -o lyon_dealers.json
5
+ lacentrale-pros --city lyon --stock -f csv -o lyon_cars.csv
6
+ lacentrale-pros C045088 --stock -o stock.json
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import csv
13
+ import json
14
+ import sys
15
+ from typing import Any, List, Optional
16
+
17
+ from . import __version__
18
+ from .client import TOKEN_ENV, LaCentralePros, ScrapeError
19
+ from .parser import CARD_FIELDS, DEALER_FIELDS, VEHICLE_FIELDS
20
+
21
+
22
+ def build_parser() -> argparse.ArgumentParser:
23
+ p = argparse.ArgumentParser(prog="lacentrale-pros",
24
+ description="Scrape La Centrale's dealer directory, dealer profiles and dealer stock.")
25
+ p.add_argument("dealers", nargs="*", help="Dealer references or URLs (C045088, https://pros.lacentrale.fr/C045088)")
26
+ p.add_argument("--city", action="append", default=[], help="Directory by city (lyon, paris, ...)")
27
+ p.add_argument("--department", action="append", default=[], help="Directory by department (rhone, nord, ...)")
28
+ p.add_argument("--region", action="append", default=[], help="Directory by region (bretagne, ile-de-france, ...)")
29
+ p.add_argument("--brand", action="append", default=[], help="Directory by brand (peugeot, bmw, ...)")
30
+ p.add_argument("--pages", type=int, default=0, help="Directory pages per search, 20 dealers each (default: all)")
31
+ p.add_argument("--details", action="store_true", help="Open every dealer page: SIREN/SIRET, founding year, hours, website")
32
+ p.add_argument("--stock", action="store_true", help="Output every vehicle in each dealer's stock (9 per page)")
33
+ p.add_argument("--limit", type=int, default=0, help="Stop after this many dealers")
34
+ p.add_argument("-t", "--token", help=f"Scrape.do API token (default: ${TOKEN_ENV})")
35
+ p.add_argument("-f", "--format", choices=("json", "csv"), default="json")
36
+ p.add_argument("-o", "--output", help="Write to this file instead of stdout")
37
+ p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
38
+ return p
39
+
40
+
41
+ def _write(rows: List[dict], fields: List[str], fmt: str, output: Optional[str]) -> None:
42
+ out = open(output, "w", newline="", encoding="utf-8") if output else sys.stdout
43
+ try:
44
+ if fmt == "json":
45
+ json.dump(rows, out, ensure_ascii=False, indent=2)
46
+ out.write("\n")
47
+ else:
48
+ writer = csv.DictWriter(out, fieldnames=fields, extrasaction="ignore")
49
+ writer.writeheader()
50
+ for row in rows:
51
+ writer.writerow({k: json.dumps(v, ensure_ascii=False) if isinstance(v, (list, dict)) else v
52
+ for k, v in row.items()})
53
+ finally:
54
+ if output:
55
+ out.close()
56
+
57
+
58
+ def main(argv: Optional[List[str]] = None) -> int:
59
+ args = build_parser().parse_args(argv)
60
+ searches = [("city", v) for v in args.city] + [("department", v) for v in args.department] + \
61
+ [("region", v) for v in args.region] + [("brand", v) for v in args.brand]
62
+ if not args.dealers and not searches:
63
+ build_parser().print_usage(sys.stderr)
64
+ print("Give dealer references or a --city/--department/--region/--brand.", file=sys.stderr)
65
+ return 2
66
+ try:
67
+ lc = LaCentralePros(token=args.token)
68
+ except ValueError as exc:
69
+ print(exc, file=sys.stderr)
70
+ return 2
71
+
72
+ cards: List[Any] = []
73
+ refs: List[str] = list(args.dealers)
74
+ failures = 0
75
+ for kind, name in searches:
76
+ try:
77
+ found = list(lc.directory(kind, name, args.pages))
78
+ except ScrapeError as exc:
79
+ failures += 1
80
+ print(f"fail {kind} {name}: {exc}", file=sys.stderr)
81
+ continue
82
+ print(f"{kind} {name}: {len(found)} dealers", file=sys.stderr)
83
+ cards.extend(found)
84
+ refs.extend(c.ref for c in found)
85
+ refs = list(dict.fromkeys(refs))
86
+ if args.limit:
87
+ refs = refs[: args.limit]
88
+ cards = [c for c in cards if c.ref in refs]
89
+
90
+ if not args.details and not args.stock:
91
+ rows, fields = [c.to_dict() for c in cards], CARD_FIELDS
92
+ if args.dealers:
93
+ for ref in args.dealers:
94
+ try:
95
+ rows.append(lc.dealer(ref).to_dict())
96
+ fields = DEALER_FIELDS
97
+ except (ScrapeError, ValueError) as exc:
98
+ failures += 1
99
+ print(f"fail {ref}: {exc}", file=sys.stderr)
100
+ else:
101
+ rows, fields = [], VEHICLE_FIELDS if args.stock else DEALER_FIELDS
102
+ for ref in refs:
103
+ try:
104
+ if args.stock:
105
+ dealer, vehicles = lc.dealer_with_stock(ref)
106
+ rows.extend(v.to_dict() for v in vehicles)
107
+ print(f"{ref} {dealer.name}: {len(vehicles)} vehicles", file=sys.stderr)
108
+ else:
109
+ rows.append(lc.dealer(ref).to_dict())
110
+ except (ScrapeError, ValueError) as exc:
111
+ failures += 1
112
+ print(f"fail {ref}: {exc}", file=sys.stderr)
113
+
114
+ _write(rows, fields, args.format, args.output)
115
+ print(f"saved {len(rows)} rows ({lc.credits_used} credits)", file=sys.stderr)
116
+ return 1 if failures and not rows else 0
117
+
118
+
119
+ if __name__ == "__main__":
120
+ sys.exit(main())
@@ -0,0 +1,164 @@
1
+ """Fetch La Centrale's dealer directory and dealer pages through the Scrape.do API.
2
+
3
+ lacentrale.fr sits behind DataDome: a plain ``requests.get`` from a script gets
4
+ ``403`` and a "Please enable JS and disable any ad blocker" page. A plain
5
+ Scrape.do request (no extra parameters) returns the real server-rendered HTML.
6
+ Each page costs 1 Scrape.do credit.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import os
12
+ import time
13
+ from typing import Iterator, List, Optional, Tuple
14
+
15
+ import requests
16
+
17
+ from .parser import (
18
+ Dealer,
19
+ DealerCard,
20
+ Vehicle,
21
+ dealer_ref,
22
+ dealer_url,
23
+ directory_url,
24
+ parse_dealer,
25
+ parse_directory,
26
+ parse_stock,
27
+ stock_url,
28
+ )
29
+
30
+ API_URL = "https://api.scrape.do/"
31
+ TOKEN_ENV = "SCRAPEDO_TOKEN"
32
+ BLOCK_MARKERS = ("Please enable JS and disable any ad blocker", "captcha-delivery.com")
33
+
34
+
35
+ class ScrapeError(RuntimeError):
36
+ def __init__(self, message: str, status_code: Optional[int] = None):
37
+ super().__init__(message)
38
+ self.status_code = status_code
39
+
40
+
41
+ class LaCentralePros:
42
+ """Read La Centrale's dealer directory, dealer profiles and every vehicle a dealer has in stock.
43
+
44
+ >>> lc = LaCentralePros(token="YOUR_SCRAPEDO_TOKEN")
45
+ >>> for card in lc.directory("city", "Lyon"):
46
+ ... print(card.ref, card.name, card.ads)
47
+ >>> dealer, cars = lc.dealer_with_stock("C045088")
48
+ """
49
+
50
+ def __init__(self, token: Optional[str] = None, timeout: float = 60.0, retries: int = 2,
51
+ session: Optional[requests.Session] = None):
52
+ self.token = token or os.environ.get(TOKEN_ENV, "")
53
+ if not self.token:
54
+ raise ValueError(f"A Scrape.do API token is required. Pass token=... or set {TOKEN_ENV}.")
55
+ self.timeout = timeout
56
+ self.retries = max(0, retries)
57
+ self.session = session or requests.Session()
58
+ self.requests_made = 0
59
+
60
+ def fetch(self, url: str) -> str:
61
+ params = {"token": self.token, "url": url}
62
+ last: Optional[ScrapeError] = None
63
+ for attempt in range(self.retries + 1):
64
+ try:
65
+ response = self.session.get(API_URL, params=params, timeout=self.timeout)
66
+ except requests.RequestException as exc:
67
+ last = ScrapeError(f"Request failed: {exc}")
68
+ else:
69
+ if response.status_code == 200:
70
+ self.requests_made += 1
71
+ if not any(marker in response.text for marker in BLOCK_MARKERS):
72
+ return response.text
73
+ last = ScrapeError(f"Blocked page for {url} (DataDome)", 200)
74
+ else:
75
+ last = ScrapeError(f"Request for {url} failed with status {response.status_code}",
76
+ response.status_code)
77
+ if response.status_code in (400, 401, 404, 410):
78
+ break
79
+ if attempt < self.retries:
80
+ time.sleep(2 * (attempt + 1))
81
+ assert last is not None
82
+ raise last
83
+
84
+ # ---- directory -----------------------------------------------------------
85
+
86
+ def directory(self, kind: str, name: str, pages: int = 0) -> Iterator[DealerCard]:
87
+ """Dealers listed for a ``city``, ``department``, ``region`` or ``brand`` (20 per page).
88
+
89
+ ``pages=0`` follows every page.
90
+ """
91
+ seen = set()
92
+ page, last_page = 0, None
93
+ while True:
94
+ if last_page is not None and page >= last_page:
95
+ break
96
+ if pages and page >= pages:
97
+ break
98
+ result = parse_directory(self.fetch(directory_url(kind, name, page)))
99
+ last_page = result.pages
100
+ fresh = [c for c in result.dealers if c.ref not in seen]
101
+ if not fresh:
102
+ break
103
+ for card in fresh:
104
+ seen.add(card.ref)
105
+ yield card
106
+ page += 1
107
+
108
+ # ---- dealer --------------------------------------------------------------
109
+
110
+ def dealer(self, ref: str) -> Dealer:
111
+ ref = dealer_ref(ref)
112
+ dealer = parse_dealer(self.fetch(dealer_url(ref)), ref)
113
+ if dealer is None:
114
+ raise ScrapeError(f"No dealer data on {dealer_url(ref)}")
115
+ return dealer
116
+
117
+ def stock(self, ref: str, vertical: str = "car", pages: int = 0) -> Iterator[Vehicle]:
118
+ """Every vehicle a dealer lists (9 per page). ``pages=0`` follows every page."""
119
+ ref = dealer_ref(ref)
120
+ seen = set()
121
+ page, last_page = 1, None
122
+ while True:
123
+ if (last_page is not None and page > last_page) or (pages and page > pages):
124
+ break
125
+ result = parse_stock(self.fetch(stock_url(ref, page, vertical)), ref)
126
+ if result is None:
127
+ raise ScrapeError(f"No stock data on {stock_url(ref, page, vertical)}")
128
+ last_page = result.pages
129
+ fresh = [v for v in result.vehicles if v.reference not in seen]
130
+ if not fresh:
131
+ break
132
+ for vehicle in fresh:
133
+ seen.add(vehicle.reference)
134
+ yield vehicle
135
+ page += 1
136
+
137
+ def dealer_with_stock(self, ref: str, pages: int = 0) -> Tuple[Dealer, List[Vehicle]]:
138
+ """Dealer profile plus its car stock. The dealer page doubles as stock page 1."""
139
+ ref = dealer_ref(ref)
140
+ html = self.fetch(dealer_url(ref))
141
+ dealer = parse_dealer(html, ref)
142
+ if dealer is None:
143
+ raise ScrapeError(f"No dealer data on {dealer_url(ref)}")
144
+ first = parse_stock(html, ref)
145
+ vehicles: List[Vehicle] = list(first.vehicles) if first else []
146
+ seen = {v.reference for v in vehicles}
147
+ last_page = first.pages if first else 1
148
+ page = 2
149
+ while page <= last_page and not (pages and page > pages):
150
+ result = parse_stock(self.fetch(stock_url(ref, page)), ref)
151
+ if result is None:
152
+ break
153
+ fresh = [v for v in result.vehicles if v.reference not in seen]
154
+ if not fresh:
155
+ break
156
+ for v in fresh:
157
+ seen.add(v.reference)
158
+ vehicles.append(v)
159
+ page += 1
160
+ return dealer, vehicles
161
+
162
+ @property
163
+ def credits_used(self) -> int:
164
+ return self.requests_made