lacentrale-pros-scraper 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lacentrale_pros_scraper-0.1.0/LICENSE +21 -0
- lacentrale_pros_scraper-0.1.0/MANIFEST.in +3 -0
- lacentrale_pros_scraper-0.1.0/PKG-INFO +102 -0
- lacentrale_pros_scraper-0.1.0/README.md +81 -0
- lacentrale_pros_scraper-0.1.0/examples/good_deals.py +28 -0
- lacentrale_pros_scraper-0.1.0/pyproject.toml +33 -0
- lacentrale_pros_scraper-0.1.0/setup.cfg +4 -0
- lacentrale_pros_scraper-0.1.0/src/lacentrale_pros/__init__.py +25 -0
- lacentrale_pros_scraper-0.1.0/src/lacentrale_pros/cli.py +120 -0
- lacentrale_pros_scraper-0.1.0/src/lacentrale_pros/client.py +164 -0
- lacentrale_pros_scraper-0.1.0/src/lacentrale_pros/parser.py +367 -0
- lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/PKG-INFO +102 -0
- lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/SOURCES.txt +20 -0
- lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/dependency_links.txt +1 -0
- lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/entry_points.txt +2 -0
- lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/requires.txt +1 -0
- lacentrale_pros_scraper-0.1.0/src/lacentrale_pros_scraper.egg-info/top_level.txt +1 -0
- lacentrale_pros_scraper-0.1.0/tests/fixtures/blocked.html +1 -0
- lacentrale_pros_scraper-0.1.0/tests/fixtures/dealer.html +1 -0
- lacentrale_pros_scraper-0.1.0/tests/fixtures/directory_lyon.html +1 -0
- lacentrale_pros_scraper-0.1.0/tests/fixtures/stock_page2.html +1 -0
- lacentrale_pros_scraper-0.1.0/tests/test_lacentrale.py +166 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Buğrahan Saka
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: lacentrale-pros-scraper
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Scrape La Centrale's dealer directory (pros.lacentrale.fr): French car dealers, SIREN/SIRET, opening hours and every vehicle in stock with Python.
|
|
5
|
+
Author: Buğrahan Saka
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/bugrahansss/lacentrale-pros-scraper
|
|
8
|
+
Project-URL: Source, https://github.com/bugrahansss/lacentrale-pros-scraper
|
|
9
|
+
Project-URL: Issues, https://github.com/bugrahansss/lacentrale-pros-scraper/issues
|
|
10
|
+
Keywords: lacentrale,la centrale,pros.lacentrale.fr,car dealers,used cars,france,web scraping,scraper,scrapedo
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
16
|
+
Requires-Python: >=3.8
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: requests>=2.25
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# La Centrale Pros Scraper
|
|
23
|
+
|
|
24
|
+
[](https://pypi.org/project/lacentrale-pros-scraper/)
|
|
25
|
+
[](https://pypi.org/project/lacentrale-pros-scraper/)
|
|
26
|
+
[](LICENSE)
|
|
27
|
+
|
|
28
|
+
Scrape [La Centrale's](https://www.lacentrale.fr) dealer directory and dealer pages ([pros.lacentrale.fr](https://pros.lacentrale.fr)) into structured data. List every car dealer in a **city, department, region or brand network**, open a dealer for its **SIREN and SIRET, founding year, verified badge, address with coordinates, opening hours, website and stock size**, and pull **every vehicle in its stock**: make, model, version, year, mileage, fuel, gearbox, price, monthly price, La Centrale's **deal rating** (very good, good, fair, bad), warranty and listing dates. JSON or CSV.
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install lacentrale-pros-scraper
|
|
32
|
+
export SCRAPEDO_TOKEN=your_token
|
|
33
|
+
lacentrale-pros --city lyon -f csv -o lyon_dealers.csv # directory cards
|
|
34
|
+
lacentrale-pros --city lyon --details -o lyon_dealers.json # + SIREN/SIRET, hours, website
|
|
35
|
+
lacentrale-pros --city lyon --stock -f csv -o lyon_cars.csv # every car of every dealer
|
|
36
|
+
lacentrale-pros C045088 --stock -o stock.json # one dealer
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Why plain requests fail
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
import requests
|
|
43
|
+
|
|
44
|
+
r = requests.get("https://pros.lacentrale.fr/C045088")
|
|
45
|
+
print(r.status_code) # 403
|
|
46
|
+
print(r.text[:200]) # ... Please enable JS and disable any ad blocker ... dd={'rt':'c' ...
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
lacentrale.fr is protected by DataDome, which answers a script with `403` and a JavaScript challenge. This package sends each request through [Scrape.do](https://scrape.do/?utm_source=github&utm_medium=readme&utm_campaign=lacentrale-pros-scraper), which returns the real page. A plain request is enough, no extra parameters: **1 credit per page**.
|
|
50
|
+
|
|
51
|
+
## Where the data is
|
|
52
|
+
|
|
53
|
+
* **Directory** (`www.lacentrale.fr/annuaire/garages-villes/<city>`, `garages-departements/<dept>`, `garages-regions/<region>`, `marques/<brand>`, `?page=N&perPage=20`): server-rendered cards with the dealer name, postcode and city, number of ads, a "Marques" badge for franchised dealers, and the dealer reference (`C` + 6 digits) in `data-tracking-meta`. Some dealers link to their own website instead of their La Centrale page, but the reference is still there.
|
|
54
|
+
* **Dealer page** (`pros.lacentrale.fr/C045088`):
|
|
55
|
+
* an `AutoDealer` JSON-LD block with name, address and coordinates. Its `openingHoursSpecification` is a **JSON string inside the JSON**: decode it a second time.
|
|
56
|
+
* an identity card ("Vendeur vérifié", "Date de création", "Stock"; big groups show "Stock national" instead).
|
|
57
|
+
* a footer with **SIREN / SIRET** and the dealer's website.
|
|
58
|
+
* `window.__PRELOADED_STATE_LISTING__`, whose `search.hits` is the first page of stock (9 vehicles) and `search.total` the stock size.
|
|
59
|
+
* **More stock**: `pros.lacentrale.fr/C045088/index?options=&page=2&vertical=car`, same state object.
|
|
60
|
+
|
|
61
|
+
In a live test, Lyon had 17 dealers in the directory and 228 cars in stock, scraped for 37 credits. 42 of those cars carried La Centrale's "very good deal" badge and 52 "good deal"; the median price was €28,940.
|
|
62
|
+
|
|
63
|
+
## Python
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from lacentrale_pros import LaCentralePros
|
|
67
|
+
|
|
68
|
+
lc = LaCentralePros(token="YOUR_SCRAPEDO_TOKEN") # or set SCRAPEDO_TOKEN
|
|
69
|
+
|
|
70
|
+
for card in lc.directory("city", "Lyon"):
|
|
71
|
+
print(card.ref, card.name, card.postcode, card.ads)
|
|
72
|
+
|
|
73
|
+
dealer, cars = lc.dealer_with_stock("C045088")
|
|
74
|
+
print(dealer.name, dealer.siret, dealer.founded, dealer.opening_hours)
|
|
75
|
+
for car in cars:
|
|
76
|
+
print(car.make, car.model, car.year, car.mileage, car.price, car.deal)
|
|
77
|
+
print(lc.credits_used)
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Fields
|
|
81
|
+
|
|
82
|
+
**Directory card:** `ref`, `name`, `postcode`, `city`, `ads`, `franchise`, `url`, `own_site`.
|
|
83
|
+
|
|
84
|
+
**Dealer:** `ref`, `url`, `name`, `dealer_type` (`CONCESSIONNAIRE`, `CENTRE_MULTIMARQUES`, ...), `verified`, `founded`, `stock`, `stock_national`, `street`, `postcode`, `city`, `country`, `latitude`, `longitude`, `opening_hours`, `siren`, `siret`, `website`, `phone`.
|
|
85
|
+
|
|
86
|
+
**Vehicle:** `reference`, `url`, `make`, `model`, `version`, `year`, `mileage`, `energy`, `gearbox`, `category`, `color`, `doors`, `price`, `monthly_from`, `deal` (`VERY_GOOD_DEAL`, `GOOD_DEAL`, `EQUITABLE_DEAL`, `BAD_DEAL`, `NOT_COMPUTED`), `warranty_months`, `first_online`, `last_update`, `department`, `pictures`, `photo`, `is_new`, `dealer_ref`, `dealer_name`.
|
|
87
|
+
|
|
88
|
+
## Use it responsibly
|
|
89
|
+
|
|
90
|
+
This is business data published by professional sellers. Some dealers are sole traders, so treat names and phone numbers as personal data under the GDPR: use them for legitimate purposes, honour objection requests, and respect La Centrale's terms. Don't republish vehicle photos. This is not legal advice.
|
|
91
|
+
|
|
92
|
+
## Tests
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
PYTHONPATH=src python -m unittest discover -s tests
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
The fixtures are fictional ("GARAGE EXEMPLE LYON", example.com).
|
|
99
|
+
|
|
100
|
+
## License
|
|
101
|
+
|
|
102
|
+
MIT
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# La Centrale Pros Scraper
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/lacentrale-pros-scraper/)
|
|
4
|
+
[](https://pypi.org/project/lacentrale-pros-scraper/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Scrape [La Centrale's](https://www.lacentrale.fr) dealer directory and dealer pages ([pros.lacentrale.fr](https://pros.lacentrale.fr)) into structured data. List every car dealer in a **city, department, region or brand network**, open a dealer for its **SIREN and SIRET, founding year, verified badge, address with coordinates, opening hours, website and stock size**, and pull **every vehicle in its stock**: make, model, version, year, mileage, fuel, gearbox, price, monthly price, La Centrale's **deal rating** (very good, good, fair, bad), warranty and listing dates. JSON or CSV.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install lacentrale-pros-scraper
|
|
11
|
+
export SCRAPEDO_TOKEN=your_token
|
|
12
|
+
lacentrale-pros --city lyon -f csv -o lyon_dealers.csv # directory cards
|
|
13
|
+
lacentrale-pros --city lyon --details -o lyon_dealers.json # + SIREN/SIRET, hours, website
|
|
14
|
+
lacentrale-pros --city lyon --stock -f csv -o lyon_cars.csv # every car of every dealer
|
|
15
|
+
lacentrale-pros C045088 --stock -o stock.json # one dealer
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Why plain requests fail
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
import requests
|
|
22
|
+
|
|
23
|
+
r = requests.get("https://pros.lacentrale.fr/C045088")
|
|
24
|
+
print(r.status_code) # 403
|
|
25
|
+
print(r.text[:200]) # ... Please enable JS and disable any ad blocker ... dd={'rt':'c' ...
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
lacentrale.fr is protected by DataDome, which answers a script with `403` and a JavaScript challenge. This package sends each request through [Scrape.do](https://scrape.do/?utm_source=github&utm_medium=readme&utm_campaign=lacentrale-pros-scraper), which returns the real page. A plain request is enough, no extra parameters: **1 credit per page**.
|
|
29
|
+
|
|
30
|
+
## Where the data is
|
|
31
|
+
|
|
32
|
+
* **Directory** (`www.lacentrale.fr/annuaire/garages-villes/<city>`, `garages-departements/<dept>`, `garages-regions/<region>`, `marques/<brand>`, `?page=N&perPage=20`): server-rendered cards with the dealer name, postcode and city, number of ads, a "Marques" badge for franchised dealers, and the dealer reference (`C` + 6 digits) in `data-tracking-meta`. Some dealers link to their own website instead of their La Centrale page, but the reference is still there.
|
|
33
|
+
* **Dealer page** (`pros.lacentrale.fr/C045088`):
|
|
34
|
+
* an `AutoDealer` JSON-LD block with name, address and coordinates. Its `openingHoursSpecification` is a **JSON string inside the JSON**: decode it a second time.
|
|
35
|
+
* an identity card ("Vendeur vérifié", "Date de création", "Stock"; big groups show "Stock national" instead).
|
|
36
|
+
* a footer with **SIREN / SIRET** and the dealer's website.
|
|
37
|
+
* `window.__PRELOADED_STATE_LISTING__`, whose `search.hits` is the first page of stock (9 vehicles) and `search.total` the stock size.
|
|
38
|
+
* **More stock**: `pros.lacentrale.fr/C045088/index?options=&page=2&vertical=car`, same state object.
|
|
39
|
+
|
|
40
|
+
In a live test, Lyon had 17 dealers in the directory and 228 cars in stock, scraped for 37 credits. 42 of those cars carried La Centrale's "very good deal" badge and 52 "good deal"; the median price was €28,940.
|
|
41
|
+
|
|
42
|
+
## Python
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from lacentrale_pros import LaCentralePros
|
|
46
|
+
|
|
47
|
+
lc = LaCentralePros(token="YOUR_SCRAPEDO_TOKEN") # or set SCRAPEDO_TOKEN
|
|
48
|
+
|
|
49
|
+
for card in lc.directory("city", "Lyon"):
|
|
50
|
+
print(card.ref, card.name, card.postcode, card.ads)
|
|
51
|
+
|
|
52
|
+
dealer, cars = lc.dealer_with_stock("C045088")
|
|
53
|
+
print(dealer.name, dealer.siret, dealer.founded, dealer.opening_hours)
|
|
54
|
+
for car in cars:
|
|
55
|
+
print(car.make, car.model, car.year, car.mileage, car.price, car.deal)
|
|
56
|
+
print(lc.credits_used)
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Fields
|
|
60
|
+
|
|
61
|
+
**Directory card:** `ref`, `name`, `postcode`, `city`, `ads`, `franchise`, `url`, `own_site`.
|
|
62
|
+
|
|
63
|
+
**Dealer:** `ref`, `url`, `name`, `dealer_type` (`CONCESSIONNAIRE`, `CENTRE_MULTIMARQUES`, ...), `verified`, `founded`, `stock`, `stock_national`, `street`, `postcode`, `city`, `country`, `latitude`, `longitude`, `opening_hours`, `siren`, `siret`, `website`, `phone`.
|
|
64
|
+
|
|
65
|
+
**Vehicle:** `reference`, `url`, `make`, `model`, `version`, `year`, `mileage`, `energy`, `gearbox`, `category`, `color`, `doors`, `price`, `monthly_from`, `deal` (`VERY_GOOD_DEAL`, `GOOD_DEAL`, `EQUITABLE_DEAL`, `BAD_DEAL`, `NOT_COMPUTED`), `warranty_months`, `first_online`, `last_update`, `department`, `pictures`, `photo`, `is_new`, `dealer_ref`, `dealer_name`.
|
|
66
|
+
|
|
67
|
+
## Use it responsibly
|
|
68
|
+
|
|
69
|
+
This is business data published by professional sellers. Some dealers are sole traders, so treat names and phone numbers as personal data under the GDPR: use them for legitimate purposes, honour objection requests, and respect La Centrale's terms. Don't republish vehicle photos. This is not legal advice.
|
|
70
|
+
|
|
71
|
+
## Tests
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
PYTHONPATH=src python -m unittest discover -s tests
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
The fixtures are fictional ("GARAGE EXEMPLE LYON", example.com).
|
|
78
|
+
|
|
79
|
+
## License
|
|
80
|
+
|
|
81
|
+
MIT
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Every car rated "very good deal" by La Centrale among a city's dealers, cheapest first.
|
|
2
|
+
|
|
3
|
+
SCRAPEDO_TOKEN=... python examples/good_deals.py lyon
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import csv
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from lacentrale_pros import LaCentralePros
|
|
10
|
+
|
|
11
|
+
city = sys.argv[1] if len(sys.argv) > 1 else "lyon"
|
|
12
|
+
lc = LaCentralePros()
|
|
13
|
+
|
|
14
|
+
deals = []
|
|
15
|
+
for card in lc.directory("city", city):
|
|
16
|
+
if not card.ads:
|
|
17
|
+
continue
|
|
18
|
+
dealer, cars = lc.dealer_with_stock(card.ref)
|
|
19
|
+
deals += [c for c in cars if c.deal == "VERY_GOOD_DEAL"]
|
|
20
|
+
print(f"{dealer.name}: {len(cars)} cars", file=sys.stderr)
|
|
21
|
+
|
|
22
|
+
deals.sort(key=lambda c: c.price or 0)
|
|
23
|
+
with open(f"{city}_very_good_deals.csv", "w", newline="", encoding="utf-8") as fh:
|
|
24
|
+
writer = csv.DictWriter(fh, fieldnames=["make", "model", "version", "year", "mileage", "price", "dealer_name", "url"],
|
|
25
|
+
extrasaction="ignore")
|
|
26
|
+
writer.writeheader()
|
|
27
|
+
writer.writerows(c.to_dict() for c in deals)
|
|
28
|
+
print(f"{len(deals)} very good deals, {lc.credits_used} credits", file=sys.stderr)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "lacentrale-pros-scraper"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Scrape La Centrale's dealer directory (pros.lacentrale.fr): French car dealers, SIREN/SIRET, opening hours and every vehicle in stock with Python."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Buğrahan Saka" }]
|
|
14
|
+
keywords = ["lacentrale", "la centrale", "pros.lacentrale.fr", "car dealers", "used cars", "france", "web scraping", "scraper", "scrapedo"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
21
|
+
]
|
|
22
|
+
dependencies = ["requests>=2.25"]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/bugrahansss/lacentrale-pros-scraper"
|
|
26
|
+
Source = "https://github.com/bugrahansss/lacentrale-pros-scraper"
|
|
27
|
+
Issues = "https://github.com/bugrahansss/lacentrale-pros-scraper/issues"
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
lacentrale-pros = "lacentrale_pros.cli:main"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages.find]
|
|
33
|
+
where = ["src"]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Scrape La Centrale's dealer directory (pros.lacentrale.fr): dealers, profiles and stock."""
|
|
2
|
+
|
|
3
|
+
from .client import LaCentralePros, ScrapeError
|
|
4
|
+
from .parser import (
|
|
5
|
+
Dealer,
|
|
6
|
+
DealerCard,
|
|
7
|
+
DirectoryPage,
|
|
8
|
+
StockPage,
|
|
9
|
+
Vehicle,
|
|
10
|
+
dealer_ref,
|
|
11
|
+
dealer_url,
|
|
12
|
+
directory_url,
|
|
13
|
+
opening_hours,
|
|
14
|
+
parse_dealer,
|
|
15
|
+
parse_directory,
|
|
16
|
+
parse_stock,
|
|
17
|
+
stock_url,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
__version__ = "0.1.0"
|
|
21
|
+
__all__ = [
|
|
22
|
+
"LaCentralePros", "ScrapeError", "Dealer", "DealerCard", "DirectoryPage", "StockPage", "Vehicle",
|
|
23
|
+
"dealer_ref", "dealer_url", "directory_url", "opening_hours", "parse_dealer", "parse_directory",
|
|
24
|
+
"parse_stock", "stock_url", "__version__",
|
|
25
|
+
]
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""Command line entry point.
|
|
2
|
+
|
|
3
|
+
lacentrale-pros --city lyon -f csv -o lyon_dealers.csv
|
|
4
|
+
lacentrale-pros --city lyon --details -o lyon_dealers.json
|
|
5
|
+
lacentrale-pros --city lyon --stock -f csv -o lyon_cars.csv
|
|
6
|
+
lacentrale-pros C045088 --stock -o stock.json
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import csv
|
|
13
|
+
import json
|
|
14
|
+
import sys
|
|
15
|
+
from typing import Any, List, Optional
|
|
16
|
+
|
|
17
|
+
from . import __version__
|
|
18
|
+
from .client import TOKEN_ENV, LaCentralePros, ScrapeError
|
|
19
|
+
from .parser import CARD_FIELDS, DEALER_FIELDS, VEHICLE_FIELDS
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
23
|
+
p = argparse.ArgumentParser(prog="lacentrale-pros",
|
|
24
|
+
description="Scrape La Centrale's dealer directory, dealer profiles and dealer stock.")
|
|
25
|
+
p.add_argument("dealers", nargs="*", help="Dealer references or URLs (C045088, https://pros.lacentrale.fr/C045088)")
|
|
26
|
+
p.add_argument("--city", action="append", default=[], help="Directory by city (lyon, paris, ...)")
|
|
27
|
+
p.add_argument("--department", action="append", default=[], help="Directory by department (rhone, nord, ...)")
|
|
28
|
+
p.add_argument("--region", action="append", default=[], help="Directory by region (bretagne, ile-de-france, ...)")
|
|
29
|
+
p.add_argument("--brand", action="append", default=[], help="Directory by brand (peugeot, bmw, ...)")
|
|
30
|
+
p.add_argument("--pages", type=int, default=0, help="Directory pages per search, 20 dealers each (default: all)")
|
|
31
|
+
p.add_argument("--details", action="store_true", help="Open every dealer page: SIREN/SIRET, founding year, hours, website")
|
|
32
|
+
p.add_argument("--stock", action="store_true", help="Output every vehicle in each dealer's stock (9 per page)")
|
|
33
|
+
p.add_argument("--limit", type=int, default=0, help="Stop after this many dealers")
|
|
34
|
+
p.add_argument("-t", "--token", help=f"Scrape.do API token (default: ${TOKEN_ENV})")
|
|
35
|
+
p.add_argument("-f", "--format", choices=("json", "csv"), default="json")
|
|
36
|
+
p.add_argument("-o", "--output", help="Write to this file instead of stdout")
|
|
37
|
+
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
38
|
+
return p
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _write(rows: List[dict], fields: List[str], fmt: str, output: Optional[str]) -> None:
|
|
42
|
+
out = open(output, "w", newline="", encoding="utf-8") if output else sys.stdout
|
|
43
|
+
try:
|
|
44
|
+
if fmt == "json":
|
|
45
|
+
json.dump(rows, out, ensure_ascii=False, indent=2)
|
|
46
|
+
out.write("\n")
|
|
47
|
+
else:
|
|
48
|
+
writer = csv.DictWriter(out, fieldnames=fields, extrasaction="ignore")
|
|
49
|
+
writer.writeheader()
|
|
50
|
+
for row in rows:
|
|
51
|
+
writer.writerow({k: json.dumps(v, ensure_ascii=False) if isinstance(v, (list, dict)) else v
|
|
52
|
+
for k, v in row.items()})
|
|
53
|
+
finally:
|
|
54
|
+
if output:
|
|
55
|
+
out.close()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
59
|
+
args = build_parser().parse_args(argv)
|
|
60
|
+
searches = [("city", v) for v in args.city] + [("department", v) for v in args.department] + \
|
|
61
|
+
[("region", v) for v in args.region] + [("brand", v) for v in args.brand]
|
|
62
|
+
if not args.dealers and not searches:
|
|
63
|
+
build_parser().print_usage(sys.stderr)
|
|
64
|
+
print("Give dealer references or a --city/--department/--region/--brand.", file=sys.stderr)
|
|
65
|
+
return 2
|
|
66
|
+
try:
|
|
67
|
+
lc = LaCentralePros(token=args.token)
|
|
68
|
+
except ValueError as exc:
|
|
69
|
+
print(exc, file=sys.stderr)
|
|
70
|
+
return 2
|
|
71
|
+
|
|
72
|
+
cards: List[Any] = []
|
|
73
|
+
refs: List[str] = list(args.dealers)
|
|
74
|
+
failures = 0
|
|
75
|
+
for kind, name in searches:
|
|
76
|
+
try:
|
|
77
|
+
found = list(lc.directory(kind, name, args.pages))
|
|
78
|
+
except ScrapeError as exc:
|
|
79
|
+
failures += 1
|
|
80
|
+
print(f"fail {kind} {name}: {exc}", file=sys.stderr)
|
|
81
|
+
continue
|
|
82
|
+
print(f"{kind} {name}: {len(found)} dealers", file=sys.stderr)
|
|
83
|
+
cards.extend(found)
|
|
84
|
+
refs.extend(c.ref for c in found)
|
|
85
|
+
refs = list(dict.fromkeys(refs))
|
|
86
|
+
if args.limit:
|
|
87
|
+
refs = refs[: args.limit]
|
|
88
|
+
cards = [c for c in cards if c.ref in refs]
|
|
89
|
+
|
|
90
|
+
if not args.details and not args.stock:
|
|
91
|
+
rows, fields = [c.to_dict() for c in cards], CARD_FIELDS
|
|
92
|
+
if args.dealers:
|
|
93
|
+
for ref in args.dealers:
|
|
94
|
+
try:
|
|
95
|
+
rows.append(lc.dealer(ref).to_dict())
|
|
96
|
+
fields = DEALER_FIELDS
|
|
97
|
+
except (ScrapeError, ValueError) as exc:
|
|
98
|
+
failures += 1
|
|
99
|
+
print(f"fail {ref}: {exc}", file=sys.stderr)
|
|
100
|
+
else:
|
|
101
|
+
rows, fields = [], VEHICLE_FIELDS if args.stock else DEALER_FIELDS
|
|
102
|
+
for ref in refs:
|
|
103
|
+
try:
|
|
104
|
+
if args.stock:
|
|
105
|
+
dealer, vehicles = lc.dealer_with_stock(ref)
|
|
106
|
+
rows.extend(v.to_dict() for v in vehicles)
|
|
107
|
+
print(f"{ref} {dealer.name}: {len(vehicles)} vehicles", file=sys.stderr)
|
|
108
|
+
else:
|
|
109
|
+
rows.append(lc.dealer(ref).to_dict())
|
|
110
|
+
except (ScrapeError, ValueError) as exc:
|
|
111
|
+
failures += 1
|
|
112
|
+
print(f"fail {ref}: {exc}", file=sys.stderr)
|
|
113
|
+
|
|
114
|
+
_write(rows, fields, args.format, args.output)
|
|
115
|
+
print(f"saved {len(rows)} rows ({lc.credits_used} credits)", file=sys.stderr)
|
|
116
|
+
return 1 if failures and not rows else 0
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
if __name__ == "__main__":
|
|
120
|
+
sys.exit(main())
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Fetch La Centrale's dealer directory and dealer pages through the Scrape.do API.
|
|
2
|
+
|
|
3
|
+
lacentrale.fr sits behind DataDome: a plain ``requests.get`` from a script gets
|
|
4
|
+
``403`` and a "Please enable JS and disable any ad blocker" page. A plain
|
|
5
|
+
Scrape.do request (no extra parameters) returns the real server-rendered HTML.
|
|
6
|
+
Each page costs 1 Scrape.do credit.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import time
|
|
13
|
+
from typing import Iterator, List, Optional, Tuple
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
from .parser import (
|
|
18
|
+
Dealer,
|
|
19
|
+
DealerCard,
|
|
20
|
+
Vehicle,
|
|
21
|
+
dealer_ref,
|
|
22
|
+
dealer_url,
|
|
23
|
+
directory_url,
|
|
24
|
+
parse_dealer,
|
|
25
|
+
parse_directory,
|
|
26
|
+
parse_stock,
|
|
27
|
+
stock_url,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
API_URL = "https://api.scrape.do/"
|
|
31
|
+
TOKEN_ENV = "SCRAPEDO_TOKEN"
|
|
32
|
+
BLOCK_MARKERS = ("Please enable JS and disable any ad blocker", "captcha-delivery.com")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ScrapeError(RuntimeError):
|
|
36
|
+
def __init__(self, message: str, status_code: Optional[int] = None):
|
|
37
|
+
super().__init__(message)
|
|
38
|
+
self.status_code = status_code
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class LaCentralePros:
|
|
42
|
+
"""Read La Centrale's dealer directory, dealer profiles and every vehicle a dealer has in stock.
|
|
43
|
+
|
|
44
|
+
>>> lc = LaCentralePros(token="YOUR_SCRAPEDO_TOKEN")
|
|
45
|
+
>>> for card in lc.directory("city", "Lyon"):
|
|
46
|
+
... print(card.ref, card.name, card.ads)
|
|
47
|
+
>>> dealer, cars = lc.dealer_with_stock("C045088")
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
def __init__(self, token: Optional[str] = None, timeout: float = 60.0, retries: int = 2,
|
|
51
|
+
session: Optional[requests.Session] = None):
|
|
52
|
+
self.token = token or os.environ.get(TOKEN_ENV, "")
|
|
53
|
+
if not self.token:
|
|
54
|
+
raise ValueError(f"A Scrape.do API token is required. Pass token=... or set {TOKEN_ENV}.")
|
|
55
|
+
self.timeout = timeout
|
|
56
|
+
self.retries = max(0, retries)
|
|
57
|
+
self.session = session or requests.Session()
|
|
58
|
+
self.requests_made = 0
|
|
59
|
+
|
|
60
|
+
def fetch(self, url: str) -> str:
|
|
61
|
+
params = {"token": self.token, "url": url}
|
|
62
|
+
last: Optional[ScrapeError] = None
|
|
63
|
+
for attempt in range(self.retries + 1):
|
|
64
|
+
try:
|
|
65
|
+
response = self.session.get(API_URL, params=params, timeout=self.timeout)
|
|
66
|
+
except requests.RequestException as exc:
|
|
67
|
+
last = ScrapeError(f"Request failed: {exc}")
|
|
68
|
+
else:
|
|
69
|
+
if response.status_code == 200:
|
|
70
|
+
self.requests_made += 1
|
|
71
|
+
if not any(marker in response.text for marker in BLOCK_MARKERS):
|
|
72
|
+
return response.text
|
|
73
|
+
last = ScrapeError(f"Blocked page for {url} (DataDome)", 200)
|
|
74
|
+
else:
|
|
75
|
+
last = ScrapeError(f"Request for {url} failed with status {response.status_code}",
|
|
76
|
+
response.status_code)
|
|
77
|
+
if response.status_code in (400, 401, 404, 410):
|
|
78
|
+
break
|
|
79
|
+
if attempt < self.retries:
|
|
80
|
+
time.sleep(2 * (attempt + 1))
|
|
81
|
+
assert last is not None
|
|
82
|
+
raise last
|
|
83
|
+
|
|
84
|
+
# ---- directory -----------------------------------------------------------
|
|
85
|
+
|
|
86
|
+
def directory(self, kind: str, name: str, pages: int = 0) -> Iterator[DealerCard]:
|
|
87
|
+
"""Dealers listed for a ``city``, ``department``, ``region`` or ``brand`` (20 per page).
|
|
88
|
+
|
|
89
|
+
``pages=0`` follows every page.
|
|
90
|
+
"""
|
|
91
|
+
seen = set()
|
|
92
|
+
page, last_page = 0, None
|
|
93
|
+
while True:
|
|
94
|
+
if last_page is not None and page >= last_page:
|
|
95
|
+
break
|
|
96
|
+
if pages and page >= pages:
|
|
97
|
+
break
|
|
98
|
+
result = parse_directory(self.fetch(directory_url(kind, name, page)))
|
|
99
|
+
last_page = result.pages
|
|
100
|
+
fresh = [c for c in result.dealers if c.ref not in seen]
|
|
101
|
+
if not fresh:
|
|
102
|
+
break
|
|
103
|
+
for card in fresh:
|
|
104
|
+
seen.add(card.ref)
|
|
105
|
+
yield card
|
|
106
|
+
page += 1
|
|
107
|
+
|
|
108
|
+
# ---- dealer --------------------------------------------------------------
|
|
109
|
+
|
|
110
|
+
def dealer(self, ref: str) -> Dealer:
|
|
111
|
+
ref = dealer_ref(ref)
|
|
112
|
+
dealer = parse_dealer(self.fetch(dealer_url(ref)), ref)
|
|
113
|
+
if dealer is None:
|
|
114
|
+
raise ScrapeError(f"No dealer data on {dealer_url(ref)}")
|
|
115
|
+
return dealer
|
|
116
|
+
|
|
117
|
+
def stock(self, ref: str, vertical: str = "car", pages: int = 0) -> Iterator[Vehicle]:
|
|
118
|
+
"""Every vehicle a dealer lists (9 per page). ``pages=0`` follows every page."""
|
|
119
|
+
ref = dealer_ref(ref)
|
|
120
|
+
seen = set()
|
|
121
|
+
page, last_page = 1, None
|
|
122
|
+
while True:
|
|
123
|
+
if (last_page is not None and page > last_page) or (pages and page > pages):
|
|
124
|
+
break
|
|
125
|
+
result = parse_stock(self.fetch(stock_url(ref, page, vertical)), ref)
|
|
126
|
+
if result is None:
|
|
127
|
+
raise ScrapeError(f"No stock data on {stock_url(ref, page, vertical)}")
|
|
128
|
+
last_page = result.pages
|
|
129
|
+
fresh = [v for v in result.vehicles if v.reference not in seen]
|
|
130
|
+
if not fresh:
|
|
131
|
+
break
|
|
132
|
+
for vehicle in fresh:
|
|
133
|
+
seen.add(vehicle.reference)
|
|
134
|
+
yield vehicle
|
|
135
|
+
page += 1
|
|
136
|
+
|
|
137
|
+
def dealer_with_stock(self, ref: str, pages: int = 0) -> Tuple[Dealer, List[Vehicle]]:
|
|
138
|
+
"""Dealer profile plus its car stock. The dealer page doubles as stock page 1."""
|
|
139
|
+
ref = dealer_ref(ref)
|
|
140
|
+
html = self.fetch(dealer_url(ref))
|
|
141
|
+
dealer = parse_dealer(html, ref)
|
|
142
|
+
if dealer is None:
|
|
143
|
+
raise ScrapeError(f"No dealer data on {dealer_url(ref)}")
|
|
144
|
+
first = parse_stock(html, ref)
|
|
145
|
+
vehicles: List[Vehicle] = list(first.vehicles) if first else []
|
|
146
|
+
seen = {v.reference for v in vehicles}
|
|
147
|
+
last_page = first.pages if first else 1
|
|
148
|
+
page = 2
|
|
149
|
+
while page <= last_page and not (pages and page > pages):
|
|
150
|
+
result = parse_stock(self.fetch(stock_url(ref, page)), ref)
|
|
151
|
+
if result is None:
|
|
152
|
+
break
|
|
153
|
+
fresh = [v for v in result.vehicles if v.reference not in seen]
|
|
154
|
+
if not fresh:
|
|
155
|
+
break
|
|
156
|
+
for v in fresh:
|
|
157
|
+
seen.add(v.reference)
|
|
158
|
+
vehicles.append(v)
|
|
159
|
+
page += 1
|
|
160
|
+
return dealer, vehicles
|
|
161
|
+
|
|
162
|
+
@property
|
|
163
|
+
def credits_used(self) -> int:
|
|
164
|
+
return self.requests_made
|