discount-nepal 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- discount_nepal-0.1.0/LICENSE +21 -0
- discount_nepal-0.1.0/PKG-INFO +59 -0
- discount_nepal-0.1.0/README.md +39 -0
- discount_nepal-0.1.0/discount_nepal/__init__.py +5 -0
- discount_nepal-0.1.0/discount_nepal/app.py +109 -0
- discount_nepal-0.1.0/discount_nepal/scrapers/__init__.py +4 -0
- discount_nepal-0.1.0/discount_nepal/scrapers/aggregate.py +76 -0
- discount_nepal-0.1.0/discount_nepal/scrapers/dolpo.py +142 -0
- discount_nepal-0.1.0/discount_nepal/scrapers/sonamgear.py +121 -0
- discount_nepal-0.1.0/discount_nepal/templates/index.html +123 -0
- discount_nepal-0.1.0/discount_nepal.egg-info/PKG-INFO +59 -0
- discount_nepal-0.1.0/discount_nepal.egg-info/SOURCES.txt +16 -0
- discount_nepal-0.1.0/discount_nepal.egg-info/dependency_links.txt +1 -0
- discount_nepal-0.1.0/discount_nepal.egg-info/entry_points.txt +2 -0
- discount_nepal-0.1.0/discount_nepal.egg-info/requires.txt +2 -0
- discount_nepal-0.1.0/discount_nepal.egg-info/top_level.txt +1 -0
- discount_nepal-0.1.0/pyproject.toml +35 -0
- discount_nepal-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Discount Nepal
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: discount-nepal
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Web app that lists discounted trekking gear from Nepali outdoor stores (Dolpo, Sonam Gear).
|
|
5
|
+
Author: Discount Nepal
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: scraper,discounts,trekking,nepal,ecommerce
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Environment :: Web Environment
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
13
|
+
Classifier: Topic :: Office/Business
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: requests>=2.31
|
|
18
|
+
Requires-Dist: beautifulsoup4>=4.12
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# Discount Nepal
|
|
22
|
+
|
|
23
|
+
A small web app that lists discounted trekking and outdoor gear from Nepali
|
|
24
|
+
stores. It scrapes **Dolpo** and **Sonam Gear** on demand and serves the results
|
|
25
|
+
through a JSON API, so the page always shows current prices.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install discount-nepal
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Run
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
discount-nepal
|
|
37
|
+
# or: discount-nepal --host 0.0.0.0 --port 8000
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Then open <http://127.0.0.1:8000>.
|
|
41
|
+
|
|
42
|
+
## How it works
|
|
43
|
+
|
|
44
|
+
- `GET /` - the product page
|
|
45
|
+
- `GET /api/products` - cached products (120s TTL)
|
|
46
|
+
- `GET /api/products?refresh=1` - forces a fresh scrape
|
|
47
|
+
|
|
48
|
+
Dolpo and Sonam Gear are scraped concurrently, so a full refresh takes about
|
|
49
|
+
5-6 seconds. The cache is warmed in the background at startup.
|
|
50
|
+
|
|
51
|
+
## Adding another store
|
|
52
|
+
|
|
53
|
+
Add a scraper under `discount_nepal/scrapers/` exposing a class with
|
|
54
|
+
`scrape()` and `get_results()`, then include it in
|
|
55
|
+
`discount_nepal.app.scrape_products()`.
|
|
56
|
+
|
|
57
|
+
## License
|
|
58
|
+
|
|
59
|
+
MIT
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Discount Nepal
|
|
2
|
+
|
|
3
|
+
A small web app that lists discounted trekking and outdoor gear from Nepali
|
|
4
|
+
stores. It scrapes **Dolpo** and **Sonam Gear** on demand and serves the results
|
|
5
|
+
through a JSON API, so the page always shows current prices.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install discount-nepal
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Run
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
discount-nepal
|
|
17
|
+
# or: discount-nepal --host 0.0.0.0 --port 8000
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Then open <http://127.0.0.1:8000>.
|
|
21
|
+
|
|
22
|
+
## How it works
|
|
23
|
+
|
|
24
|
+
- `GET /` - the product page
|
|
25
|
+
- `GET /api/products` - cached products (120s TTL)
|
|
26
|
+
- `GET /api/products?refresh=1` - forces a fresh scrape
|
|
27
|
+
|
|
28
|
+
Dolpo and Sonam Gear are scraped concurrently, so a full refresh takes about
|
|
29
|
+
5-6 seconds. The cache is warmed in the background at startup.
|
|
30
|
+
|
|
31
|
+
## Adding another store
|
|
32
|
+
|
|
33
|
+
Add a scraper under `discount_nepal/scrapers/` exposing a class with
|
|
34
|
+
`scrape()` and `get_results()`, then include it in
|
|
35
|
+
`discount_nepal.app.scrape_products()`.
|
|
36
|
+
|
|
37
|
+
## License
|
|
38
|
+
|
|
39
|
+
MIT
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import threading
|
|
4
|
+
import time
|
|
5
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
6
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from urllib.parse import urlparse, parse_qs
|
|
9
|
+
|
|
10
|
+
from .scrapers.dolpo import DolpoScraper
|
|
11
|
+
from .scrapers.sonamgear import SonamGearScraper
|
|
12
|
+
|
|
13
|
+
TEMPLATE_PATH = Path(__file__).resolve().parent / "templates" / "index.html"
|
|
14
|
+
CACHE_TTL_SECONDS = 120
|
|
15
|
+
|
|
16
|
+
_lock = threading.Lock()
|
|
17
|
+
_cache = {"products": [], "fetched_at": 0.0, "ready": False}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def scrape_products():
|
|
21
|
+
"""Scrape both stores concurrently; total time is the slower store, not the sum."""
|
|
22
|
+
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
23
|
+
dolpo_future = pool.submit(DolpoScraper().scrape)
|
|
24
|
+
sonam_future = pool.submit(SonamGearScraper().scrape)
|
|
25
|
+
dolpo = dolpo_future.result()
|
|
26
|
+
sonam = sonam_future.result()
|
|
27
|
+
return dolpo + sonam
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def refresh_cache():
|
|
31
|
+
products = scrape_products()
|
|
32
|
+
_cache.update(products=products, fetched_at=time.time(), ready=True)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_products(force=False):
|
|
36
|
+
with _lock:
|
|
37
|
+
fresh = _cache["ready"] and time.time() - _cache["fetched_at"] < CACHE_TTL_SECONDS
|
|
38
|
+
if fresh and not force:
|
|
39
|
+
return _cache
|
|
40
|
+
refresh_cache()
|
|
41
|
+
return _cache
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Handler(BaseHTTPRequestHandler):
|
|
45
|
+
def do_GET(self):
|
|
46
|
+
parsed = urlparse(self.path)
|
|
47
|
+
if parsed.path == "/api/products":
|
|
48
|
+
force = parse_qs(parsed.query).get("refresh", ["0"])[0] == "1"
|
|
49
|
+
self.send_json(get_products(force=force))
|
|
50
|
+
return
|
|
51
|
+
if parsed.path in ("/", "/index.html"):
|
|
52
|
+
self.send_file(TEMPLATE_PATH, "text/html; charset=utf-8")
|
|
53
|
+
return
|
|
54
|
+
self.send_error(404)
|
|
55
|
+
|
|
56
|
+
def send_json(self, payload):
|
|
57
|
+
body = json.dumps({
|
|
58
|
+
"products": payload["products"],
|
|
59
|
+
"fetched_at": payload["fetched_at"],
|
|
60
|
+
}, ensure_ascii=False).encode("utf-8")
|
|
61
|
+
self.send_response(200)
|
|
62
|
+
self.send_header("Content-Type", "application/json; charset=utf-8")
|
|
63
|
+
self.send_header("Cache-Control", "no-store")
|
|
64
|
+
self.send_header("Content-Length", str(len(body)))
|
|
65
|
+
self.end_headers()
|
|
66
|
+
self.wfile.write(body)
|
|
67
|
+
|
|
68
|
+
def send_file(self, path, content_type):
|
|
69
|
+
if not path.exists():
|
|
70
|
+
self.send_error(500, "index.html missing")
|
|
71
|
+
return
|
|
72
|
+
body = path.read_bytes()
|
|
73
|
+
self.send_response(200)
|
|
74
|
+
self.send_header("Content-Type", content_type)
|
|
75
|
+
self.send_header("Cache-Control", "no-store")
|
|
76
|
+
self.send_header("Content-Length", str(len(body)))
|
|
77
|
+
self.end_headers()
|
|
78
|
+
self.wfile.write(body)
|
|
79
|
+
|
|
80
|
+
def log_message(self, *args):
|
|
81
|
+
pass
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def serve(host="127.0.0.1", port=8000):
|
|
85
|
+
# Warm the cache in the background so the first page load is instant.
|
|
86
|
+
threading.Thread(target=refresh_cache, daemon=True).start()
|
|
87
|
+
server = ThreadingHTTPServer((host, port), Handler)
|
|
88
|
+
print(f"Discount Nepal running at http://{host}:{port}")
|
|
89
|
+
try:
|
|
90
|
+
server.serve_forever()
|
|
91
|
+
except KeyboardInterrupt:
|
|
92
|
+
print("\nShutting down.")
|
|
93
|
+
finally:
|
|
94
|
+
server.server_close()
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def main():
|
|
98
|
+
parser = argparse.ArgumentParser(
|
|
99
|
+
prog="discount-nepal",
|
|
100
|
+
description="Web app listing discounted trekking gear from Dolpo and Sonam Gear.",
|
|
101
|
+
)
|
|
102
|
+
parser.add_argument("--host", default="127.0.0.1")
|
|
103
|
+
parser.add_argument("--port", type=int, default=8000)
|
|
104
|
+
args = parser.parse_args()
|
|
105
|
+
serve(args.host, args.port)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
if __name__ == "__main__":
|
|
109
|
+
main()
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import requests
|
|
4
|
+
from bs4 import BeautifulSoup
|
|
5
|
+
from urllib.parse import urljoin
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
|
|
8
|
+
from .dolpo import DolpoScraper
|
|
9
|
+
from .sonamgear import SonamGearScraper
|
|
10
|
+
|
|
11
|
+
EVEREST_URL = 'https://everestoutfit.com/shop/'
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class NepaliGearScraper:
|
|
15
|
+
def __init__(self):
|
|
16
|
+
self.headers = {
|
|
17
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
18
|
+
}
|
|
19
|
+
self.results = []
|
|
20
|
+
|
|
21
|
+
def get_results(self):
|
|
22
|
+
return self.results
|
|
23
|
+
|
|
24
|
+
def scrape_dolpo(self):
|
|
25
|
+
scraper = DolpoScraper()
|
|
26
|
+
scraper.scrape()
|
|
27
|
+
self.results.extend(scraper.get_results())
|
|
28
|
+
|
|
29
|
+
def scrape_sonam(self):
|
|
30
|
+
scraper = SonamGearScraper()
|
|
31
|
+
scraper.scrape()
|
|
32
|
+
self.results.extend(scraper.get_results())
|
|
33
|
+
|
|
34
|
+
def scrape_everest(self):
|
|
35
|
+
try:
|
|
36
|
+
response = requests.get(EVEREST_URL, headers=self.headers, timeout=15)
|
|
37
|
+
response.raise_for_status()
|
|
38
|
+
soup = BeautifulSoup(response.content, 'html.parser')
|
|
39
|
+
products = soup.select('div.rtwpvg-product.product') or soup.find_all('div', class_='product-item')
|
|
40
|
+
|
|
41
|
+
for product in products:
|
|
42
|
+
name_tag = product.select_one('h2.woocommerce-loop-product__title a') or product.find('h2', class_='product-name')
|
|
43
|
+
price_tag = product.find('span', class_='woocommerce-Price-amount')
|
|
44
|
+
link = product.select_one('h2.woocommerce-loop-product__title a') or product.find('a', href=True)
|
|
45
|
+
image_tag = product.find('img')
|
|
46
|
+
price = price_tag.get_text(strip=True) if price_tag else 'N/A'
|
|
47
|
+
|
|
48
|
+
self.results.append({
|
|
49
|
+
'website': 'Everest Outfit',
|
|
50
|
+
'name': name_tag.get_text(strip=True) if name_tag else 'N/A',
|
|
51
|
+
'regular_price': price,
|
|
52
|
+
'sale_price': price,
|
|
53
|
+
'url': urljoin(EVEREST_URL, link['href']) if link and link.has_attr('href') else 'N/A',
|
|
54
|
+
'image_url': urljoin(EVEREST_URL, image_tag.get('src')) if image_tag and image_tag.get('src') else 'N/A',
|
|
55
|
+
'timestamp': datetime.now().isoformat()
|
|
56
|
+
})
|
|
57
|
+
except Exception as e:
|
|
58
|
+
print(f"Error scraping Everest Outfit: {str(e)}")
|
|
59
|
+
|
|
60
|
+
def scrape_all(self):
|
|
61
|
+
self.results = []
|
|
62
|
+
self.scrape_dolpo()
|
|
63
|
+
self.scrape_sonam()
|
|
64
|
+
self.scrape_everest()
|
|
65
|
+
return self.results
|
|
66
|
+
|
|
67
|
+
def save_results(self, filename='sale_products.json'):
|
|
68
|
+
with open(filename, 'w', encoding='utf-8') as f:
|
|
69
|
+
json.dump(self.results, f, ensure_ascii=False, indent=2)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
if __name__ == "__main__":
|
|
73
|
+
scraper = NepaliGearScraper()
|
|
74
|
+
results = scraper.scrape_all()
|
|
75
|
+
scraper.save_results()
|
|
76
|
+
print(f"Found {len(results)} products.")
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
from bs4 import BeautifulSoup
|
|
3
|
+
from urllib.parse import urljoin, urlparse
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
BASE_URL = 'https://dolpo.com.np/shop/?paged={page}'
|
|
8
|
+
MAX_PAGES = 20
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class DolpoScraper:
|
|
12
|
+
def __init__(self):
|
|
13
|
+
self.headers = {
|
|
14
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
15
|
+
}
|
|
16
|
+
self.results = []
|
|
17
|
+
|
|
18
|
+
def get_results(self):
|
|
19
|
+
return self.results
|
|
20
|
+
|
|
21
|
+
def _parse_price(self, container):
|
|
22
|
+
regular_price = sale_price = 'N/A'
|
|
23
|
+
price_wrapper = container.find('div', class_='price-wrapper') or container.find('span', class_='price')
|
|
24
|
+
if price_wrapper:
|
|
25
|
+
regular_tag = price_wrapper.find('del')
|
|
26
|
+
sale_tag = price_wrapper.find('ins')
|
|
27
|
+
if regular_tag and sale_tag:
|
|
28
|
+
regular_price = (regular_tag.find('span', class_='woocommerce-Price-amount') or regular_tag).get_text(strip=True)
|
|
29
|
+
sale_price = (sale_tag.find('span', class_='woocommerce-Price-amount') or sale_tag).get_text(strip=True)
|
|
30
|
+
elif sale_tag:
|
|
31
|
+
sale_price = (sale_tag.find('span', class_='woocommerce-Price-amount') or sale_tag).get_text(strip=True)
|
|
32
|
+
regular_price = sale_price
|
|
33
|
+
elif regular_tag:
|
|
34
|
+
regular_price = (regular_tag.find('span', class_='woocommerce-Price-amount') or regular_tag).get_text(strip=True)
|
|
35
|
+
sale_price = regular_price
|
|
36
|
+
else:
|
|
37
|
+
amount = price_wrapper.find('span', class_='woocommerce-Price-amount') or price_wrapper.find('bdi')
|
|
38
|
+
if amount:
|
|
39
|
+
text = amount.get_text(strip=True)
|
|
40
|
+
regular_price = sale_price = text
|
|
41
|
+
else:
|
|
42
|
+
text = price_wrapper.get_text(strip=True)
|
|
43
|
+
if text:
|
|
44
|
+
regular_price = sale_price = text
|
|
45
|
+
if price_wrapper is None and container.select_one('.price'):
|
|
46
|
+
text = container.select_one('.price').get_text(strip=True)
|
|
47
|
+
if text:
|
|
48
|
+
regular_price = sale_price = text
|
|
49
|
+
return regular_price, sale_price
|
|
50
|
+
|
|
51
|
+
def _parse_product(self, product):
|
|
52
|
+
# Modern layout
|
|
53
|
+
name = None
|
|
54
|
+
url = None
|
|
55
|
+
image_url = None
|
|
56
|
+
|
|
57
|
+
title_tag = product.select_one('a.title') or product.select_one('.card-product-info a')
|
|
58
|
+
if title_tag:
|
|
59
|
+
name = title_tag.get_text(strip=True)
|
|
60
|
+
url = title_tag.get('href')
|
|
61
|
+
|
|
62
|
+
if not name:
|
|
63
|
+
name_tag = product.find('p', class_='name') or product.find('h2') or product.find('h3') or product.find('h4')
|
|
64
|
+
name = name_tag.get_text(strip=True) if name_tag else 'N/A'
|
|
65
|
+
|
|
66
|
+
if not url:
|
|
67
|
+
link = product.find('a', href=True)
|
|
68
|
+
url = link['href'] if link else 'N/A'
|
|
69
|
+
|
|
70
|
+
image_tag = product.find('img')
|
|
71
|
+
if image_tag:
|
|
72
|
+
image_url = image_tag.get('data-src') or image_tag.get('src') or 'N/A'
|
|
73
|
+
|
|
74
|
+
if url and url != 'N/A' and not url.startswith('http'):
|
|
75
|
+
url = urljoin('https://dolpo.com.np', url)
|
|
76
|
+
if image_url and image_url != 'N/A' and not image_url.startswith('http'):
|
|
77
|
+
image_url = urljoin('https://dolpo.com.np', image_url)
|
|
78
|
+
|
|
79
|
+
regular_price, sale_price = self._parse_price(product)
|
|
80
|
+
|
|
81
|
+
return {
|
|
82
|
+
'website': 'Dolpo',
|
|
83
|
+
'name': name or 'N/A',
|
|
84
|
+
'regular_price': regular_price,
|
|
85
|
+
'sale_price': sale_price,
|
|
86
|
+
'url': url or 'N/A',
|
|
87
|
+
'image_url': image_url or 'N/A',
|
|
88
|
+
'timestamp': datetime.now().isoformat()
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
def scrape(self):
|
|
92
|
+
self.results = []
|
|
93
|
+
seen_urls = set()
|
|
94
|
+
|
|
95
|
+
for base in [BASE_URL]:
|
|
96
|
+
page = 1
|
|
97
|
+
while page <= MAX_PAGES:
|
|
98
|
+
url = base.format(page=page)
|
|
99
|
+
if url in seen_urls:
|
|
100
|
+
break
|
|
101
|
+
seen_urls.add(url)
|
|
102
|
+
try:
|
|
103
|
+
response = requests.get(url, headers=self.headers, timeout=15)
|
|
104
|
+
response.raise_for_status()
|
|
105
|
+
except requests.HTTPError as e:
|
|
106
|
+
if e.response is not None and e.response.status_code == 404:
|
|
107
|
+
break
|
|
108
|
+
print(f"Error fetching {url}: {str(e)}")
|
|
109
|
+
break
|
|
110
|
+
except Exception as e:
|
|
111
|
+
print(f"Error fetching {url}: {str(e)}")
|
|
112
|
+
break
|
|
113
|
+
|
|
114
|
+
soup = BeautifulSoup(response.content, 'html.parser')
|
|
115
|
+
products = soup.select('div.card-product')
|
|
116
|
+
if not products:
|
|
117
|
+
products = soup.find_all('div', class_='sale')
|
|
118
|
+
|
|
119
|
+
if not products:
|
|
120
|
+
break
|
|
121
|
+
|
|
122
|
+
for product in products:
|
|
123
|
+
try:
|
|
124
|
+
self.results.append(self._parse_product(product))
|
|
125
|
+
except Exception as e:
|
|
126
|
+
print(f"Error parsing product: {str(e)}")
|
|
127
|
+
continue
|
|
128
|
+
|
|
129
|
+
page += 1
|
|
130
|
+
|
|
131
|
+
return self.results
|
|
132
|
+
|
|
133
|
+
def save_results(self, filename='scraped_dolpo.json'):
|
|
134
|
+
with open(filename, 'w', encoding='utf-8') as f:
|
|
135
|
+
json.dump(self.results, f, ensure_ascii=False, indent=2)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
if __name__ == "__main__":
|
|
139
|
+
scraper = DolpoScraper()
|
|
140
|
+
scraper.scrape()
|
|
141
|
+
scraper.save_results()
|
|
142
|
+
print(f"\nFound {len(scraper.results)} products. Results saved to scraped_dolpo.json.")
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from urllib.parse import urljoin
|
|
5
|
+
|
|
6
|
+
import requests
|
|
7
|
+
from bs4 import BeautifulSoup
|
|
8
|
+
|
|
9
|
+
BASE_URL = 'https://sonamgear.com'
|
|
10
|
+
LIST_URL = BASE_URL + '/products?cat=1&n=1'
|
|
11
|
+
DETAIL_URL = BASE_URL + '/getproductdetail'
|
|
12
|
+
MAX_WORKERS = 12
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SonamGearScraper:
|
|
16
|
+
def __init__(self):
|
|
17
|
+
self.headers = {
|
|
18
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
19
|
+
'X-Requested-With': 'XMLHttpRequest',
|
|
20
|
+
'Referer': LIST_URL,
|
|
21
|
+
}
|
|
22
|
+
self.results = []
|
|
23
|
+
|
|
24
|
+
def get_results(self):
|
|
25
|
+
return self.results
|
|
26
|
+
|
|
27
|
+
def _get_product_ids(self):
|
|
28
|
+
response = requests.get(LIST_URL, headers=self.headers, timeout=20)
|
|
29
|
+
response.raise_for_status()
|
|
30
|
+
soup = BeautifulSoup(response.content, 'html.parser')
|
|
31
|
+
ids = []
|
|
32
|
+
for element in soup.select('.product-list'):
|
|
33
|
+
product_id = element.get('data-product_id')
|
|
34
|
+
if product_id and product_id not in ids:
|
|
35
|
+
ids.append(product_id)
|
|
36
|
+
return ids
|
|
37
|
+
|
|
38
|
+
def _fetch_product(self, product_id):
|
|
39
|
+
response = requests.post(
|
|
40
|
+
DETAIL_URL,
|
|
41
|
+
headers=self.headers,
|
|
42
|
+
data={'product_id': product_id},
|
|
43
|
+
timeout=20,
|
|
44
|
+
)
|
|
45
|
+
response.raise_for_status()
|
|
46
|
+
payload = response.json()
|
|
47
|
+
return BeautifulSoup(payload.get('view', ''), 'html.parser').find('div', class_='inner')
|
|
48
|
+
|
|
49
|
+
def _parse_product(self, product):
|
|
50
|
+
name_tag = product.find('h4', class_='mt-3 text-height')
|
|
51
|
+
name = name_tag.get_text(strip=True) if name_tag else 'N/A'
|
|
52
|
+
|
|
53
|
+
regular_price = 'N/A'
|
|
54
|
+
sale_price = 'N/A'
|
|
55
|
+
sale_price_tag = product.find('span', class_='underline')
|
|
56
|
+
if sale_price_tag:
|
|
57
|
+
regular_tag = sale_price_tag.find('span', class_='price-actual')
|
|
58
|
+
if regular_tag:
|
|
59
|
+
regular_price = regular_tag.get_text(strip=True)
|
|
60
|
+
sale_spans = [span for span in sale_price_tag.find_all('span', recursive=False)
|
|
61
|
+
if 'price-actual' not in (span.get('class') or [])]
|
|
62
|
+
if sale_spans:
|
|
63
|
+
sale_price = sale_spans[0].get_text(strip=True)
|
|
64
|
+
elif regular_price == 'N/A':
|
|
65
|
+
sale_price = sale_price_tag.get_text(strip=True)
|
|
66
|
+
|
|
67
|
+
if sale_price == 'N/A':
|
|
68
|
+
sale_price = regular_price
|
|
69
|
+
|
|
70
|
+
img_container = product.find('div', class_='img-container')
|
|
71
|
+
link = img_container.find('a', href=True) if img_container else product.find('a', href=True)
|
|
72
|
+
product_url = link['href'] if link else 'N/A'
|
|
73
|
+
|
|
74
|
+
image_url = 'N/A'
|
|
75
|
+
image_tag = img_container.find('img') if img_container else product.find('img')
|
|
76
|
+
if image_tag:
|
|
77
|
+
image_url = image_tag.get('src') or image_tag.get('data-src') or 'N/A'
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
'website': 'Sonam Gear',
|
|
81
|
+
'name': name,
|
|
82
|
+
'regular_price': regular_price,
|
|
83
|
+
'sale_price': sale_price,
|
|
84
|
+
'url': urljoin(BASE_URL, product_url),
|
|
85
|
+
'image_url': urljoin(BASE_URL, image_url) if image_url != 'N/A' else 'N/A',
|
|
86
|
+
'timestamp': datetime.now().isoformat(),
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
def scrape(self):
|
|
90
|
+
self.results = []
|
|
91
|
+
try:
|
|
92
|
+
product_ids = self._get_product_ids()
|
|
93
|
+
except Exception as e:
|
|
94
|
+
print(f"Error loading Sonam Gear product list: {str(e)}")
|
|
95
|
+
return self.results
|
|
96
|
+
|
|
97
|
+
def fetch(product_id):
|
|
98
|
+
try:
|
|
99
|
+
return self._parse_product(self._fetch_product(product_id))
|
|
100
|
+
except Exception as e:
|
|
101
|
+
print(f"Error parsing product {product_id}: {str(e)}")
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
|
|
105
|
+
for product in pool.map(fetch, product_ids):
|
|
106
|
+
if product:
|
|
107
|
+
self.results.append(product)
|
|
108
|
+
|
|
109
|
+
print(f"Found {len(self.results)} products.")
|
|
110
|
+
return self.results
|
|
111
|
+
|
|
112
|
+
def save_results(self, filename='scraped_sonamgear.json'):
|
|
113
|
+
with open(filename, 'w', encoding='utf-8') as f:
|
|
114
|
+
json.dump(self.results, f, ensure_ascii=False, indent=2)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
if __name__ == "__main__":
|
|
118
|
+
scraper = SonamGearScraper()
|
|
119
|
+
scraper.scrape()
|
|
120
|
+
scraper.save_results()
|
|
121
|
+
print(f"\nFound {len(scraper.results)} products on sale. Results saved to scraped_sonamgear.json.")
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<title>Discount Nepal</title>
|
|
7
|
+
<style>
|
|
8
|
+
:root{--bg:#f6f7f9;--card:#fff;--text:#111827;--muted:#6b7280;--accent:#2563eb;--border:#e5e7eb;--danger:#b91c1c}
|
|
9
|
+
*{box-sizing:border-box}
|
|
10
|
+
body{margin:0;font-family:system-ui,-apple-system,Arial,sans-serif;background:var(--bg);color:var(--text)}
|
|
11
|
+
header{max-width:1200px;margin:0 auto;padding:24px 24px 8px}
|
|
12
|
+
h1{margin:0 0 4px}
|
|
13
|
+
.meta{color:var(--muted);font-size:13px}
|
|
14
|
+
.controls{display:flex;gap:12px;flex-wrap:wrap;margin:14px 0 4px;align-items:center}
|
|
15
|
+
input,select,button{padding:10px 12px;border-radius:8px;font-size:14px;border:1px solid var(--border)}
|
|
16
|
+
input,select{background:#fff}
|
|
17
|
+
input{flex:1;min-width:180px}
|
|
18
|
+
button{background:var(--accent);color:#fff;border-color:var(--accent);cursor:pointer}
|
|
19
|
+
button:disabled{opacity:.6;cursor:not-progress}
|
|
20
|
+
.grid{max-width:1200px;margin:0 auto;padding:16px 24px 32px;display:grid;gap:16px;grid-template-columns:repeat(auto-fill,minmax(230px,1fr))}
|
|
21
|
+
.card{background:var(--card);border:1px solid var(--border);border-radius:12px;overflow:hidden;display:flex;flex-direction:column}
|
|
22
|
+
.card img{width:100%;height:170px;object-fit:cover;background:#ececec}
|
|
23
|
+
.body{padding:12px;display:flex;flex-direction:column;gap:6px;flex:1}
|
|
24
|
+
.title{font-weight:600;font-size:14px;line-height:1.3}
|
|
25
|
+
.site{font-size:12px;color:var(--muted)}
|
|
26
|
+
.price{margin-top:auto;font-size:13px}
|
|
27
|
+
.sale{color:var(--danger);font-weight:700}
|
|
28
|
+
.badge{display:inline-block;background:#fee2e2;color:var(--danger);border-radius:999px;padding:2px 8px;font-size:12px;font-weight:600}
|
|
29
|
+
a{color:var(--accent);text-decoration:none;font-size:13px}
|
|
30
|
+
.count{color:var(--muted);font-size:13px;margin-left:auto;white-space:nowrap}
|
|
31
|
+
.status{padding:0 24px 16px;color:var(--muted);font-size:14px}
|
|
32
|
+
</style>
|
|
33
|
+
</head>
|
|
34
|
+
<body>
|
|
35
|
+
<header>
|
|
36
|
+
<h1>Discount Nepal</h1>
|
|
37
|
+
<div class="meta" id="meta">Loading latest products from Dolpo and Sonam Gear…</div>
|
|
38
|
+
<div class="controls">
|
|
39
|
+
<input id="search" type="search" placeholder="Search products…">
|
|
40
|
+
<select id="site"><option value="">All sites</option></select>
|
|
41
|
+
<button id="refresh">Refresh data</button>
|
|
42
|
+
<span class="count" id="count"></span>
|
|
43
|
+
</div>
|
|
44
|
+
</header>
|
|
45
|
+
<div class="status" id="status"></div>
|
|
46
|
+
<main class="grid" id="grid"></main>
|
|
47
|
+
<script>
|
|
48
|
+
let products = [];
|
|
49
|
+
const grid = document.getElementById('grid');
|
|
50
|
+
const statusEl = document.getElementById('status');
|
|
51
|
+
const metaEl = document.getElementById('meta');
|
|
52
|
+
const search = document.getElementById('search');
|
|
53
|
+
const site = document.getElementById('site');
|
|
54
|
+
const refreshBtn = document.getElementById('refresh');
|
|
55
|
+
const countEl = document.getElementById('count');
|
|
56
|
+
|
|
57
|
+
function esc(v){return String(v ?? '').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]))}
|
|
58
|
+
function price(v){const m=String(v||'').replace(/,/g,'').match(/\d+(\.\d+)?/);return m?parseFloat(m[0]):null}
|
|
59
|
+
function discount(p){const r=price(p.regular_price),s=price(p.sale_price);return r&&s&&r>0&&s<r?Math.round((r-s)/r*100):0}
|
|
60
|
+
|
|
61
|
+
function render(){
|
|
62
|
+
const term = search.value.trim().toLowerCase();
|
|
63
|
+
const siteName = site.value;
|
|
64
|
+
const items = products.filter(p =>
|
|
65
|
+
(!term || (p.name||'').toLowerCase().includes(term)) &&
|
|
66
|
+
(!siteName || p.website === siteName));
|
|
67
|
+
grid.innerHTML = '';
|
|
68
|
+
for (const p of items){
|
|
69
|
+
const d = discount(p);
|
|
70
|
+
const card = document.createElement('article');
|
|
71
|
+
card.className = 'card';
|
|
72
|
+
card.innerHTML = `
|
|
73
|
+
<img src="${esc(p.image_url)}" alt="" loading="lazy">
|
|
74
|
+
<div class="body">
|
|
75
|
+
<div class="title">${esc(p.name)}</div>
|
|
76
|
+
<div class="site">${esc(p.website)}</div>
|
|
77
|
+
${d ? `<div><span class="badge">${d}% off</span></div>` : ''}
|
|
78
|
+
<div class="price">
|
|
79
|
+
<span class="sale">${esc(p.sale_price || 'N/A')}</span>
|
|
80
|
+
${(p.regular_price && p.regular_price !== p.sale_price) ? `<span> was ${esc(p.regular_price)}</span>` : ''}
|
|
81
|
+
</div>
|
|
82
|
+
<a href="${esc(p.url)}" target="_blank" rel="noopener">View product</a>
|
|
83
|
+
</div>`;
|
|
84
|
+
grid.appendChild(card);
|
|
85
|
+
}
|
|
86
|
+
countEl.textContent = `${items.length} of ${products.length} products`;
|
|
87
|
+
statusEl.textContent = items.length ? '' : 'No matching products.';
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function updateSites(){
|
|
91
|
+
const names = [...new Set(products.map(p=>p.website).filter(Boolean))].sort();
|
|
92
|
+
site.innerHTML = '<option value="">All sites</option>' + names.map(n=>`<option>${esc(n)}</option>`).join('');
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async function load(force){
|
|
96
|
+
refreshBtn.disabled = true;
|
|
97
|
+
countEl.textContent = '';
|
|
98
|
+
metaEl.textContent = force ? 'Scraping latest data…' : 'Loading products…';
|
|
99
|
+
statusEl.textContent = '';
|
|
100
|
+
try {
|
|
101
|
+
const res = await fetch('/api/products' + (force ? '?refresh=1' : ''));
|
|
102
|
+
if (!res.ok) throw new Error('HTTP ' + res.status);
|
|
103
|
+
const data = await res.json();
|
|
104
|
+
products = data.products || [];
|
|
105
|
+
updateSites();
|
|
106
|
+
const when = new Date((data.fetched_at||0)*1000).toLocaleTimeString();
|
|
107
|
+
metaEl.textContent = `Latest data scraped at ${when} — ${products.length} products from Dolpo and Sonam Gear.`;
|
|
108
|
+
render();
|
|
109
|
+
} catch (e) {
|
|
110
|
+
statusEl.textContent = 'Failed to load products: ' + e.message;
|
|
111
|
+
metaEl.textContent = '';
|
|
112
|
+
} finally {
|
|
113
|
+
refreshBtn.disabled = false;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
search.addEventListener('input', render);
|
|
118
|
+
site.addEventListener('change', render);
|
|
119
|
+
refreshBtn.addEventListener('click', () => load(true));
|
|
120
|
+
load(false);
|
|
121
|
+
</script>
|
|
122
|
+
</body>
|
|
123
|
+
</html>
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: discount-nepal
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Web app that lists discounted trekking gear from Nepali outdoor stores (Dolpo, Sonam Gear).
|
|
5
|
+
Author: Discount Nepal
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: scraper,discounts,trekking,nepal,ecommerce
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Environment :: Web Environment
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
13
|
+
Classifier: Topic :: Office/Business
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: requests>=2.31
|
|
18
|
+
Requires-Dist: beautifulsoup4>=4.12
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# Discount Nepal
|
|
22
|
+
|
|
23
|
+
A small web app that lists discounted trekking and outdoor gear from Nepali
|
|
24
|
+
stores. It scrapes **Dolpo** and **Sonam Gear** on demand and serves the results
|
|
25
|
+
through a JSON API, so the page always shows current prices.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install discount-nepal
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Run
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
discount-nepal
|
|
37
|
+
# or: discount-nepal --host 0.0.0.0 --port 8000
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Then open <http://127.0.0.1:8000>.
|
|
41
|
+
|
|
42
|
+
## How it works
|
|
43
|
+
|
|
44
|
+
- `GET /` - the product page
|
|
45
|
+
- `GET /api/products` - cached products (120s TTL)
|
|
46
|
+
- `GET /api/products?refresh=1` - forces a fresh scrape
|
|
47
|
+
|
|
48
|
+
Dolpo and Sonam Gear are scraped concurrently, so a full refresh takes about
|
|
49
|
+
5-6 seconds. The cache is warmed in the background at startup.
|
|
50
|
+
|
|
51
|
+
## Adding another store
|
|
52
|
+
|
|
53
|
+
Add a scraper under `discount_nepal/scrapers/` exposing a class with
|
|
54
|
+
`scrape()` and `get_results()`, then include it in
|
|
55
|
+
`discount_nepal.app.scrape_products()`.
|
|
56
|
+
|
|
57
|
+
## License
|
|
58
|
+
|
|
59
|
+
MIT
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
discount_nepal/__init__.py
|
|
5
|
+
discount_nepal/app.py
|
|
6
|
+
discount_nepal.egg-info/PKG-INFO
|
|
7
|
+
discount_nepal.egg-info/SOURCES.txt
|
|
8
|
+
discount_nepal.egg-info/dependency_links.txt
|
|
9
|
+
discount_nepal.egg-info/entry_points.txt
|
|
10
|
+
discount_nepal.egg-info/requires.txt
|
|
11
|
+
discount_nepal.egg-info/top_level.txt
|
|
12
|
+
discount_nepal/scrapers/__init__.py
|
|
13
|
+
discount_nepal/scrapers/aggregate.py
|
|
14
|
+
discount_nepal/scrapers/dolpo.py
|
|
15
|
+
discount_nepal/scrapers/sonamgear.py
|
|
16
|
+
discount_nepal/templates/index.html
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
discount_nepal
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "discount-nepal"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Web app that lists discounted trekking gear from Nepali outdoor stores (Dolpo, Sonam Gear)."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Discount Nepal" }]
|
|
14
|
+
keywords = ["scraper", "discounts", "trekking", "nepal", "ecommerce"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Environment :: Web Environment",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Topic :: Internet :: WWW/HTTP :: Dynamic Content",
|
|
21
|
+
"Topic :: Office/Business",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"requests>=2.31",
|
|
25
|
+
"beautifulsoup4>=4.12",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.scripts]
|
|
29
|
+
discount-nepal = "discount_nepal.app:main"
|
|
30
|
+
|
|
31
|
+
[tool.setuptools.packages.find]
|
|
32
|
+
include = ["discount_nepal*"]
|
|
33
|
+
|
|
34
|
+
[tool.setuptools.package-data]
|
|
35
|
+
discount_nepal = ["templates/*.html"]
|