scrapedesk 1.0.0__py3-none-any.whl

scrapedesk/__init__.py

@@ -0,0 +1,7 @@
+"""ScrapeDesk - a friendly desktop web scraper for images, tables, links and text."""
+
+__version__ = "1.0.0"
+
+from .core import ImageItem, PageResult, ScrapeSettings, TableItem, WebScraper  # noqa: E402
+
+__all__ = ["ScrapeSettings", "WebScraper", "PageResult", "ImageItem", "TableItem", "__version__"]

scrapedesk/__main__.py

@@ -0,0 +1,15 @@
+"""`python -m scrapedesk` opens the GUI; `python -m scrapedesk --cli ...` runs the command line."""
+
+import sys
+
+
+def main():
+    if len(sys.argv) > 1 and sys.argv[1] == "--cli":
+        from .cli import main as cli_main
+        return cli_main(sys.argv[2:])
+    from .gui.app import main as gui_main
+    return gui_main()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scrapedesk/cli.py

@@ -0,0 +1,100 @@
+"""
+Command-line interface for ScrapeDesk (no GUI needed).
+
+Examples
+  scrapedesk-cli https://example.com --images --download
+  scrapedesk-cli https://en.wikipedia.org/wiki/List_of_largest_cities --tables -o cities.xlsx
+  scrapedesk-cli https://books.toscrape.com --crawl --max-pages 20 --report report.xlsx
+"""
+
+import argparse
+import sys
+
+from .core import ScrapeSettings, WebScraper, USER_AGENTS
+from . import exporters, __version__
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="scrapedesk-cli", description="ScrapeDesk - scrape images, tables, "
+                                 "links and text from websites (command line)")
+    ap.add_argument("-V", "--version", action="version", version=f"%(prog)s {__version__}")
+    ap.add_argument("urls", nargs="+", help="URL(s) or local .html file(s)")
+    ap.add_argument("--images", action="store_true", help="extract images")
+    ap.add_argument("--tables", action="store_true", help="extract tables")
+    ap.add_argument("--links", action="store_true", help="extract links")
+    ap.add_argument("--text", action="store_true", help="extract text/meta/contacts")
+    ap.add_argument("--download", action="store_true", help="download the images found")
+    ap.add_argument("--dir", help="image download folder")
+    ap.add_argument("--min-width", type=int, default=0)
+    ap.add_argument("--min-height", type=int, default=0)
+    ap.add_argument("--types", help="comma separated image types, e.g. jpg,png,webp")
+    ap.add_argument("-o", "--output", help="export tables to this file (.xlsx .csv .json .html .db .md)")
+    ap.add_argument("--report", help="export EVERYTHING to .xlsx / .json / .db")
+    ap.add_argument("--crawl", action="store_true")
+    ap.add_argument("--depth", type=int, default=1)
+    ap.add_argument("--max-pages", type=int, default=20)
+    ap.add_argument("--paginate", action="store_true", help="follow 'next page' links")
+    ap.add_argument("--js", choices=["none", "playwright", "selenium"], default="none")
+    ap.add_argument("--delay", type=float, default=0.5)
+    ap.add_argument("--proxy", default="")
+    ap.add_argument("--ua", choices=list(USER_AGENTS), default="Chrome (Windows)")
+    ap.add_argument("--css", action="append", default=[], metavar="NAME=SELECTOR[@attr]",
+                    help="custom CSS field, e.g. price=.price_color or link='h3 a@href'")
+    ap.add_argument("--row", default="", help="record container CSS selector for --css fields")
+    ap.add_argument("--profile", help="load settings from a profile .json saved by the GUI")
+    a = ap.parse_args(argv)
+
+    if a.profile:
+        with open(a.profile, encoding="utf-8") as fh:
+            s = ScrapeSettings.from_json(fh.read())
+    else:
+        s = ScrapeSettings()
+    any_flag = a.images or a.tables or a.links or a.text
+    if any_flag or not a.profile:
+        everything = not any_flag
+        s.extract_images = a.images or everything
+        s.extract_tables = a.tables or everything
+        s.extract_links = a.links or everything
+        s.extract_text = s.extract_meta = s.extract_contacts = a.text or everything
+    s.crawl, s.max_depth, s.max_pages, s.follow_pagination = a.crawl, a.depth, a.max_pages, a.paginate
+    s.render_js, s.delay, s.proxy, s.user_agent = a.js, a.delay, a.proxy, USER_AGENTS[a.ua]
+    s.min_width, s.min_height = a.min_width, a.min_height
+    if a.dir:
+        s.download_dir = a.dir
+    if a.types:
+        s.image_types = [t.strip().lower() for t in a.types.split(",")]
+    for spec in a.css:
+        name, _, q = spec.partition("=")
+        q, _, attr = q.partition("@")
+        s.custom_selectors.append({"name": name, "type": "css", "query": q, "attr": attr or "text"})
+    s.custom_row_selector = a.row
+
+    sc = WebScraper(s, log=print)
+    results = sc.run(a.urls)
+    images = [i for r in results for i in r.images]
+    tables = [t for r in results for t in r.tables]
+    print(f"\nPages: {len(results)}  Images: {len(images)}  Tables: {len(tables)}  "
+          f"Links: {sum(len(r.links) for r in results)}  Custom rows: {sum(len(r.custom) for r in results)}")
+    for t in tables[:20]:
+        print(f"  - {t.name}: {len(t.rows)} rows x {max(len(x) for x in t.rows)} cols")
+    if a.download and images:
+        sc.download_images(images)
+    if a.output:
+        custom = [row for r in results for row in r.custom]
+        frames = exporters.tables_to_frames(tables)
+        if custom:
+            frames = {"Custom": exporters.records_to_df(custom), **frames}
+        if frames:
+            for f in exporters.export_dataframes(frames, a.output):
+                print("saved", f)
+        else:
+            print("No tables / custom data to export.")
+    if a.report:
+        for f in exporters.export_full_report(results, a.report):
+            print("saved", f)
+    sc.close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())