dvdcompare-scraper 0.1.0__py3-none-any.whl

dvdcompare/cli.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from dataclasses import asdict
+
+from .models import Release
+from .parser import format_runtime
+from .scraper import get_film, get_film_by_url, search
+
+
+def select_releases(releases: list[Release], selector: str) -> list[Release]:
+    """Filter releases by 1-based index or case-insensitive name substring.
+
+    Raises LookupError if no release matches the selector.
+    """
+    try:
+        idx = int(selector) - 1
+        idx = min(idx, len(releases) - 1)
+        return [releases[idx]]
+    except ValueError:
+        pass
+
+    keyword = selector.lower()
+    matched = [r for r in releases if keyword in r.name.lower()]
+    if matched:
+        return matched
+
+    names = "\n".join(f"  {i}. {r.name}" for i, r in enumerate(releases, 1))
+    raise LookupError(
+        f"No release matching '{selector}'.\nAvailable releases:\n{names}"
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Scrape disc extras metadata from dvdcompare.net",
+    )
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("query", nargs="?", help="Search query (film title)")
+    group.add_argument("--id", type=int, dest="film_id", help="dvdcompare film ID")
+    group.add_argument("--url", help="dvdcompare film page URL")
+
+    parser.add_argument(
+        "--release",
+        default="1",
+        help="Release number (1-based) or name keyword (default: 1)",
+    )
+    parser.add_argument(
+        "--all-releases",
+        action="store_true",
+        help="Show all releases instead of just one",
+    )
+    parser.add_argument("--json", action="store_true", help="Output as JSON")
+
+    args = parser.parse_args()
+    asyncio.run(_run(args))
+
+
+async def _run(args: argparse.Namespace) -> None:
+    if args.film_id:
+        film = await get_film(args.film_id)
+    elif args.url:
+        film = await get_film_by_url(args.url)
+    else:
+        results = await search(args.query)
+        if not results:
+            print("No results found.", file=sys.stderr)
+            sys.exit(1)
+        if len(results) > 1:
+            print(f"Found {len(results)} results:", file=sys.stderr)
+            for i, r in enumerate(results, 1):
+                print(f"  {i}. {r.title} (fid={r.film_id})", file=sys.stderr)
+            print(
+                "Using first result. Use --id to select a specific one.",
+                file=sys.stderr,
+            )
+        film = await get_film(results[0].film_id)
+
+    if not args.all_releases and film.releases:
+        film.releases = select_releases(film.releases, args.release)
+
+    if args.json:
+        print(json.dumps(asdict(film), indent=2))
+    else:
+        _print_text(film, args)
+
+
+def _print_text(film, args: argparse.Namespace) -> None:
+    header = film.title
+    if film.format:
+        header += f" ({film.format})"
+    if film.year:
+        header += f" ({film.year})"
+    print(header)
+    if film.director:
+        print(f"Director: {film.director}")
+    if film.imdb_id:
+        print(f"IMDB: {film.imdb_id}")
+    print()
+
+    for release in film.releases:
+        line = f"--- {release.name}"
+        if release.year:
+            line += f" [{release.year}]"
+        line += " ---"
+        print(line)
+
+        for disc in release.discs:
+            print(f"\n  DISC {disc.number} ({disc.format})")
+            if disc.is_film:
+                print("    The Film")
+            for feature in disc.features:
+                _print_feature(feature, indent=4)
+        print()
+
+
+def _print_feature(feature, indent: int = 4) -> None:
+    prefix = " " * indent
+    parts = [f'{prefix}"{feature.title}"']
+    if feature.year:
+        parts.append(str(feature.year))
+    if feature.feature_type:
+        parts.append(feature.feature_type)
+    if feature.runtime_seconds is not None:
+        rt = format_runtime(feature.runtime_seconds)
+        if feature.is_play_all:
+            parts.append(f"(Play All - {rt})")
+        else:
+            parts.append(f"({rt})")
+    if feature.technical_notes:
+        parts.append(f"[{feature.technical_notes}]")
+    print(" ".join(parts))
+
+    for child in feature.children:
+        _print_feature(child, indent=indent + 2)

dvdcompare/models.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Feature:
+    """A single bonus feature on a disc."""
+
+    title: str
+    runtime_seconds: int | None = None
+    feature_type: str | None = None
+    year: int | None = None
+    technical_notes: str | None = None
+    is_play_all: bool = False
+    children: list[Feature] = field(default_factory=list)
+
+
+@dataclass
+class Disc:
+    """A single disc in a release."""
+
+    number: int
+    format: str
+    is_film: bool = False
+    features: list[Feature] = field(default_factory=list)
+
+
+@dataclass
+class Release:
+    """A regional release of a film."""
+
+    name: str
+    year: int | None = None
+    discs: list[Disc] = field(default_factory=list)
+
+
+@dataclass
+class FilmComparison:
+    """A complete film comparison page from dvdcompare.net."""
+
+    title: str
+    year: int | None = None
+    format: str | None = None
+    director: str | None = None
+    imdb_url: str | None = None
+    imdb_id: str | None = None
+    film_id: int | None = None
+    releases: list[Release] = field(default_factory=list)
+
+
+@dataclass
+class SearchResult:
+    """A single result from a dvdcompare.net search."""
+
+    title: str
+    url: str
+    film_id: int | None = None

dvdcompare/parser.py

@@ -0,0 +1,351 @@
+from __future__ import annotations
+
+import html as html_module
+import re
+
+from bs4 import BeautifulSoup
+
+from .models import Disc, Feature, FilmComparison, Release, SearchResult
+
+_DISC_WORDS = {
+    "ONE": 1,
+    "TWO": 2,
+    "THREE": 3,
+    "FOUR": 4,
+    "FIVE": 5,
+    "SIX": 6,
+    "SEVEN": 7,
+    "EIGHT": 8,
+    "NINE": 9,
+    "TEN": 10,
+}
+
+
+def parse_runtime(s: str) -> int:
+    """Parse ``MM:SS``, ``H:MM:SS``, or ``NNN mins`` into total seconds."""
+    s = s.strip()
+    mins_match = re.match(r"^(\d+)\s*mins?$", s)
+    if mins_match:
+        return int(mins_match.group(1)) * 60
+    parts = s.split(":")
+    if len(parts) == 2:
+        return int(parts[0]) * 60 + int(parts[1])
+    if len(parts) == 3:
+        return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
+    return 0
+
+
+def format_runtime(seconds: int) -> str:
+    """Format total seconds as ``MM:SS`` or ``H:MM:SS``."""
+    mins, secs = divmod(seconds, 60)
+    hours, mins = divmod(mins, 60)
+    if hours:
+        return f"{hours}:{mins:02d}:{secs:02d}"
+    return f"{mins}:{secs:02d}"
+
+
+def _disc_number(word: str) -> int:
+    word = word.upper()
+    if word in _DISC_WORDS:
+        return _DISC_WORDS[word]
+    try:
+        return int(word)
+    except ValueError:
+        return 0
+
+
+def parse_feature_line(line: str) -> Feature:
+    """Parse a single feature text line into a :class:`Feature`."""
+    technical_notes = None
+    is_play_all = False
+    runtime_seconds = None
+    year = None
+    feature_type = None
+
+    # 1. Extract technical notes [...]
+    tech_match = re.search(r"\s*\[([^\]]+)\]", line)
+    if tech_match:
+        technical_notes = tech_match.group(1)
+        line = line[: tech_match.start()] + line[tech_match.end() :]
+
+    # 2. Strip trailing colon (group indicator) -- detected by caller
+    line = line.strip()
+    if line.endswith(":"):
+        line = line[:-1].strip()
+
+    # 3. Extract runtime  (MM:SS) / (H:MM:SS) / (Play All - MM:SS) / (NNN mins)
+    #    Also handles "with Play All option - MM:SS"
+    runtime_match = re.search(
+        r"\((?:(?:with )?(Play All)(?: option)? - )?(\d{1,3}:\d{2}(?::\d{2})?|\d+\s*mins?)\)",
+        line,
+    )
+    if runtime_match:
+        if runtime_match.group(1):
+            is_play_all = True
+        runtime_seconds = parse_runtime(runtime_match.group(2))
+        line = line[: runtime_match.start()] + line[runtime_match.end() :]
+        line = line.strip()
+
+    # 3b. Strip trailing colon again (may be exposed after runtime removal)
+    if line.endswith(":"):
+        line = line[:-1].strip()
+
+    # 4. Extract quoted title vs unquoted
+    quote_match = re.match(r'^["\u201c](.+?)["\u201d](.*)$', line)
+    if quote_match:
+        title = quote_match.group(1).strip()
+        rest = quote_match.group(2).strip()
+    else:
+        title = line.strip()
+        rest = ""
+
+    # 5. From rest, extract year and type
+    if rest:
+        year_match = re.match(r"^(\d{4})\s*(.*)", rest)
+        if year_match:
+            year = int(year_match.group(1))
+            feature_type = year_match.group(2).strip() or None
+        else:
+            feature_type = rest.strip() or None
+
+    # Normalize whitespace in title
+    title = re.sub(r"\s+", " ", title).strip()
+
+    return Feature(
+        title=title,
+        runtime_seconds=runtime_seconds,
+        feature_type=feature_type,
+        year=year,
+        technical_notes=technical_notes,
+        is_play_all=is_play_all,
+    )
+
+
+def parse_extras(extras_html: str) -> list[Disc]:
+    """Parse the inner HTML of an extras description div into :class:`Disc` objects."""
+    # Replace <br> variants with newlines
+    text = re.sub(r"<br\s*/?>", "\n", extras_html)
+    # Remove all remaining HTML tags
+    text = re.sub(r"<[^>]+>", "", text)
+    # Decode HTML entities
+    text = html_module.unescape(text)
+
+    lines = [ln.strip() for ln in text.split("\n")]
+
+    discs: list[Disc] = []
+    current_disc: Disc | None = None
+    current_group: Feature | None = None
+
+    for line in lines:
+        if not line:
+            continue
+
+        # Disc header: DISC ONE (Blu-ray 4K)  or  DISC ONE
+        disc_match = re.match(r"^DISC\s+(\w+)(?:\s+\((.+)\))?$", line)
+        if disc_match:
+            current_disc = Disc(
+                number=_disc_number(disc_match.group(1)),
+                format=disc_match.group(2) or "",
+            )
+            discs.append(current_disc)
+            current_group = None
+            continue
+
+        # "* The Film" marker (possibly with a variant title suffix).
+        # dvdcompare uses a leading asterisk to flag the main feature.
+        if line.startswith("*"):
+            stripped = line.lstrip("*").strip()
+            is_film_marker = (
+                not stripped  # bare "*"
+                or stripped.lower().startswith("the film")
+            )
+            if is_film_marker and current_disc:
+                current_disc.is_film = True
+            # Exact "* The Film" (no extra info) -> skip entirely
+            if not stripped or stripped.lower() == "the film":
+                continue
+            # Otherwise strip the asterisk and keep as a feature
+            line = stripped
+            # fall through to feature parsing below
+
+        if current_disc is None:
+            continue
+
+        # Sub-feature (starts with "- ")
+        if line.startswith("- "):
+            feature = parse_feature_line(line[2:])
+            if current_group:
+                current_group.children.append(feature)
+            else:
+                current_disc.features.append(feature)
+            continue
+
+        feature = parse_feature_line(line)
+        current_disc.features.append(feature)
+
+        # Detect group header (trailing colon or play-all)
+        is_group = line.rstrip().endswith(":") or feature.is_play_all
+
+        if is_group:
+            current_group = feature
+        else:
+            current_group = None
+
+    return discs
+
+
+def parse_film_page(html: str) -> FilmComparison:
+    """Parse a dvdcompare.net film comparison page into a :class:`FilmComparison`."""
+    soup = BeautifulSoup(html, "html.parser")
+
+    # --- Title, format, year from <h2> ---
+    title = ""
+    year = None
+    disc_format = None
+    h2 = soup.find("h2")
+    if h2:
+        h2_text = h2.get_text(strip=True)
+        # Last (YYYY) is the year
+        year_match = re.search(r"\((\d{4})\)\s*$", h2_text)
+        if year_match:
+            year = int(year_match.group(1))
+            rest = h2_text[: year_match.start()].strip()
+        else:
+            rest = h2_text
+        # Format in parens at end of remainder
+        fmt_match = re.search(r"\(([^)]+)\)\s*$", rest)
+        if fmt_match:
+            disc_format = fmt_match.group(1)
+            title = rest[: fmt_match.start()].strip()
+        else:
+            title = rest
+
+    # --- IMDB ---
+    imdb_url = None
+    imdb_id = None
+    imdb_link = soup.find("a", href=re.compile(r"imdb\.com/title/"))
+    if imdb_link:
+        imdb_url = imdb_link["href"]
+        id_match = re.search(r"/(tt\d+)", imdb_url)
+        if id_match:
+            imdb_id = id_match.group(1)
+
+    # --- Director ---
+    director = None
+    content_div = soup.find("div", id="content")
+    if content_div:
+        dir_match = re.search(
+            r"Director:\s*(.+?)(?:\n|$)", content_div.get_text()
+        )
+        if dir_match:
+            director = dir_match.group(1).strip()
+
+    # --- Film ID ---
+    film_id = None
+    form = soup.find("form", action=re.compile(r"film\.php\?fid="))
+    if form:
+        fid_match = re.search(r"fid=(\d+)", form["action"])
+        if fid_match:
+            film_id = int(fid_match.group(1))
+
+    # --- Releases ---
+    releases: list[Release] = []
+    table = soup.find("table", attrs={"border": "0", "align": "center"})
+    if table:
+        for tr in table.find_all("tr"):
+            ul = tr.find("ul", class_="dvd")
+            if not ul:
+                continue
+
+            h3 = ul.find("h3")
+            if not h3:
+                continue
+
+            # Release name and year
+            release_year = None
+            year_span = h3.find("span", class_="disc-release-year")
+            if year_span:
+                ry_match = re.search(r"\[(\d{4})", year_span.get_text())
+                if ry_match:
+                    release_year = int(ry_match.group(1))
+                release_name = h3.get_text(strip=True).replace(
+                    year_span.get_text(strip=True), ""
+                ).strip()
+            else:
+                release_name = h3.get_text(strip=True)
+
+            # Find extras
+            discs: list[Disc] = []
+            for li in ul.find_all("li", recursive=False):
+                label_div = li.find("div", class_="label")
+                if label_div and "Extras:" in label_div.get_text():
+                    desc_div = li.find("div", class_="description")
+                    if desc_div:
+                        discs = parse_extras(desc_div.decode_contents())
+                    break
+
+            releases.append(
+                Release(name=release_name, year=release_year, discs=discs)
+            )
+
+    return FilmComparison(
+        title=title,
+        year=year,
+        format=disc_format,
+        director=director,
+        imdb_url=imdb_url,
+        imdb_id=imdb_id,
+        film_id=film_id,
+        releases=releases,
+    )
+
+
+def parse_search_results(html: str) -> list[SearchResult]:
+    """Parse a dvdcompare.net search results page.
+
+    When the search returns exactly one hit, dvdcompare emits a JavaScript
+    redirect (``location.href="film.php?fid=..."``)) instead of a clickable
+    ``<a>`` tag.  This function handles both cases.
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    results: list[SearchResult] = []
+    seen: set[int] = set()
+
+    for link in soup.find_all("a", href=re.compile(r"film\.php\?fid=\d+")):
+        text = link.get_text(strip=True)
+        if not text:
+            continue
+        href = link["href"]
+        fid_match = re.search(r"fid=(\d+)", href)
+        film_id = int(fid_match.group(1)) if fid_match else None
+
+        if film_id and film_id in seen:
+            continue
+        if film_id:
+            seen.add(film_id)
+
+        if not href.startswith("http"):
+            href = f"https://www.dvdcompare.net/comparisons/{href}"
+
+        results.append(SearchResult(title=text, url=href, film_id=film_id))
+
+    # Single-result pages use a JS redirect instead of <a> links.
+    if not results:
+        for script in soup.find_all("script"):
+            content = script.string or ""
+            m = re.search(
+                r'location\.href\s*=\s*"(film\.php\?fid=(\d+))"', content
+            )
+            if m:
+                fid = int(m.group(2))
+                href = f"https://www.dvdcompare.net/comparisons/{m.group(1)}"
+                # Try to grab the title from the <h2> nearby.
+                h2 = soup.find("h2")
+                title = ""
+                if h2:
+                    italic = h2.find("i")
+                    title = italic.get_text(strip=True) if italic else h2.get_text(strip=True)
+                results.append(SearchResult(title=title, url=href, film_id=fid))
+                break
+
+    return results

dvdcompare/scraper.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import httpx
+
+from .models import FilmComparison, SearchResult
+from .parser import parse_film_page, parse_search_results
+
+BASE_URL = "https://www.dvdcompare.net"
+
+_HEADERS = {
+    "User-Agent": "dvdcompare-scraper/0.1 (disc extras lookup)",
+}
+
+
+async def search(
+    query: str, *, client: httpx.AsyncClient | None = None
+) -> list[SearchResult]:
+    """Search dvdcompare.net for a film title."""
+    own_client = client is None
+    if own_client:
+        client = httpx.AsyncClient(headers=_HEADERS)
+    try:
+        resp = await client.post(
+            f"{BASE_URL}/comparisons/search.php",
+            data={"param": query, "searchtype": "text"},
+            follow_redirects=True,
+        )
+        resp.raise_for_status()
+        return parse_search_results(resp.text)
+    finally:
+        if own_client:
+            await client.aclose()
+
+
+async def find_film(
+    title: str,
+    disc_format: str | None = None,
+    *,
+    client: httpx.AsyncClient | None = None,
+) -> FilmComparison:
+    """Search for a title and return the best-matching FilmComparison.
+
+    If *disc_format* is provided (e.g. ``"Blu-ray 4K"``), results whose
+    title contains that format string are preferred.  Falls back to the
+    first result when no format-specific match exists.
+
+    Raises ``LookupError`` if no search results are found.
+    """
+    results = await search(title, client=client)
+    if not results:
+        raise LookupError(f"No dvdcompare results for '{title}'")
+
+    best = results[0]
+    if disc_format:
+        fmt_lower = disc_format.lower()
+        for r in results:
+            if fmt_lower in r.title.lower():
+                best = r
+                break
+
+    return await get_film(best.film_id, client=client)
+
+
+async def get_film(
+    film_id: int, *, client: httpx.AsyncClient | None = None
+) -> FilmComparison:
+    """Fetch and parse a film comparison page by dvdcompare film ID."""
+    url = f"{BASE_URL}/comparisons/film.php?fid={film_id}"
+    return await get_film_by_url(url, client=client)
+
+
+async def get_film_by_url(
+    url: str, *, client: httpx.AsyncClient | None = None
+) -> FilmComparison:
+    """Fetch and parse a film comparison page by full URL."""
+    own_client = client is None
+    if own_client:
+        client = httpx.AsyncClient(headers=_HEADERS)
+    try:
+        resp = await client.get(url, follow_redirects=True)
+        resp.raise_for_status()
+        return parse_film_page(resp.text)
+    finally:
+        if own_client:
+            await client.aclose()

dvdcompare_scraper-0.1.0.dist-info/METADATA

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: dvdcompare-scraper
+Version: 0.1.0
+Summary: Scrape disc extras metadata from dvdcompare.net
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.27
+Requires-Dist: beautifulsoup4>=4.12
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23; extra == "dev"

dvdcompare_scraper-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+dvdcompare/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dvdcompare/cli.py,sha256=JXNLrUtOrRcYgpxUwJvNtjxqyVZnc4XK-5K2S34XK28,4247
+dvdcompare/models.py,sha256=4WyDHA-rjBztESjPACEafU1ojABtPj3SZ2Nl_4iUAp4,1243
+dvdcompare/parser.py,sha256=Si85v2vSfYg4Dz0YPw2Sn04NT3PRrnMy2-HE2JEw69I,11185
+dvdcompare/scraper.py,sha256=NuqRrJdi6A8ksuYRnqnZB8u99dQcgTrlqvN4u0Toprk,2534
+dvdcompare_scraper-0.1.0.dist-info/METADATA,sha256=N6CBL34gXsLuZ9bUCIdHkjRqNfFaQSFRvz5jtg_7wVw,321
+dvdcompare_scraper-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+dvdcompare_scraper-0.1.0.dist-info/entry_points.txt,sha256=zaHml0l2HCdLmBuRZb5mHs2h4DpSATtXgzXU1odQn00,51
+dvdcompare_scraper-0.1.0.dist-info/top_level.txt,sha256=5GNbi4d9JgBreQn7-36QkVLN9_DqTcoupXwWGsz6hRU,11
+dvdcompare_scraper-0.1.0.dist-info/RECORD,,

dvdcompare_scraper-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dvdcompare_scraper-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dvdcompare = dvdcompare.cli:main

dvdcompare_scraper-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dvdcompare