pyfanedit 0.1.1a1__py3-none-any.whl

pyfanedit/__init__.py

@@ -0,0 +1,20 @@
+from pyfanedit.client import CATEGORIES, FaneditClient
+from pyfanedit.models import (
+    FaneditDetail, FaneditSummary,
+    NewsArticle, Review, ReviewRatings,
+    ReviewerEntry, UserReviewEntry,
+)
+from pyfanedit.version import __version__
+
+__all__ = [
+    "__version__",
+    "FaneditClient",
+    "FaneditSummary",
+    "FaneditDetail",
+    "ReviewerEntry",
+    "UserReviewEntry",
+    "NewsArticle",
+    "Review",
+    "ReviewRatings",
+    "CATEGORIES",
+]

pyfanedit/client.py

@@ -0,0 +1,294 @@
+"""High-level fanedit.org client."""
+from __future__ import annotations
+
+from typing import Iterator, List, Optional
+
+from pyfanedit.models import (
+    FaneditDetail, FaneditSummary,
+    NewsArticle, ReviewerEntry, UserReviewEntry,
+)
+from pyfanedit.parsers import (
+    parse_detail_page, parse_listing_page,
+    parse_news_article, parse_news_listing,
+    parse_reviewer_rank_page, parse_user_reviews_page,
+)
+from pyfanedit.session import Session
+
+# Category slug → URL path
+CATEGORIES = {
+    "fanfix": "category/fanedit-listings/fanfix/",
+    "fanmix": "category/fanedit-listings/fanmix/",
+    "extended": "category/fanedit-listings/extended-edition/",
+    "tv_to_movie": "category/fanedit-listings/tv-to-movie/",
+    "movie_to_tv": "category/fanedit-listings/movie-to-tv/",
+    "shorts": "category/fanedit-listings/shorts/",
+    "special": "category/fanedit-listings/custom-special-edition/",
+    "documentary": "category/fanedit-listings/documentary-review/",
+    "preservation": "category/preservation-listings/",
+    "unapproved": "category/unapproved-fanedits/",
+}
+
+ORDER_CHOICES = ("rdate", "date", "modified", "alpha", "rratio", "rvote")
+
+
+class FaneditClient:
+    """Scraping client for fanedit.org / IFDB."""
+
+    def __init__(self, impersonate: str = "chrome120", cache_ttl: float = 300.0) -> None:
+        self._s = Session(impersonate=impersonate, cache_ttl=cache_ttl)
+
+    # ------------------------------------------------------------------
+    # Category browsing
+    # ------------------------------------------------------------------
+
+    def get_category(
+        self,
+        category: str,
+        page: int = 1,
+    ) -> tuple[List[FaneditSummary], Optional[str]]:
+        """Return one page of a category.
+
+        ``category`` may be a key from ``CATEGORIES`` or a full URL path.
+        Returns (items, next_page_url).
+        """
+        path = CATEGORIES.get(category, category)
+        params = {"pg": page} if page > 1 else None
+        html = self._s.get(path, params=params)
+        return parse_listing_page(html)
+
+    def iter_category(self, category: str, max_pages: int = 0) -> Iterator[FaneditSummary]:
+        """Yield all fanedits in a category across pages."""
+        page = 1
+        while True:
+            items, next_url = self.get_category(category, page=page)
+            yield from items
+            if not next_url or (max_pages and page >= max_pages):
+                break
+            page += 1
+
+    # ------------------------------------------------------------------
+    # Search
+    # ------------------------------------------------------------------
+
+    def search(
+        self,
+        keywords: str,
+        scope: str = "title",
+        query_type: str = "all",
+        order: str = "rdate",
+        page: int = 1,
+    ) -> tuple[List[FaneditSummary], Optional[str]]:
+        """Search the IFDB.
+
+        Args:
+            keywords: search terms
+            scope: "title" | "reviews"
+            query_type: "all" | "any" | "exact"
+            order: one of ORDER_CHOICES
+            page: 1-based page number
+
+        Returns:
+            (items, next_page_url)
+        """
+        params: dict = {
+            "query": query_type,
+            "scope": scope,
+            "keywords": keywords,
+            "order": order,
+        }
+        if page > 1:
+            params["pg"] = page
+        html = self._s.get("fanedit-search/search-results/", params=params)
+        return parse_listing_page(html)
+
+    def iter_search(
+        self,
+        keywords: str,
+        scope: str = "title",
+        query_type: str = "all",
+        order: str = "rdate",
+        max_pages: int = 0,
+    ) -> Iterator[FaneditSummary]:
+        """Yield all search results across pages."""
+        page = 1
+        while True:
+            items, next_url = self.search(
+                keywords, scope=scope, query_type=query_type, order=order, page=page
+            )
+            yield from items
+            if not next_url or (max_pages and page >= max_pages):
+                break
+            page += 1
+
+    # ------------------------------------------------------------------
+    # Tag / franchise browsing
+    # ------------------------------------------------------------------
+
+    def get_by_tag(
+        self,
+        tag_type: str,
+        tag_value: str,
+        page: int = 1,
+    ) -> tuple[List[FaneditSummary], Optional[str]]:
+        """Browse by a tag, e.g. get_by_tag("franchise", "star-wars").
+
+        Common tag_type values: franchise, faneditorname, originalmovietitle,
+        fanedittype, faneditreleasedate, award.
+        """
+        path = f"fanedit-search/tag/{tag_type}/{tag_value}/"
+        params: dict = {"criteria": "2"}
+        if page > 1:
+            params["pg"] = page
+        html = self._s.get(path, params=params)
+        return parse_listing_page(html)
+
+    def iter_by_tag(
+        self, tag_type: str, tag_value: str, max_pages: int = 0
+    ) -> Iterator[FaneditSummary]:
+        page = 1
+        while True:
+            items, next_url = self.get_by_tag(tag_type, tag_value, page=page)
+            yield from items
+            if not next_url or (max_pages and page >= max_pages):
+                break
+            page += 1
+
+    # ------------------------------------------------------------------
+    # Curated lists
+    # ------------------------------------------------------------------
+
+    def get_latest(self, page: int = 1) -> tuple[List[FaneditSummary], Optional[str]]:
+        html = self._s.get("latest-ifdb-fanedits/", params={"pg": page} if page > 1 else None)
+        return parse_listing_page(html)
+
+    def get_top_trusted_rated(self, page: int = 1) -> tuple[List[FaneditSummary], Optional[str]]:
+        html = self._s.get("top-trusted-reviewer-rated-fanedits/", params={"pg": page} if page > 1 else None)
+        return parse_listing_page(html)
+
+    def get_top_user_rated(self, page: int = 1) -> tuple[List[FaneditSummary], Optional[str]]:
+        html = self._s.get("top-user-rated-fanedits/", params={"pg": page} if page > 1 else None)
+        return parse_listing_page(html)
+
+    def get_most_popular(self, page: int = 1) -> tuple[List[FaneditSummary], Optional[str]]:
+        html = self._s.get("most-popular/", params={"pg": page} if page > 1 else None)
+        return parse_listing_page(html)
+
+    def get_award_winners(self, page: int = 1) -> tuple[List[FaneditSummary], Optional[str]]:
+        return self.get_by_tag("award", "fanedit-of-the-month", page=page)
+
+    # ------------------------------------------------------------------
+    # Detail
+    # ------------------------------------------------------------------
+
+    def get_detail(self, url: str) -> FaneditDetail:
+        """Fetch and parse a single fanedit detail page.
+
+        ``url`` may be a full URL or a slug like ``/star-wars-despecialized/``.
+        """
+        if not url.startswith("http"):
+            url = "https://fanedit.org/" + url.strip("/") + "/"
+        html = self._s.get(url)
+        return parse_detail_page(html, url)
+
+    # ------------------------------------------------------------------
+    # Reviewer leaderboard
+    # ------------------------------------------------------------------
+
+    def get_reviewer_rank(self, page: int = 1) -> tuple[List[ReviewerEntry], Optional[str]]:
+        """Return one page of the reviewer leaderboard (50 per page, ~3 400 total).
+
+        Each entry includes numeric user_id, which can be passed directly to
+        ``get_user_reviews()`` without needing to know the username.
+        """
+        html = self._s.get("reviewer-rank/", params={"pg": page} if page > 1 else None)
+        return parse_reviewer_rank_page(html)
+
+    def iter_reviewer_rank(self, max_pages: int = 0) -> Iterator[ReviewerEntry]:
+        """Yield all reviewers from the leaderboard across pages."""
+        page = 1
+        while True:
+            entries, next_url = self.get_reviewer_rank(page=page)
+            yield from entries
+            if not next_url or (max_pages and page >= max_pages):
+                break
+            page += 1
+
+    # ------------------------------------------------------------------
+    # Reviews by user
+    # ------------------------------------------------------------------
+
+    REVIEW_ORDER_CHOICES = (
+        "rdate",     # most recent
+        "date",      # oldest
+        "rating",    # most positive
+        "rrating",   # most critical
+        "updated",   # last updated
+        "helpful",   # most helpful
+        "rhelpful",  # least helpful
+        "discussed", # most discussed
+    )
+
+    def get_user_reviews(
+        self,
+        user_id: int,
+        page: int = 1,
+        order: str = "rdate",
+    ) -> tuple[List[UserReviewEntry], Optional[str]]:
+        """Return one page of reviews written by a specific user.
+
+        Args:
+            user_id: numeric jReviews user ID (from ReviewerEntry.user_id or
+                     the /reviewer-rank/ page anchor id="user-N")
+            page: 1-based page number
+            order: one of REVIEW_ORDER_CHOICES
+        """
+        params: dict = {"order": order}
+        if page > 1:
+            params["pg"] = page
+        html = self._s.get(f"my-reviews/{user_id}/", params=params)
+        return parse_user_reviews_page(html)
+
+    def iter_user_reviews(
+        self,
+        user_id: int,
+        order: str = "rdate",
+        max_pages: int = 0,
+    ) -> Iterator[UserReviewEntry]:
+        """Yield all reviews written by a user across pages."""
+        page = 1
+        while True:
+            reviews, next_url = self.get_user_reviews(user_id, page=page, order=order)
+            yield from reviews
+            if not next_url or (max_pages and page >= max_pages):
+                break
+            page += 1
+
+    def get_latest_user_reviews(self, page: int = 1) -> tuple[List[FaneditSummary], Optional[str]]:
+        """Return the latest user reviews feed (all users)."""
+        html = self._s.get("latest-user-reviews/", params={"pg": page} if page > 1 else None)
+        return parse_listing_page(html)
+
+    def get_latest_trusted_reviews(self, page: int = 1) -> tuple[List[FaneditSummary], Optional[str]]:
+        """Return the latest trusted-reviewer reviews feed."""
+        html = self._s.get("latest-trusted-reviewer-reviews/", params={"pg": page} if page > 1 else None)
+        return parse_listing_page(html)
+
+    # ------------------------------------------------------------------
+    # News
+    # ------------------------------------------------------------------
+
+    def get_news(self) -> List[NewsArticle]:
+        """Return the news front page article cards (up to ~15 articles)."""
+        html = self._s.get("forums/news-publisher/")
+        return parse_news_listing(html)
+
+    def get_news_article(self, url: str) -> NewsArticle:
+        """Fetch a full news article, including body text and mentioned IFDB URLs.
+
+        ``url`` may be a full URL or a forums-relative path like
+        ``/forums/news-publisher/some-article.185/``.
+        """
+        if not url.startswith("http"):
+            url = "https://fanedit.org" + url
+        html = self._s.get(url)
+        return parse_news_article(html, url)

pyfanedit/models.py

@@ -0,0 +1,140 @@
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class ReviewRatings(BaseModel):
+    overall: Optional[float] = None
+    audio_video_quality: Optional[float] = None
+    audio_editing: Optional[float] = None
+    visual_editing: Optional[float] = None
+    narrative: Optional[float] = None
+    enjoyment: Optional[float] = None
+
+
+class Review(BaseModel):
+    reviewer: Optional[str] = None
+    reviewer_url: Optional[str] = None
+    reviewer_rank: Optional[str] = None
+    reviewer_review_count: Optional[int] = None
+    date: Optional[str] = None
+    ratings: ReviewRatings = Field(default_factory=ReviewRatings)
+    body: Optional[str] = None
+    discussion_url: Optional[str] = None
+    helpful_yes: Optional[int] = None
+    helpful_no: Optional[int] = None
+
+
+class FaneditSummary(BaseModel):
+    """Lightweight fanedit as returned from list/search pages."""
+    # Canonical IDs
+    fanedit_id: Optional[int] = None        # WordPress post ID — stable numeric ID
+    slug: Optional[str] = None             # URL slug, e.g. "star-wars-begins"
+
+    title: str
+    url: str
+    cover_url: Optional[str] = None
+    faneditor: Optional[str] = None
+    original_title: Optional[str] = None
+    fanedit_type: Optional[str] = None
+    franchise: Optional[str] = None
+    release_date: Optional[str] = None
+    running_time: Optional[str] = None
+    synopsis: Optional[str] = None
+    editor_rating: Optional[float] = None
+    user_rating: Optional[float] = None
+    user_rating_count: Optional[int] = None
+    views: Optional[int] = None
+    updated: Optional[str] = None
+
+
+class FaneditDetail(BaseModel):
+    """Full fanedit detail as parsed from its own page."""
+    # Canonical IDs
+    fanedit_id: Optional[int] = None        # WordPress post ID — stable numeric ID
+    slug: Optional[str] = None             # URL slug
+
+    title: str
+    url: str
+    cover_url: Optional[str] = None
+    faneditor: Optional[str] = None
+    original_title: Optional[str] = None
+    genre: Optional[List[str]] = None
+    franchise: Optional[List[str]] = None
+    fanedit_type: Optional[str] = None
+
+    # Source film metadata
+    original_release_date: Optional[str] = None
+    original_running_time: Optional[str] = None
+    imdb_id: Optional[str] = None          # e.g. "tt0076759" — from embedded IMDB link
+
+    # Edit metadata
+    fanedit_release_date: Optional[str] = None
+    fanedit_running_time: Optional[str] = None
+    time_cut: Optional[str] = None
+    time_added: Optional[str] = None
+    subtitles: Optional[str] = None
+    available_in: Optional[str] = None     # HD / SD / Surround Sound etc.
+    release_information: Optional[str] = None  # Digital / Physical etc.
+
+    # Content
+    synopsis: Optional[str] = None
+    additional_notes: Optional[str] = None
+    special_thanks: Optional[str] = None
+    cuts_and_additions: Optional[str] = None
+    intention: Optional[str] = None         # editor's stated intent
+    awards: Optional[str] = None            # e.g. Fanedit of the Month
+
+    # Ratings
+    editor_rating: Optional[float] = None
+    user_rating: Optional[float] = None
+    user_rating_count: Optional[int] = None
+
+    # Reviews
+    editor_reviews: List[Review] = Field(default_factory=list)
+    user_reviews: List[Review] = Field(default_factory=list)
+
+    # Raw overflow for any unmapped fields
+    extra_fields: Dict[str, str] = Field(default_factory=dict)
+
+
+class ReviewerEntry(BaseModel):
+    """One row from the reviewer leaderboard."""
+    rank: int
+    user_id: int                    # numeric jReviews user ID (stable)
+    username: str
+    profile_url: str                # /members/{username}/
+    reviews_url: str                # /my-reviews/{user_id}/
+    review_count: int
+    helpful_yes: Optional[int] = None
+    helpful_pct: Optional[float] = None  # e.g. 76.61
+
+
+class UserReviewEntry(BaseModel):
+    """One review from a user's review list page."""
+    fanedit_title: str
+    fanedit_url: str
+    fanedit_type: Optional[str] = None
+    date: Optional[str] = None
+    ratings: ReviewRatings = Field(default_factory=ReviewRatings)
+    discussion_url: Optional[str] = None
+    comment_count: Optional[int] = None
+
+
+class NewsArticle(BaseModel):
+    """A news article from the front page or article page."""
+    # from listing card
+    thread_id: int                  # XenForo thread ID
+    title: str
+    url: str                        # full URL to article
+    thumbnail_url: Optional[str] = None
+    author: Optional[str] = None
+    author_user_id: Optional[int] = None
+    published_at: Optional[str] = None  # ISO datetime string
+    reading_time: Optional[str] = None
+    # from article page (only when fetched individually)
+    views: Optional[int] = None
+    category: Optional[str] = None
+    body_html: Optional[str] = None
+    body_text: Optional[str] = None
+    mentioned_fanedit_urls: List[str] = Field(default_factory=list)

pyfanedit/parsers.py

@@ -0,0 +1,690 @@
+"""HTML parsers for fanedit.org pages."""
+from __future__ import annotations
+
+import re
+from typing import List, Optional, Tuple
+
+from bs4 import BeautifulSoup, Tag
+
+from pyfanedit.models import (
+    FaneditDetail, FaneditSummary, NewsArticle,
+    Review, ReviewRatings, ReviewerEntry, UserReviewEntry,
+)
+
+# Map of raw label text → FaneditDetail field name
+_DETAIL_FIELD_MAP = {
+    "faneditor name:": "faneditor",
+    "original movie/show title:": "original_title",
+    "fanedit type:": "fanedit_type",
+    "original release date:": "original_release_date",
+    "original running time:": "original_running_time",
+    "fanedit release date:": "fanedit_release_date",
+    "fanedit running time:": "fanedit_running_time",
+    "time cut:": "time_cut",
+    "time added:": "time_added",
+    "subtitles available:": "subtitles",
+    "available in:": "available_in",
+    "synopsis:": "synopsis",
+    "additional notes:": "additional_notes",
+    "special thanks:": "special_thanks",
+    "release information:": "release_information",
+    "cuts and additions:": "cuts_and_additions",
+    "intention:": "intention",
+    "awards:": "awards",
+}
+
+_SUMMARY_FIELD_MAP = {
+    "faneditor name:": "faneditor",
+    "original movie/show title:": "original_title",
+    "fanedit type:": "fanedit_type",
+    "fanedit release date:": "release_date",
+    "fanedit running time:": "running_time",
+    "synopsis:": "synopsis",
+    "franchise:": "franchise",
+}
+
+_REVIEW_RATING_MAP = {
+    "overall rating": "overall",
+    "audio/video quality": "audio_video_quality",
+    "audio editing": "audio_editing",
+    "visual editing": "visual_editing",
+    "narrative": "narrative",
+    "enjoyment": "enjoyment",
+}
+
+
+def _parse_overall_rating(el: Optional[Tag]) -> Tuple[Optional[float], Optional[int]]:
+    if el is None:
+        return None, None
+    val_el = el.find(class_="jrRatingValue")
+    if val_el is None:
+        return None, None
+    text = val_el.get_text(" ", strip=True)
+    count = None
+    count_m = re.search(r"\((\d+)\)", text)
+    if count_m:
+        count = int(count_m.group(1))
+    rating_m = re.match(r"[\d.]+", text)
+    rating = float(rating_m.group()) if rating_m else None
+    return rating, count
+
+
+def _cover_url(soup: BeautifulSoup) -> Optional[str]:
+    img = soup.find(class_="jrMediaPhoto")
+    if img is None:
+        return None
+    src = img.get("data-jr-src") or img.get("src", "")
+    return None if src.startswith("data:") else (src or None)
+
+
+def _imdb_id(soup: BeautifulSoup) -> Optional[str]:
+    for a in soup.find_all("a", href=True):
+        m = re.search(r"/(tt\d+)", a["href"])
+        if m:
+            return m.group(1)
+    return None
+
+
+def _wp_post_id(soup: BeautifulSoup) -> Optional[int]:
+    body = soup.find("body")
+    if body:
+        for cls in body.get("class", []):
+            m = re.match(r"postid-(\d+)", cls)
+            if m:
+                return int(m.group(1))
+    return None
+
+
+def _slug_from_url(url: str) -> str:
+    return url.rstrip("/").rsplit("/", 1)[-1]
+
+
+def _parse_review_ratings(rating_table: Tag) -> ReviewRatings:
+    kwargs: dict = {}
+    for row in rating_table.find_all(class_="fwd-table-row"):
+        cells = row.find_all(class_=["jrRatingLabel", "jrRatingValue"])
+        if len(cells) < 2:
+            continue
+        label = cells[0].get_text(strip=True).lower()
+        val_text = cells[-1].get_text(strip=True)
+        field = _REVIEW_RATING_MAP.get(label)
+        if field:
+            try:
+                kwargs[field] = float(val_text)
+            except ValueError:
+                pass
+    return ReviewRatings(**kwargs)
+
+
+def _parse_review(review_el: Tag) -> Review:
+    # reviewer identity (left panel)
+    reviewer = reviewer_url = reviewer_rank = None
+    reviewer_review_count = None
+    left = review_el.find(class_="jrReviewLayoutLeft")
+    if left:
+        author_el = left.find(itemprop="name")
+        if author_el:
+            reviewer = author_el.get_text(strip=True)
+        author_url_el = left.find(itemprop="url")
+        if author_url_el:
+            reviewer_url = author_url_el.get("href")
+        rank_el = left.find(class_="jrReviewerRank")
+        if rank_el:
+            reviewer_rank = rank_el.get_text(strip=True)
+        rev_count_el = left.find(class_="jrReviewerReviews")
+        if rev_count_el:
+            m = re.search(r"(\d+)", rev_count_el.get_text())
+            if m:
+                reviewer_review_count = int(m.group(1))
+
+    # date
+    date = None
+    time_el = review_el.find("time", class_="jrReviewCreated")
+    if time_el:
+        date = time_el.get("datetime") or time_el.get_text(strip=True)
+
+    # ratings
+    rating_table = review_el.find(class_="jrRatingTable")
+    ratings = _parse_review_ratings(rating_table) if rating_table else ReviewRatings()
+
+    # body text
+    body = None
+    comment_el = review_el.find(class_="jrReviewComment")
+    if comment_el:
+        body = comment_el.get_text(" ", strip=True)
+
+    # discussion / helpful
+    discussion_url = None
+    helpful_yes = helpful_no = None
+    footer = review_el.find(class_="jrReviewActions")
+    if footer:
+        discuss_a = footer.find(class_="jrDiscussReview")
+        if discuss_a:
+            discussion_url = discuss_a.get("href")
+        vote_el = footer.find(class_="jr-review-vote")
+        if vote_el:
+            yes_el = vote_el.find(class_="jrVoteYes")
+            no_el = vote_el.find(class_="jrVoteNo")
+            if yes_el:
+                try:
+                    helpful_yes = int(yes_el.find(class_="count-text").get_text(strip=True))
+                except (AttributeError, ValueError):
+                    pass
+            if no_el:
+                try:
+                    helpful_no = int(no_el.find(class_="count-text").get_text(strip=True))
+                except (AttributeError, ValueError):
+                    pass
+
+    return Review(
+        reviewer=reviewer,
+        reviewer_url=reviewer_url,
+        reviewer_rank=reviewer_rank,
+        reviewer_review_count=reviewer_review_count,
+        date=date,
+        ratings=ratings,
+        body=body,
+        discussion_url=discussion_url,
+        helpful_yes=helpful_yes,
+        helpful_no=helpful_no,
+    )
+
+
+def _parse_reviews(soup: BeautifulSoup, container_class: str) -> List[Review]:
+    section = soup.find(class_=container_class)
+    if section is None:
+        return []
+    reviews = []
+    for el in section.find_all(class_="jrReviewLayout"):
+        # skip the summary placeholder (no jrReviewLayoutLeft = no real reviewer)
+        if el.find(class_="jrReviewLayoutLeft") is None:
+            continue
+        reviews.append(_parse_review(el))
+    return reviews
+
+
+def _parse_outer(outer: Tag) -> Optional[FaneditSummary]:
+    title_el = outer.find(class_="jrListingTitle")
+    if title_el is None:
+        return None
+    a = title_el.find("a")
+    if a is None:
+        return None
+
+    title = a.get_text(strip=True)
+    url = a["href"]
+    slug = _slug_from_url(url)
+
+    # cover
+    cover_url: Optional[str] = None
+    thumb = outer.find(class_="jrListingThumbnail")
+    if thumb:
+        img = thumb.find("img")
+        if img:
+            src = img.get("data-jr-src") or img.get("src", "")
+            if not src.startswith("data:"):
+                cover_url = src
+
+    # ratings
+    editor_r, _ = _parse_overall_rating(outer.find(class_="jrOverallEditor"))
+    user_r, user_cnt = _parse_overall_rating(outer.find(class_="jrOverallUser"))
+
+    # views
+    views: Optional[int] = None
+    for span in outer.find_all("span"):
+        if span.find(class_="jrIconGraph"):
+            try:
+                views = int(span.get_text(strip=True))
+            except ValueError:
+                pass
+
+    # date
+    updated: Optional[str] = None
+    date_el = outer.find(class_="jrDateValue")
+    if date_el:
+        updated = date_el.get_text(strip=True)
+
+    # fanedit type from jrListingCategory (search results layout)
+    fanedit_type: Optional[str] = None
+    cat_el = outer.find(class_="jrListingCategory")
+    if cat_el:
+        fanedit_type = cat_el.get_text(strip=True)
+
+    # custom fields
+    kwargs: dict = {}
+    for row in outer.find_all(class_="jrFieldRow"):
+        lbl = row.find(class_="jrFieldLabel")
+        val = row.find(class_="jrFieldValue")
+        if lbl and val:
+            key = lbl.get_text(strip=True).lower()
+            mapped = _SUMMARY_FIELD_MAP.get(key)
+            if mapped:
+                kwargs[mapped] = val.get_text(" ", strip=True)
+
+    if fanedit_type and "fanedit_type" not in kwargs:
+        kwargs["fanedit_type"] = fanedit_type
+
+    return FaneditSummary(
+        slug=slug,
+        title=title,
+        url=url,
+        cover_url=cover_url,
+        editor_rating=editor_r,
+        user_rating=user_r,
+        user_rating_count=user_cnt,
+        views=views,
+        updated=updated,
+        **kwargs,
+    )
+
+
+def parse_listing_page(html: str) -> Tuple[List[FaneditSummary], Optional[str]]:
+    """Return (items, next_page_url) from a category/search listing page."""
+    soup = BeautifulSoup(html, "html.parser")
+    items: List[FaneditSummary] = []
+
+    # Category pages use "jr-layout-outer"; search/tag pages use "jrRow"
+    candidates = soup.find_all(class_="jr-layout-outer") or [
+        el for el in soup.find_all(class_="jrRow")
+        if "jrDataListHeader" not in (el.get("class") or [])
+    ]
+
+    for outer in candidates:
+        item = _parse_outer(outer)
+        if item:
+            items.append(item)
+
+    # next page
+    next_url: Optional[str] = None
+    pagenav = soup.find(class_="jrPagination")
+    if pagenav:
+        current = pagenav.find(class_="jrPageCurrent")
+        if current:
+            nxt = current.find_next_sibling("a")
+            if nxt and nxt.get("href"):
+                next_url = nxt["href"]
+
+    return items, next_url
+
+
+def parse_detail_page(html: str, url: str) -> FaneditDetail:
+    """Parse a single fanedit detail page."""
+    soup = BeautifulSoup(html, "html.parser")
+
+    # title
+    title_el = soup.find(class_="jrListingTitle") or soup.find("h1", class_="contentheading")
+    title = title_el.get_text(strip=True) if title_el else _slug_from_url(url)
+
+    fanedit_id = _wp_post_id(soup)
+    slug = _slug_from_url(url)
+    cover_url = _cover_url(soup)
+    imdb_id = _imdb_id(soup)
+
+    # ratings
+    editor_r, _ = _parse_overall_rating(soup.find(class_="jrOverallEditor"))
+    user_r, user_cnt = _parse_overall_rating(soup.find(class_="jrOverallUser"))
+
+    # fields — deduplicated by label
+    seen: dict = {}
+    for row in soup.find_all(class_="jrFieldRow"):
+        lbl = row.find(class_="jrFieldLabel")
+        val = row.find(class_="jrFieldValue")
+        if lbl and val:
+            key = lbl.get_text(strip=True).lower()
+            if key not in seen:
+                seen[key] = val
+
+    known: dict = {}
+    extra: dict = {}
+    for raw_key, val_el in seen.items():
+        mapped = _DETAIL_FIELD_MAP.get(raw_key)
+        text = val_el.get_text(" ", strip=True)
+        if mapped:
+            known[mapped] = text
+        elif raw_key == "genre:":
+            known["genre"] = [a.get_text(strip=True) for a in val_el.find_all("a")] or [text]
+        elif raw_key == "franchise:":
+            known["franchise"] = [a.get_text(strip=True) for a in val_el.find_all("a")] or [text]
+        else:
+            extra[raw_key] = text
+
+    editor_reviews = _parse_reviews(soup, "jrEditorReviewsContainer")
+    user_reviews = _parse_reviews(soup, "jrUserReviewsContainer")
+
+    return FaneditDetail(
+        fanedit_id=fanedit_id,
+        slug=slug,
+        title=title,
+        url=url,
+        cover_url=cover_url,
+        imdb_id=imdb_id,
+        editor_rating=editor_r,
+        user_rating=user_r,
+        user_rating_count=user_cnt,
+        editor_reviews=editor_reviews,
+        user_reviews=user_reviews,
+        extra_fields=extra,
+        **known,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Reviewer leaderboard
+# ---------------------------------------------------------------------------
+
+def parse_reviewer_rank_page(html: str) -> Tuple[List[ReviewerEntry], Optional[str]]:
+    """Parse one page of /reviewer-rank/. Returns (entries, next_page_url)."""
+    soup = BeautifulSoup(html, "html.parser")
+    entries: List[ReviewerEntry] = []
+
+    for row in soup.find_all(class_="jrRow"):
+        if "jrDataListHeader" in (row.get("class") or []):
+            continue
+
+        rank_col = row.find(class_="jrCenterAlign")
+        if rank_col is None:
+            continue
+        user_id_m = re.match(r"user-(\d+)", rank_col.get("id", ""))
+        if not user_id_m:
+            continue
+        user_id = int(user_id_m.group(1))
+        try:
+            rank = int(rank_col.get_text(strip=True))
+        except ValueError:
+            continue
+
+        # username + profile URL
+        author_el = row.find(class_="jrReviewAuthor")
+        if author_el is None:
+            continue
+        a = author_el.find("a")
+        username = a.get_text(strip=True) if a else ""
+        profile_url = a["href"] if a else ""
+
+        # review count + helpful votes
+        content = row.find(class_="jrRankContent")
+        review_count = 0
+        helpful_yes = None
+        helpful_pct = None
+        if content:
+            rev_a = content.find("a")
+            if rev_a:
+                reviews_url = rev_a["href"]
+                m = re.search(r"(\d+)", rev_a.get_text())
+                if m:
+                    review_count = int(m.group(1))
+            else:
+                reviews_url = f"https://fanedit.org/my-reviews/{user_id}/"
+            text = content.get_text(" ", strip=True)
+            hm = re.search(r"Helpful votes:\s*(\d+)\s*\(([0-9.]+)%\)", text)
+            if hm:
+                helpful_yes = int(hm.group(1))
+                helpful_pct = float(hm.group(2))
+        else:
+            reviews_url = f"https://fanedit.org/my-reviews/{user_id}/"
+
+        entries.append(ReviewerEntry(
+            rank=rank,
+            user_id=user_id,
+            username=username,
+            profile_url=profile_url,
+            reviews_url=reviews_url,
+            review_count=review_count,
+            helpful_yes=helpful_yes,
+            helpful_pct=helpful_pct,
+        ))
+
+    next_url: Optional[str] = None
+    pagenav = soup.find(class_="jrPagination")
+    if pagenav:
+        current = pagenav.find(class_="jrPageCurrent")
+        if current:
+            nxt = current.find_next_sibling("a")
+            if nxt and nxt.get("href"):
+                next_url = nxt["href"]
+
+    return entries, next_url
+
+
+# ---------------------------------------------------------------------------
+# Reviews by user
+# ---------------------------------------------------------------------------
+
+def parse_user_reviews_page(html: str) -> Tuple[List[UserReviewEntry], Optional[str]]:
+    """Parse one page of /my-reviews/{user_id}/. Returns (reviews, next_page_url)."""
+    soup = BeautifulSoup(html, "html.parser")
+    reviews: List[UserReviewEntry] = []
+
+    for el in soup.find_all(class_="jrReviewListLayout"):
+        # fanedit link
+        listing_title = el.find(class_="jrListingTitle")
+        if listing_title is None:
+            continue
+        a = listing_title.find("a")
+        if a is None:
+            continue
+        fanedit_url = a["href"]
+        if not fanedit_url.startswith("http"):
+            fanedit_url = "https://fanedit.org" + fanedit_url
+        fanedit_title = a.get_text(strip=True)
+
+        fanedit_type: Optional[str] = None
+        cat_el = el.find(class_="jrListingCategory")
+        if cat_el:
+            fanedit_type = cat_el.get_text(strip=True)
+
+        date: Optional[str] = None
+        date_el = el.find(class_="jrReviewCreated")
+        if date_el:
+            date = date_el.get("datetime") or date_el.get_text(strip=True)
+
+        rating_table = el.find(class_="jrRatingTable")
+        ratings = _parse_review_ratings(rating_table) if rating_table else ReviewRatings()
+
+        discussion_url: Optional[str] = None
+        comment_count: Optional[int] = None
+        for btn_a in el.find_all("a", class_="jrButton"):
+            href = btn_a.get("href", "")
+            if "/discussions/" in href:
+                discussion_url = href
+                m = re.search(r"Comments?\s*\((\d+)\)", btn_a.get_text())
+                if m:
+                    comment_count = int(m.group(1))
+
+        reviews.append(UserReviewEntry(
+            fanedit_title=fanedit_title,
+            fanedit_url=fanedit_url,
+            fanedit_type=fanedit_type,
+            date=date,
+            ratings=ratings,
+            discussion_url=discussion_url,
+            comment_count=comment_count,
+        ))
+
+    next_url: Optional[str] = None
+    pagenav = soup.find(class_="jrPagination")
+    if pagenav:
+        current = pagenav.find(class_="jrPageCurrent")
+        if current:
+            nxt = current.find_next_sibling("a")
+            if nxt and nxt.get("href"):
+                next_url = nxt["href"]
+
+    return reviews, next_url
+
+
+# ---------------------------------------------------------------------------
+# News
+# ---------------------------------------------------------------------------
+
+def _thread_id_from_card(card: Tag) -> Optional[int]:
+    for cls in card.get("class", []):
+        m = re.match(r"js-threadListItem-(\d+)", cls)
+        if m:
+            return int(m.group(1))
+    return None
+
+
+def parse_news_listing(html: str) -> List[NewsArticle]:
+    """Parse the news front page (/forums/news-publisher/)."""
+    soup = BeautifulSoup(html, "html.parser")
+    articles: List[NewsArticle] = []
+
+    for card in soup.find_all(class_="newsCard-grid-item"):
+        thread_id = _thread_id_from_card(card)
+        if thread_id is None:
+            continue
+
+        title_el = card.find(class_="newsCard-grid-title")
+        if title_el is None:
+            continue
+        a = title_el.find("a")
+        if a is None:
+            continue
+        title = a.get_text(strip=True)
+        href = a["href"]
+        url = href if href.startswith("http") else "https://fanedit.org" + href
+
+        thumbnail_url: Optional[str] = None
+        img = card.find("img", class_="newsCard-grid-image-link")
+        if img:
+            thumbnail_url = img.get("src")
+
+        author: Optional[str] = None
+        author_user_id: Optional[int] = None
+        avatar_a = card.find("a", attrs={"data-user-id": True})
+        if avatar_a:
+            author_img = avatar_a.find("img")
+            if author_img:
+                author = author_img.get("alt")
+            try:
+                author_user_id = int(avatar_a["data-user-id"])
+            except (ValueError, KeyError):
+                pass
+
+        published_at: Optional[str] = None
+        time_el = card.find("time")
+        if time_el:
+            published_at = time_el.get("datetime")
+
+        reading_time: Optional[str] = None
+        for li in card.find_all("li", class_="newsCard-date"):
+            text = li.get_text(strip=True)
+            if "min read" in text:
+                # strip SVG title prefix ("Reading time2 min read" → "2 min read")
+                m = re.search(r"(\d+\s*min read)", text)
+                reading_time = m.group(1) if m else text
+
+        articles.append(NewsArticle(
+            thread_id=thread_id,
+            title=title,
+            url=url,
+            thumbnail_url=thumbnail_url,
+            author=author,
+            author_user_id=author_user_id,
+            published_at=published_at,
+            reading_time=reading_time,
+        ))
+
+    return articles
+
+
+def parse_news_article(html: str, url: str) -> NewsArticle:
+    """Parse a single news article page."""
+    soup = BeautifulSoup(html, "html.parser")
+
+    # thread ID from article data attr
+    article_el = soup.find("article", class_="newsBody-main")
+    thread_id = 0
+    if article_el:
+        lb_id = article_el.get("data-lb-id", "")
+        m = re.search(r"(\d+)", lb_id)
+        if m:
+            thread_id = int(m.group(1))
+
+    # title
+    title_el = soup.find("h1", class_="p-title-value") or soup.find("h1")
+    title = title_el.get_text(strip=True) if title_el else ""
+
+    # thumbnail
+    thumbnail_url: Optional[str] = None
+    thumb_img = soup.find("img", class_="newsView-newsThumbnail-header")
+    if thumb_img:
+        thumbnail_url = thumb_img.get("src")
+
+    # author + published
+    author: Optional[str] = None
+    author_user_id: Optional[int] = None
+    published_at: Optional[str] = None
+    reading_time: Optional[str] = None
+    views: Optional[int] = None
+
+    desc = soup.find(class_="p-description")
+    if desc:
+        author_a = desc.find("a", attrs={"data-user-id": True})
+        if author_a:
+            author = author_a.get_text(strip=True)
+            try:
+                author_user_id = int(author_a["data-user-id"])
+            except (ValueError, KeyError):
+                pass
+        time_el = desc.find("time")
+        if time_el:
+            published_at = time_el.get("datetime")
+        for li in desc.find_all("li"):
+            text = li.get_text(strip=True)
+            if "min read" in text:
+                m = re.search(r"(\d+\s*min read)", text)
+                reading_time = m.group(1) if m else text
+
+    # view count from pairs--justified
+    for pair in soup.find_all(class_="pairs--justified"):
+        text = pair.get_text(" ", strip=True)
+        m = re.search(r"Views\s+([\d,]+)", text)
+        if m:
+            try:
+                views = int(m.group(1).replace(",", ""))
+            except ValueError:
+                pass
+
+    # category from breadcrumb
+    category: Optional[str] = None
+    crumbs = soup.find_all(class_="p-breadcrumbs")
+    if not crumbs:
+        crumbs = soup.find_all(attrs={"itemprop": "breadcrumb"})
+    if crumbs:
+        links = crumbs[-1].find_all("a") if crumbs else []
+        if links:
+            category = links[-1].get_text(strip=True)
+
+    # body
+    body_html: Optional[str] = None
+    body_text: Optional[str] = None
+    bb = soup.find(class_="bbWrapper")
+    if bb:
+        body_html = str(bb)
+        body_text = bb.get_text(" ", strip=True)
+
+    # mentioned IFDB fanedit URLs
+    mentioned: List[str] = []
+    if bb:
+        for a in bb.find_all("a", href=True):
+            href = a["href"]
+            if "fanedit.org" in href and "/forums/" not in href and href not in mentioned:
+                mentioned.append(href)
+
+    return NewsArticle(
+        thread_id=thread_id,
+        title=title,
+        url=url,
+        thumbnail_url=thumbnail_url,
+        author=author,
+        author_user_id=author_user_id,
+        published_at=published_at,
+        reading_time=reading_time,
+        views=views,
+        category=category,
+        body_html=body_html,
+        body_text=body_text,
+        mentioned_fanedit_urls=mentioned,
+    )

pyfanedit/session.py

@@ -0,0 +1,84 @@
+"""HTTP layer using curl_cffi to bypass TLS fingerprinting."""
+from __future__ import annotations
+
+import time
+from threading import Lock
+from typing import Any, Dict, Mapping, Optional, Tuple
+
+from curl_cffi import requests
+
+SITE_URL = "https://fanedit.org"
+_CacheKey = Tuple[str, str, Tuple[Tuple[str, str], ...]]
+
+
+class Session:
+    def __init__(
+        self,
+        impersonate: str = "chrome120",
+        cache_ttl: float = 300.0,
+        cache_size: int = 512,
+    ) -> None:
+        self._session = requests.Session(impersonate=impersonate)
+        self.cache_ttl = cache_ttl
+        self.cache_size = cache_size
+        self._cache: Dict[_CacheKey, Tuple[float, str]] = {}
+        self._lock = Lock()
+
+    def _key(self, url: str, params: Optional[Mapping[str, Any]]) -> _CacheKey:
+        items = tuple(sorted((k, str(v)) for k, v in (params or {}).items()))
+        return ("GET", url, items)
+
+    def _cache_get(self, key: _CacheKey) -> Optional[str]:
+        with self._lock:
+            hit = self._cache.get(key)
+        if hit is None:
+            return None
+        ts, text = hit
+        if self.cache_ttl and (time.time() - ts) > self.cache_ttl:
+            with self._lock:
+                self._cache.pop(key, None)
+            return None
+        return text
+
+    def _cache_put(self, key: _CacheKey, text: str) -> None:
+        if not self.cache_size:
+            return
+        with self._lock:
+            if len(self._cache) >= self.cache_size and self._cache:
+                oldest = min(self._cache.items(), key=lambda kv: kv[1][0])[0]
+                self._cache.pop(oldest, None)
+            self._cache[key] = (time.time(), text)
+
+    def get(
+        self,
+        path: str,
+        params: Optional[Mapping[str, Any]] = None,
+        use_cache: bool = True,
+    ) -> str:
+        url = path if path.startswith("http") else SITE_URL + "/" + path.lstrip("/")
+        key = self._key(url, params)
+        if use_cache:
+            cached = self._cache_get(key)
+            if cached is not None:
+                return cached
+        r = self._session.get(url, params=params, headers={
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+        })
+        r.raise_for_status()
+        if use_cache:
+            self._cache_put(key, r.text)
+        return r.text
+
+    def post(self, path: str, data: Optional[Mapping[str, Any]] = None) -> str:
+        url = path if path.startswith("http") else SITE_URL + "/" + path.lstrip("/")
+        r = self._session.post(url, data=data, headers={
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Referer": SITE_URL + "/fanedit-search/",
+        })
+        r.raise_for_status()
+        return r.text
+
+
+default_session = Session()

pyfanedit/version.py

@@ -0,0 +1,8 @@
+# START_VERSION_BLOCK
+VERSION_MAJOR = 0
+VERSION_MINOR = 1
+VERSION_BUILD = 1
+VERSION_ALPHA = 1
+# END_VERSION_BLOCK
+
+__version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + (f"a{VERSION_ALPHA}" if VERSION_ALPHA else "")

pyfanedit-0.1.1a1.dist-info/METADATA

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: pyfanedit
+Version: 0.1.1a1
+Summary: Scraping client for fanedit.org IFDB
+Author-email: JarbasAi <jarbasai@mailfence.com>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/OpenJarbas/pyfanedit
+Requires-Python: >=3.8
+Requires-Dist: beautifulsoup4
+Requires-Dist: curl_cffi
+Requires-Dist: pydantic>=2.0
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"

pyfanedit-0.1.1a1.dist-info/RECORD

@@ -0,0 +1,10 @@
+pyfanedit/__init__.py,sha256=Sqwkh5LofhWhxJAT9_dWIYvaP5KoqdGqmGtE2oeLgso,455
+pyfanedit/client.py,sha256=oFdXBz3lR-2XV68teHXcwfqphqfD26VPlr3fUPvJNYU,11162
+pyfanedit/models.py,sha256=EbTmHAEkXa1vr284lTTeiKjo4O97rATRf6lq3KgcT6A,4902
+pyfanedit/parsers.py,sha256=gEZd_LZOXRHU8y1Hw0i9hXiw0nb8vYY5joJh36CwPAc,22791
+pyfanedit/session.py,sha256=1dQQ3qNQFzeif3gvNau6lXAPeEGIdGWpcqLwMqti1i4,2891
+pyfanedit/version.py,sha256=SiSP9OTFnysxlBkTvA4uRCQo7NNiTz_VVAXwdU1qfzg,229
+pyfanedit-0.1.1a1.dist-info/METADATA,sha256=F5jpYUD9u1djkHlE0VOEHOHDoxsNhdxQQEdzSswZimk,399
+pyfanedit-0.1.1a1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+pyfanedit-0.1.1a1.dist-info/top_level.txt,sha256=h2lYuDjGN-I_aix96ORMaTbtuWo9Yvsx__E4dBNguyk,10
+pyfanedit-0.1.1a1.dist-info/RECORD,,

pyfanedit-0.1.1a1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pyfanedit-0.1.1a1.dist-info/top_level.txt

@@ -0,0 +1 @@
+pyfanedit