factorio-mod-workshop 0.1.0__py3-none-any.whl

fmw/portal.py

@@ -0,0 +1,336 @@
+"""Factorio mod portal access: official JSON API plus discussion scraping.
+
+The portal has a public read-only API for mod metadata, but discussions are
+HTML only. The markup is server-rendered with stable semantic classes, so
+BeautifulSoup is enough - no browser needed, no login needed to read.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from dataclasses import dataclass, field, asdict
+from typing import Iterable, NamedTuple
+
+import httpx
+from bs4 import BeautifulSoup
+
+BASE = "https://mods.factorio.com"
+API_BASE = f"{BASE}/api"
+USER_AGENT = "factorio-mod-workshop/0.1 (+https://github.com/AivanF/factorio-mod-workshop) mod-maintenance-tooling"
+
+THREAD_ID_RE = re.compile(r"/discussion/([0-9a-f]{24})")
+REQUEST_DELAY = 0.4  # be a polite citizen; the portal is a free service
+
+
+class PortalError(RuntimeError):
+    pass
+
+
+class NotFound(PortalError):
+    """The portal returned 404 - the mod or thread does not exist there."""
+
+
+@dataclass
+class Message:
+    author: str
+    posted_at: str          # ISO 8601, exact (from the title= attribute)
+    edited_at: str | None
+    body: str               # markdown-ish plain text
+    is_op: bool = False
+
+
+@dataclass
+class ThreadSummary:
+    id: str
+    mod: str
+    title: str
+    tag: str                # human name, e.g. "Bugs"
+    author: str
+    posted_at: str
+    reply_count: int
+    last_author: str | None
+    last_at: str | None
+
+    @property
+    def url(self) -> str:
+        return f"{BASE}/mod/{self.mod}/discussion/{self.id}"
+
+
+@dataclass
+class Thread:
+    id: str
+    mod: str
+    title: str
+    tag: str
+    messages: list[Message] = field(default_factory=list)
+
+    @property
+    def url(self) -> str:
+        return f"{BASE}/mod/{self.mod}/discussion/{self.id}"
+
+    def to_dict(self) -> dict:
+        d = asdict(self)
+        d["url"] = self.url
+        return d
+
+
+class Portal:
+    def __init__(self, timeout: float = 20.0, delay: float = REQUEST_DELAY):
+        self._client = httpx.Client(
+            headers={"User-Agent": USER_AGENT},
+            timeout=timeout,
+            follow_redirects=True,
+        )
+        self._delay = delay
+        self._last_request = 0.0
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> "Portal":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+    def _get(self, url: str) -> httpx.Response:
+        wait = self._delay - (time.monotonic() - self._last_request)
+        if wait > 0:
+            time.sleep(wait)
+        last_exc: Exception | None = None
+        for attempt in range(3):
+            try:
+                resp = self._client.get(url)
+                self._last_request = time.monotonic()
+                if resp.status_code == 404:
+                    raise NotFound(f"404 Not Found: {url}")
+                if resp.status_code >= 500:
+                    raise httpx.HTTPError(f"{resp.status_code} from {url}")
+                resp.raise_for_status()
+                return resp
+            except NotFound:
+                raise
+            except Exception as exc:  # network hiccup / 5xx
+                last_exc = exc
+                time.sleep(1.5 * (attempt + 1))
+        raise PortalError(f"Failed to fetch {url}: {last_exc}")
+
+    # ------------------------------------------------------------------
+    # Official API
+    # ------------------------------------------------------------------
+    def mod_info(self, mod: str) -> dict:
+        """Full mod metadata: releases, deps, changelog, downloads, tags."""
+        return self._get(f"{API_BASE}/mods/{mod}/full").json()
+
+    # ------------------------------------------------------------------
+    # Discussions (HTML)
+    # ------------------------------------------------------------------
+    def _soup(self, url: str) -> BeautifulSoup:
+        return BeautifulSoup(self._get(url).text, "html.parser")
+
+    def discussion_tags(self, mod: str) -> list[tuple[str, str]]:
+        """Return [(code, name)] of discussion tag filters offered for the mod."""
+        soup = self._soup(f"{BASE}/mod/{mod}/discussion")
+        return self._tags_from_soup(soup)
+
+    @staticmethod
+    def _tags_from_soup(soup: BeautifulSoup) -> list[tuple[str, str]]:
+        tags: list[tuple[str, str]] = []
+        holder = soup.select_one(".discussion-tags-filter")
+        if not holder:
+            return tags
+        for a in holder.select("a[href]"):
+            code = a["href"].rstrip("/").rsplit("/", 1)[-1]
+            name = a.get_text(strip=True)
+            if code and name:
+                tags.append((code, name))
+        return tags
+
+    def thread_list(self, mod: str, max_pages: int = 50) -> tuple[list[ThreadSummary], list[str]]:
+        """All discussion threads of a mod, plus a list of warnings.
+
+        The listing shows 50 threads per page and is paginated as
+        /discussion/page/N (note: a ?page=N query parameter is silently
+        ignored, which looks exactly like truncation). We walk pages until one
+        comes back empty.
+        """
+        warnings: list[str] = []
+        by_id: dict[str, ThreadSummary] = {}
+
+        for page in range(1, max_pages + 1):
+            url = f"{BASE}/mod/{mod}/discussion"
+            if page > 1:
+                url += f"/page/{page}"
+            rows = list(self._rows_from_soup(self._soup(url), mod))
+            if not rows:
+                break
+            for summary in rows:
+                by_id[summary.id] = summary
+        else:
+            warnings.append(
+                f"stopped after {max_pages} pages - there may be more threads"
+            )
+
+        ordered = sorted(by_id.values(), key=lambda t: t.last_at or t.posted_at, reverse=True)
+        return ordered, warnings
+
+    @staticmethod
+    def _rows_from_soup(soup: BeautifulSoup, mod: str) -> Iterable[ThreadSummary]:
+        for tr in soup.select("tr.discussion-list-message"):
+            link = tr.select_one(".discussion-list-message-title a[href]")
+            if not link:
+                continue
+            m = THREAD_ID_RE.search(link["href"])
+            if not m:
+                continue
+            tds = tr.find_all("td", recursive=False)
+
+            title_cell = tds[0]
+            author_a = title_cell.select_one('a[href^="/user/"]')
+            posted_span = title_cell.select_one("span[title]")
+
+            tag = tds[1].get_text(strip=True) if len(tds) >= 2 else ""
+
+            reply_count = 0
+            last_author = None
+            last_at = None
+            if len(tds) >= 3:
+                meta = tds[2]
+                count_m = re.search(r"(\d+)", meta.get_text(" ", strip=True))
+                if count_m:
+                    reply_count = int(count_m.group(1))
+                last_a = meta.select_one('a[href^="/user/"]')
+                if last_a:
+                    last_author = last_a.get_text(strip=True)
+                last_span = meta.select_one("span[title]")
+                if last_span:
+                    last_at = last_span["title"]
+
+            posted_at = posted_span["title"] if posted_span else ""
+            yield ThreadSummary(
+                id=m.group(1),
+                mod=mod,
+                title=link.get("title") or link.get_text(strip=True),
+                tag=tag or "",
+                author=author_a.get_text(strip=True) if author_a else "",
+                posted_at=posted_at,
+                reply_count=reply_count,
+                last_author=last_author,
+                last_at=last_at or posted_at,
+            )
+
+    def thread(self, mod: str, thread_id: str) -> Thread:
+        soup = self._soup(f"{BASE}/mod/{mod}/discussion/{thread_id}")
+        title_el = soup.select_one(".discussion-detail-title")
+        if not title_el:
+            raise PortalError(f"Not a discussion page: {mod}/{thread_id}")
+
+        tag = ""
+        badge = title_el.select_one(".discussion-message-tag")
+        if badge:
+            tag = badge.get("title") or badge.get_text(strip=True)
+            badge.extract()
+        title = title_el.get_text(" ", strip=True)
+
+        messages: list[Message] = []
+        for i, block in enumerate(soup.select(".discussion-message")):
+            author_el = block.select_one(".discussion-message-author a")
+            body_el = block.select_one(".discussion-message-body")
+            if not author_el or body_el is None:
+                continue
+            stamps = [d["title"] for d in block.select(".ml8.grey[title]")]
+            messages.append(
+                Message(
+                    author=author_el.get_text(strip=True),
+                    posted_at=stamps[0] if stamps else "",
+                    edited_at=stamps[1] if len(stamps) > 1 else None,
+                    body=html_to_text(body_el),
+                    is_op=(i == 0),
+                )
+            )
+        return Thread(id=thread_id, mod=mod, title=title, tag=tag, messages=messages)
+
+
+def html_to_text(el) -> str:
+    """Flatten a discussion body to readable plain text, keeping line breaks."""
+    # A <br> is usually followed by a literal newline in the source, so replacing
+    # it with "\n" directly would double every line break inside a paragraph.
+    # Use a sentinel and swallow the surrounding whitespace instead.
+    for br in el.find_all("br"):
+        br.replace_with("\x00")
+    for p in el.find_all("p"):
+        p.insert_before("\n\n")
+        p.insert_after("\n\n")
+    text = el.get_text()
+    text = re.sub(r"[ \t]*\x00[ \t]*\n?", "\n", text)
+    text = re.sub(r"[ \t]+\n", "\n", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
+
+
+# Portal tag names are long and truncating them mid-word ("Gener", "Ideas")
+# reads badly, so display uses fixed abbreviations instead.
+TAG_ABBREV = {
+    "Bugs": "bug",
+    "General": "gen",
+    "Ideas & suggestions": "idea",
+    "Announcements": "ann",
+}
+
+
+def abbrev_tag(tag: str) -> str:
+    return TAG_ABBREV.get(tag, (tag or "?").lower()[:4])
+
+
+# ----------------------------------------------------------------------
+# Title markers
+# ----------------------------------------------------------------------
+# The portal has no thread state, so maintainers annotate the title once they
+# have dealt with a thread. The convention varies: this repository's own mods
+# use "[FIXED] title" on some mods and "title (fixed)" on others, so both
+# bracket styles and both positions have to be accepted. Derived at read time
+# rather than stored, so editing a title on the portal takes effect without a
+# re-sync.
+MARKER_PREFIX_RE = re.compile(r"^\s*[\[(]\s*([^\])]{1,24}?)\s*[\])]\s*")
+MARKER_SUFFIX_RE = re.compile(r"\s*[\[(]\s*([^\])]{1,24}?)\s*[\])]\s*$")
+
+RESOLVED_MARKERS = {
+    "FIXED", "DONE", "SOLVED", "CLOSED", "EXPLAINED", "ANSWERED",
+    "RELEASED", "WONTFIX", "DUPLICATE", "INVALID", "OUTDATED",
+}
+# Not a resolution: the maintainer judged the report itself mistaken.
+WRONG_MARKERS = {"WRONG", "INVALID REPORT", "NOT A BUG"}
+
+
+class TitleMarker(NamedTuple):
+    marker: str | None      # the raw text found, e.g. "FIXED" or "fixed?"
+    resolved: bool          # maintainer considers the thread dealt with
+    wrong: bool             # maintainer considers the report mistaken
+    clean: str              # title with the marker removed
+
+
+def parse_marker(title: str) -> TitleMarker:
+    """Split a thread title into its maintainer marker and the rest.
+
+    A trailing question mark ("[fixed?]") means the maintainer was unsure, so
+    it does not count as resolved.
+    """
+    title = title or ""
+    for regex in (MARKER_PREFIX_RE, MARKER_SUFFIX_RE):
+        m = regex.search(title)
+        if not m:
+            continue
+        raw = m.group(1).strip()
+        key = raw.rstrip("?").strip().upper()
+        if key not in RESOLVED_MARKERS and key not in WRONG_MARKERS:
+            continue  # an ordinary parenthetical, e.g. "Crash (Space Exploration)"
+        tentative = raw.endswith("?")
+        clean = (title[: m.start()] + title[m.end():]).strip()
+        return TitleMarker(
+            marker=raw,
+            resolved=key in RESOLVED_MARKERS and not tentative,
+            wrong=key in WRONG_MARKERS,
+            clean=clean,
+        )
+    return TitleMarker(marker=None, resolved=False, wrong=False, clean=title.strip())

fmw/store.py

@@ -0,0 +1,141 @@
+"""Workspace persistence: cached portal snapshots and change detection."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .config import Workspace
+from .portal import Thread, ThreadSummary
+
+
+def now_iso() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+
+def _write_json(path: Path, data: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2, ensure_ascii=False, sort_keys=False) + "\n")
+
+
+def _read_json(path: Path) -> dict | None:
+    if not path.exists():
+        return None
+    return json.loads(path.read_text())
+
+
+# ----------------------------------------------------------------------
+# Thread list snapshots
+# ----------------------------------------------------------------------
+def load_list(ws: Workspace, mod: str) -> dict | None:
+    return _read_json(ws.list_file(mod))
+
+
+def save_list(ws: Workspace, mod: str, threads: list[ThreadSummary], warnings: list[str]) -> None:
+    _write_json(
+        ws.list_file(mod),
+        {
+            "mod": mod,
+            "fetched_at": now_iso(),
+            "warnings": warnings,
+            "threads": [asdict(t) for t in threads],
+        },
+    )
+
+
+def load_thread(ws: Workspace, mod: str, thread_id: str) -> dict | None:
+    return _read_json(ws.thread_file(mod, thread_id))
+
+
+def save_thread(ws: Workspace, thread: Thread) -> None:
+    data = thread.to_dict()
+    data["fetched_at"] = now_iso()
+    _write_json(ws.thread_file(thread.mod, thread.id), data)
+
+
+def cached_thread_ids(ws: Workspace, mod: str) -> set[str]:
+    d = ws.mod_cache_dir(mod) / "threads"
+    if not d.is_dir():
+        return set()
+    return {p.stem for p in d.glob("*.json")}
+
+
+# ----------------------------------------------------------------------
+# Change detection
+# ----------------------------------------------------------------------
+def diff_lists(old: dict | None, new_threads: list[ThreadSummary]) -> list[dict]:
+    """Compare a freshly fetched thread list against the cached snapshot."""
+    previous = {t["id"]: t for t in (old or {}).get("threads", [])}
+    changes: list[dict] = []
+    for t in new_threads:
+        prev = previous.get(t.id)
+        if prev is None:
+            changes.append(
+                {
+                    "kind": "new_thread",
+                    "id": t.id,
+                    "title": t.title,
+                    "tag": t.tag,
+                    "author": t.author,
+                    "at": t.posted_at,
+                    "replies": t.reply_count,
+                }
+            )
+            continue
+        if t.reply_count != prev.get("reply_count") or (t.last_at or "") != (prev.get("last_at") or ""):
+            changes.append(
+                {
+                    "kind": "new_replies",
+                    "id": t.id,
+                    "title": t.title,
+                    "tag": t.tag,
+                    "author": t.last_author,
+                    "at": t.last_at,
+                    "replies": t.reply_count,
+                    "replies_before": prev.get("reply_count"),
+                }
+            )
+    return changes
+
+
+def append_changelog(ws: Workspace, mod: str, changes: list[dict]) -> None:
+    if not changes:
+        return
+    path = ws.mod_cache_dir(mod) / "changelog.jsonl"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    stamp = now_iso()
+    with path.open("a") as fh:
+        for c in changes:
+            fh.write(json.dumps({"seen_at": stamp, "mod": mod, **c}, ensure_ascii=False) + "\n")
+
+
+def read_changelog(ws: Workspace, mod: str, since: str | None = None) -> list[dict]:
+    path = ws.mod_cache_dir(mod) / "changelog.jsonl"
+    if not path.exists():
+        return []
+    out = []
+    for line in path.read_text().splitlines():
+        if not line.strip():
+            continue
+        rec = json.loads(line)
+        if since and rec.get("seen_at", "") < since:
+            continue
+        out.append(rec)
+    return out
+
+
+def find_thread_mod(ws: Workspace, thread_id: str) -> str | None:
+    """Locate which mod a cached thread belongs to, given only its id."""
+    if not ws.cache_dir.is_dir():
+        return None
+    for mod_dir in sorted(ws.cache_dir.iterdir()):
+        if (mod_dir / "threads" / f"{thread_id}.json").exists():
+            return mod_dir.name
+    # fall back to the list snapshots, in case the thread body was never fetched
+    for mod_dir in sorted(ws.cache_dir.iterdir()):
+        data = _read_json(mod_dir / "threads.json")
+        if data and any(t["id"] == thread_id for t in data.get("threads", [])):
+            return mod_dir.name
+    return None

fmw/templates/workspace/.gitignore

@@ -0,0 +1,5 @@
+# Machine-specific: where your mods are checked out, and your local settings.
+# Everything else here - the registry, the tracker, the cache - is shareable,
+# so a workspace can be a git repository several people collaborate in.
+config.yaml
+paths.yaml

fmw/templates/workspace/README.md

@@ -0,0 +1,31 @@
+# fmw workspace
+
+This directory holds all mod-specific and personal data for the
+[factorio-mod-workshop](https://github.com/AivanF/factorio-mod-workshop) toolkit.
+It is intentionally kept out of the tool repository so the tooling can be shared
+and the data can stay private.
+
+Make it its own git repository:
+
+```sh
+git init
+git add -A && git commit -m "workspace"
+```
+
+## Layout
+
+| Path | Owner | Shared? | Contents |
+| --- | --- | --- | --- |
+| `config.yaml` | you | no, gitignored | which directories to scan, Discord details |
+| `paths.yaml` | `fmw discover` | no, gitignored | where each mod is checked out on this machine |
+| `mods.yaml` | `fmw discover` + you | yes | mod registry, `maintained` and `notes` are yours |
+| `cache/<mod>/threads.json` | tool | yes | snapshot of the discussion list |
+| `cache/<mod>/threads/<id>.json` | tool | yes | full thread contents |
+| `cache/<mod>/changelog.jsonl` | tool | yes | append-only log of detected changes |
+| `threads/<mod>.yaml` | you + agents | yes | your judgement about each conversation |
+| `tracker/<mod>/*.md` | you + agents | yes | tickets: code changes, error or feature |
+| `notes/` | you | yes | anything else |
+
+Everything except `config.yaml` and `paths.yaml` is machine-independent, so a workspace can be a git repository that several maintainers collaborate in. Each of them copies `config.example.yaml` to `config.yaml` and runs `fmw discover` to generate their own `paths.yaml`.
+
+Never hand-edit `cache/` - it is overwritten on every `fmw sync`.

fmw/templates/workspace/config.example.yaml

@@ -0,0 +1,11 @@
+# Copy to config.yaml and edit. Machine-specific, so it is gitignored.
+
+# Directories scanned by `fmw discover` for mods (any dir containing info.json).
+mod_repos:
+  - ~/path/to/your/factorio-mods-monorepo
+  - ~/path/to/another-mod-repo
+
+# Optional: where you talk to users outside the mod portal.
+discord:
+  invite: ""
+  notes: ""

fmw/templates/workspace/config.yaml

@@ -0,0 +1,11 @@
+# Local, machine-specific configuration. Never committed to the public repo.
+
+# Directories scanned by `fmw discover` for mods (any dir containing info.json).
+mod_repos:
+  - ~/path/to/your/factorio-mods-monorepo
+  - ~/path/to/another-mod-repo
+
+# Optional: where you talk to users outside the mod portal.
+discord:
+  invite: ""
+  notes: ""

fmw/templates/workspace/mods.yaml

@@ -0,0 +1,6 @@
+# Mod registry. Regenerated (merged, never clobbered) by `fmw discover`.
+# Fields you own and should edit by hand:
+#   maintained: active | frozen | unknown   (frozen mods are skipped unless --all)
+#   notes:      free text, read by agents before touching the mod
+#   priority:   optional integer, higher = more attention
+mods: {}

fmw/templates/workspace/tracker/README.md

@@ -0,0 +1,40 @@
+# Tracker
+
+One Markdown file per item, `tracker/<mod>/<slug>.md`.
+
+```markdown
+---
+id: a3f9c1           # random, stable, never changes; references use this alone
+mods: [Mining_Drones_Remastered]   # first is primary and decides the directory
+kind: error          # error | feature - tickets are code changes only
+status: confirmed    # new | needs-info | confirmed | in-progress | needs-testing | fixed | released | wontfix | duplicate
+severity: crash      # optional: crash | major | minor | trivial
+title: Unknown flag "hidden" when disabling vanilla drills
+created: 2026-09-01
+updated: 2026-09-01
+affects: ["2.2.0"]
+fixed_in: ""
+sources:
+  - portal:6a963b260970b611cd38bc3b
+  - discord:https://discord.com/channels/...
+  - self
+related: []
+duplicate_of: ""
+---
+
+## Summary
+## Analysis
+## Resolution
+## Test plan
+```
+
+Every enum value is listed in the toolkit's `docs/workspace.md`, under "Field values", along with which ones the code actually acts on.
+
+Rules that keep the tooling useful:
+
+- `sources:` is the join key. `portal:<thread-id>` entries are what `fmw inbox`, the derived reply obligation and the ticket lines in `fmw threads` look at, so always record them.
+- Several reports of the same problem belong in **one** item with several sources, not in several items. Use `duplicate_of` only when a separate item already exists.
+- Ideas live here as `kind: feature`, whether a user asked or you thought of it. Larger roadmap notes belong in `../notes/`, not here.
+- Threads that need an answer rather than a change never become tickets. Use `fmw mark` instead.
+- Update `status` and `fixed_in` when you release, so the next agent does not re-investigate a solved problem.
+- `needs-testing` means written but not verified in game - the furthest an AI agent can take a fix on its own. Fill in the `## Test plan` section when setting it, and run `fmw testplan` to check a whole batch in one Factorio session.