rti-mcp 0.3.2__py3-none-any.whl

rti_mcp/__init__.py

@@ -0,0 +1,18 @@
+"""MCP server for querying the RTI Online portal (rtionline.gov.in) via a
+long-lived "View History" session URL."""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _dist_version
+
+try:
+    # Installed (including `pip install -e .`): the built metadata is the
+    # authority, so this reports exactly what pip resolved.
+    __version__ = _dist_version("rti-mcp")
+except PackageNotFoundError:  # pragma: no cover - running from a bare checkout
+    try:
+        # Generated by setuptools-scm at build time; absent until then.
+        from ._version import __version__
+    except ImportError:
+        __version__ = "0+unknown"
+
+__all__ = ["__version__"]

rti_mcp/__main__.py

@@ -0,0 +1,3 @@
+from .server import main
+
+main()

rti_mcp/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.3.2'
+__version_tuple__ = version_tuple = (0, 3, 2)
+
+__commit_id__ = commit_id = 'gbf61f29c6'

rti_mcp/client.py

@@ -0,0 +1,636 @@
+"""Scraper for the RTI Online citizen "View History" area.
+
+Everything hangs off one seed URL -- the tokenised `citizen_view_history.php`
+link the portal hands out after an OTP + captcha login. That URL authenticates
+itself (`emailchk`/`cellchk`/`urletoken` are server-side encrypted blobs), so
+no cookie jar has to be carried over from the browser.
+
+Page graph
+----------
+    citizen_view_history.php          dashboard: 6 counts + 6 drill-down links
+      -> list_action_status_new.php   every application in a (category, status)
+                                      bucket, in one table, no paging
+           -> regdetails.php          full application: PA, applicant, text
+           -> finalstatus_viewhistory.php
+                                      current status, remarks, reply document
+                -> viewPDF.php        reply PDF
+                -> pdfDocument.php    document attached to the request
+
+Navigation rules (established empirically -- the portal enforces them with
+bare 403s, not error pages):
+
+1. A list fetch mints `regId`/`token` params for that page's rows only.
+   Fetching any other list invalidates the previous page's links, so detail
+   links cannot be cached across runs.
+2. Several detail pages may be read off one list fetch, but only while that
+   list fetch is the most recent *successful* navigation.
+3. Any 403 poisons the session: the next request fails too, whatever it is.
+   Re-walking seed -> list clears it.
+
+`_walk_to` and `_fetch_detail` encode those rules; callers just ask for an
+application and get its details.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+from collections.abc import Iterable
+from dataclasses import asdict, dataclass, field
+from typing import Any
+from urllib.parse import parse_qs, urljoin, urlparse
+
+import requests
+from bs4 import BeautifulSoup
+
+from . import config
+
+CATEGORIES = ("request", "appeal")
+STATUSES = ("registered", "disposed", "pending")
+
+# Labels as the portal renders them, normalised to our status keys.
+_STATUS_LABELS = {
+    "registered": "registered",
+    "disposed of": "disposed",
+    "disposedof": "disposed",
+    "pending": "pending",
+}
+
+# e.g. QCCBN/R/E/26/00003; transferred requests carry hop suffixes:
+# BSNKN/R/E/26/00071/23. Authority codes are not plain alphanumerics -- some
+# carry punctuation from the department's initials (DOP&T, DOA&C), so those
+# characters have to be allowed or the rows are dropped as unrecognised.
+_REG_NO_RE = re.compile(r"^[A-Z0-9&.\-]{3,12}/[A-Z]/[A-Z]/\d{2,4}/\d+(?:/\d+)*$")
+_STATUS_DATE_RE = re.compile(r"^(.*?)\s*\((\d{2}/\d{2}/\d{4})\)\s*$", re.S)
+
+MIN_REQUEST_GAP = 0.6  # be polite to a government server
+NETWORK_RETRIES = 3
+RETRY_STATUS = {429, 500, 502, 503, 504}
+
+
+class RTIError(RuntimeError):
+    """Any failure talking to the portal."""
+
+
+class Blocked(RTIError):
+    """A 403 -- the navigation token was stale, or the session is poisoned."""
+
+
+class SessionExpired(RTIError):
+    """The seed URL no longer authenticates."""
+
+    def __init__(self, detail: str = "") -> None:
+        super().__init__(
+            "The RTI Online session URL is no longer valid"
+            + (f" ({detail})" if detail else "")
+            + ". Log in at https://rtionline.gov.in with OTP + captcha, click "
+            "'View History', copy the whole URL from the address bar, and pass "
+            "it to the `rti_set_session_url` tool."
+        )
+
+
+def _clean(text: str) -> str:
+    return re.sub(r"\s+", " ", text.replace("\xa0", " ")).strip()
+
+
+def _cell_text(cell) -> str:
+    return _clean(cell.get_text(" ", strip=True))
+
+
+@dataclass
+class Application:
+    """One row of an application list, plus the links needed to drill in.
+
+    `details_url`/`status_url` are only usable in the session that fetched
+    them, and only until the next list fetch -- see the module docstring.
+    """
+
+    registration_number: str
+    applicant: str
+    date_of_receipt: str  # ISO, e.g. 2026-08-06
+    status: str
+    status_date: str  # dd/mm/yyyy
+    category: str  # request | appeal
+    bucket: str  # registered | disposed | pending
+    details_url: str = ""
+    status_url: str = ""
+
+    def public(self) -> dict[str, Any]:
+        d = asdict(self)
+        d.pop("details_url", None)
+        d.pop("status_url", None)
+        return d
+
+
+@dataclass
+class Dashboard:
+    user_name: str = ""
+    email: str = ""
+    as_on: str = ""
+    counts: dict[str, dict[str, int]] = field(default_factory=dict)
+    list_urls: dict[str, dict[str, str]] = field(default_factory=dict)
+
+    def public(self) -> dict[str, Any]:
+        return {
+            "user_name": self.user_name,
+            "email": self.email,
+            "as_on": self.as_on,
+            "counts": self.counts,
+        }
+
+
+class RTIClient:
+    def __init__(self, session_url: str | None = None, ttl: int | None = None) -> None:
+        self.session_url = (session_url or config.get_session_url() or "").strip()
+        if not self.session_url:
+            raise RTIError(
+                "No RTI session URL configured. Set the RTI_HISTORY_URL environment "
+                "variable, or call the `rti_set_session_url` tool with the "
+                "citizen_view_history.php URL from your logged-in browser."
+            )
+        self.ttl = config.DEFAULT_TTL if ttl is None else ttl
+        self.http = requests.Session()
+        self.http.headers.update(
+            {
+                "User-Agent": config.USER_AGENT,
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.9",
+            }
+        )
+        self._dashboard: Dashboard | None = None
+        self._last_request = 0.0
+        # Which list the session's live tokens currently belong to, and the
+        # rows parsed from that fetch.
+        self._live_key: tuple[str, str] | None = None
+        self._live_rows: list[Application] = []
+        # Table rows whose registration number did not parse, per (category,
+        # bucket). A silently-dropped row is indistinguishable from an
+        # application that does not exist, so they are kept and reported.
+        self._unparsed: dict[tuple[str, str], list[str]] = {}
+
+    # ----------------------------------------------------------------- http
+
+    def _get(self, url: str, *, timeout: int = 120) -> requests.Response:
+        gap = time.monotonic() - self._last_request
+        if gap < MIN_REQUEST_GAP:
+            time.sleep(MIN_REQUEST_GAP - gap)
+
+        last = ""
+        for attempt in range(NETWORK_RETRIES):
+            try:
+                resp = self.http.get(url, timeout=timeout)
+            except requests.RequestException as exc:
+                last = f"network error: {exc}"
+            else:
+                self._last_request = time.monotonic()
+                if resp.status_code == 200:
+                    return resp
+                if resp.status_code == 403:
+                    # Token/navigation problem, not something a plain retry
+                    # fixes -- the caller has to re-walk.
+                    self._live_key = None
+                    raise Blocked(f"HTTP 403 for {url}")
+                last = f"HTTP {resp.status_code}"
+                if resp.status_code not in RETRY_STATUS:
+                    break
+            self._last_request = time.monotonic()
+            if attempt < NETWORK_RETRIES - 1:
+                time.sleep(2.0 * (2**attempt))
+        raise RTIError(f"RTI Online request failed ({last}): {url}")
+
+    def _get_html(self, url: str, *, timeout: int = 120) -> tuple[str, str]:
+        """Return (html, final_url). Raises SessionExpired on a bounce to login."""
+        resp = self._get(url, timeout=timeout)
+        html = resp.text
+        if ("login.php" in resp.url or "captcha.php" in html.lower()) and (
+            "UserName" not in html
+        ):
+            raise SessionExpired("redirected to the login page")
+        return html, resp.url
+
+    # ------------------------------------------------------------ dashboard
+
+    def dashboard(self, refresh: bool = False) -> Dashboard:
+        """Fetch the seed page. Also resets the portal's navigation state."""
+        if self._dashboard is not None and not refresh:
+            return self._dashboard
+
+        html, page_url = self._get_html(self.session_url)
+        self._live_key = None
+        self._live_rows = []
+
+        soup = BeautifulSoup(html, "html.parser")
+        text = _clean(soup.get_text(" ", strip=True))
+
+        dash = Dashboard()
+        if m := re.search(r"UserName\s*:-\s*(.*?)\s*Email\s*:-\s*(\S+@\S+)", text):
+            dash.user_name, dash.email = m.group(1), m.group(2)
+        if m := re.search(r"Status as on\s*(\d{2}-\d{2}-\d{4})", text):
+            dash.as_on = m.group(1)
+
+        # Drill-down links sit in href, or inside redirectUrl('...') on an
+        # onclick. Group by the `action` param: first distinct value is
+        # Requests, second is Appeals.
+        anchors: list[tuple[str, str]] = []
+        for a in soup.find_all("a"):
+            raw = a.get("href") or a.get("onclick") or ""
+            if m := re.search(r"(list_action_status_new\.php\?[^'\"()]+)", raw):
+                anchors.append((_clean(a.get_text(" ", strip=True)), m.group(1)))
+
+        if not anchors or not dash.user_name:
+            raise SessionExpired("the dashboard did not render an application summary")
+
+        order: list[str] = []
+        for _, href in anchors:
+            action = parse_qs(urlparse(href).query).get("action", [""])[0]
+            if action not in order:
+                order.append(action)
+
+        for label, href in anchors:
+            action = parse_qs(urlparse(href).query).get("action", [""])[0]
+            idx = order.index(action)
+            if idx > 1:
+                continue
+            category = CATEGORIES[idx]
+            count = None
+            key = _STATUS_LABELS.get(label.lower().strip())
+            if key is None:
+                if cm := re.fullmatch(r"\[(\d+)\]", label):
+                    # A count anchor shares its href with its label anchor.
+                    count = int(cm.group(1))
+                    key = self._status_for_href(anchors, href)
+                if key is None:
+                    continue
+            dash.list_urls.setdefault(category, {})[key] = urljoin(
+                page_url, href.replace("&amp;", "&")
+            )
+            if count is not None:
+                dash.counts.setdefault(category, {})[key] = count
+
+        for category in CATEGORIES:
+            dash.counts.setdefault(category, {})
+            dash.list_urls.setdefault(category, {})
+
+        self._dashboard = dash
+        return dash
+
+    @staticmethod
+    def _status_for_href(anchors: Iterable[tuple[str, str]], href: str) -> str | None:
+        for label, other in anchors:
+            if other == href:
+                if key := _STATUS_LABELS.get(label.lower().strip()):
+                    return key
+        return None
+
+    # ---------------------------------------------------------------- cache
+
+    def _cache_path(self, category: str, bucket: str):
+        return config.CACHE_DIR / f"{category}_{bucket}.json"
+
+    def _read_cache(self, category: str, bucket: str) -> list[Application] | None:
+        try:
+            blob = json.loads(
+                self._cache_path(category, bucket).read_text(encoding="utf-8")
+            )
+        except (OSError, ValueError):
+            return None
+        if blob.get("session_url") != self.session_url:
+            return None
+        if time.time() - blob.get("fetched_at", 0) > self.ttl:
+            return None
+        rows = []
+        for row in blob.get("rows", []):
+            app = Application(**row)
+            # Cached tokens are dead by definition; drop them so nothing tries
+            # to reuse them.
+            app.details_url = app.status_url = ""
+            rows.append(app)
+        self._unparsed[(category, bucket)] = blob.get("unparsed", [])
+        return rows
+
+    def _write_cache(self, category: str, bucket: str, rows: list[Application]) -> None:
+        config.CACHE_DIR.mkdir(parents=True, exist_ok=True)
+        self._cache_path(category, bucket).write_text(
+            json.dumps(
+                {
+                    "session_url": self.session_url,
+                    "fetched_at": time.time(),
+                    "rows": [asdict(r) for r in rows],
+                    "unparsed": self._unparsed.get((category, bucket), []),
+                },
+                indent=1,
+            ),
+            encoding="utf-8",
+        )
+
+    def cache_age(self, category: str, bucket: str) -> float | None:
+        try:
+            blob = json.loads(
+                self._cache_path(category, bucket).read_text(encoding="utf-8")
+            )
+        except (OSError, ValueError):
+            return None
+        return time.time() - blob.get("fetched_at", 0)
+
+    def unparsed_rows(self) -> dict[str, list[str]]:
+        """Rows seen in a list table but not recognised as applications.
+
+        Should always be empty. Anything here means the portal changed its
+        registration-number format and those applications are missing from
+        every result the server returns.
+        """
+        found = {
+            f"{category}/{bucket}": regs
+            for (category, bucket), regs in self._unparsed.items()
+            if regs
+        }
+        # Also read what earlier runs persisted, so this reports honestly even
+        # when no list has been fetched in this process.
+        for category in CATEGORIES:
+            for bucket in STATUSES:
+                key = f"{category}/{bucket}"
+                if key in found:
+                    continue
+                try:
+                    blob = json.loads(
+                        self._cache_path(category, bucket).read_text(encoding="utf-8")
+                    )
+                except (OSError, ValueError):
+                    continue
+                if regs := blob.get("unparsed"):
+                    found[key] = regs
+        return found
+
+    def clear_cache(self) -> int:
+        removed = 0
+        if config.CACHE_DIR.exists():
+            for path in config.CACHE_DIR.glob("*.json"):
+                path.unlink()
+                removed += 1
+        self._dashboard = None
+        self._live_key = None
+        self._live_rows = []
+        self._unparsed = {}
+        return removed
+
+    # ----------------------------------------------------------------- list
+
+    def _fetch_list(self, category: str, bucket: str) -> list[Application]:
+        """Fetch a bucket's list page, making its row links the live ones."""
+        dash = self.dashboard()
+        url = dash.list_urls.get(category, {}).get(bucket)
+        if not url:
+            raise RTIError(f"No {bucket} {category} list link on the dashboard")
+        html, page_url = self._get_html(url, timeout=240)
+        rows = self._parse_list(html, page_url, category, bucket)
+        self._live_key = (category, bucket)
+        self._live_rows = rows
+        self._write_cache(category, bucket, rows)
+        return rows
+
+    def applications(
+        self, category: str, bucket: str, refresh: bool = False
+    ) -> list[Application]:
+        category, bucket = category.lower(), bucket.lower()
+        if category not in CATEGORIES:
+            raise RTIError(f"category must be one of {CATEGORIES}, got {category!r}")
+        if bucket not in STATUSES:
+            raise RTIError(f"status must be one of {STATUSES}, got {bucket!r}")
+        if not refresh:
+            if cached := self._read_cache(category, bucket):
+                return cached
+        return self._fetch_list(category, bucket)
+
+    def _parse_list(
+        self, html: str, page_url: str, category: str, bucket: str
+    ) -> list[Application]:
+        soup = BeautifulSoup(html, "html.parser")
+        out: list[Application] = []
+        unparsed: list[str] = []
+        for tr in soup.find_all("tr"):
+            cells = tr.find_all("td")
+            if len(cells) < 5:
+                continue
+            reg_no = _cell_text(cells[1])
+            if not _REG_NO_RE.match(reg_no):
+                unparsed.append(reg_no)
+                continue
+
+            raw_date = _cell_text(cells[3])  # "2026-08-06 06/08/2026"
+            status_raw = _cell_text(cells[4])
+            status, status_date = status_raw, ""
+            if m := _STATUS_DATE_RE.match(status_raw):
+                status, status_date = _clean(m.group(1)), m.group(2)
+
+            row_html = str(tr)
+            details = re.search(r"regdetails\.php\?[^'\"\s]+", row_html)
+            final = re.search(r"finalstatus_viewhistory\.php\?[^'\"\s]+", row_html)
+
+            out.append(
+                Application(
+                    registration_number=reg_no,
+                    applicant=_cell_text(cells[2]),
+                    date_of_receipt=raw_date.split(" ")[0] if raw_date else "",
+                    status=status,
+                    status_date=status_date,
+                    category=category,
+                    bucket=bucket,
+                    details_url=urljoin(page_url, details.group(0).replace("&amp;", "&"))
+                    if details
+                    else "",
+                    status_url=urljoin(page_url, final.group(0).replace("&amp;", "&"))
+                    if final
+                    else "",
+                )
+            )
+        self._unparsed[(category, bucket)] = unparsed
+        return out
+
+    def all_applications(self, refresh: bool = False) -> list[Application]:
+        """Every application, both categories.
+
+        The 'registered' bucket is the superset -- its count equals disposed +
+        pending -- so one fetch per category covers everything.
+        """
+        out: list[Application] = []
+        for category in CATEGORIES:
+            out.extend(self.applications(category, "registered", refresh=refresh))
+        return out
+
+    def find(self, registration_number: str, refresh: bool = False) -> Application:
+        wanted = registration_number.strip().upper()
+        for row in self.all_applications(refresh=refresh):
+            if row.registration_number.upper() == wanted:
+                return row
+        if not refresh:
+            return self.find(registration_number, refresh=True)
+        raise RTIError(
+            f"No application found with registration number {registration_number!r}. "
+            "Use `rti_search` to look it up by public authority or partial number."
+        )
+
+    # --------------------------------------------------------------- detail
+
+    def _live_lookup(self, registration_number: str) -> Application | None:
+        """The row from the currently-live list, if it holds this application."""
+        if not self._live_rows:
+            return None
+        wanted = registration_number.upper()
+        return next(
+            (r for r in self._live_rows if r.registration_number.upper() == wanted),
+            None,
+        )
+
+    def _walk_to(self, category: str, bucket: str) -> list[Application]:
+        """Make `bucket`'s row links live, re-walking from the seed if needed."""
+        if self._live_key == (category, bucket) and self._live_rows:
+            return self._live_rows
+        try:
+            return self._fetch_list(category, bucket)
+        except Blocked:
+            self.dashboard(refresh=True)  # clears a poisoned session
+            return self._fetch_list(category, bucket)
+
+    def _fetch_detail(self, app: Application, attr: str) -> dict[str, Any]:
+        """Read one detail page for `app`, honouring the navigation rules."""
+        category, bucket = app.category, app.bucket
+
+        def attempt() -> dict[str, Any]:
+            # Any list already live in this session works if it holds the row,
+            # so a run of status checks costs one list fetch, not one each.
+            live = self._live_lookup(app.registration_number)
+            if live is None or not getattr(live, attr):
+                rows = self._walk_to(category, bucket)
+                live = next(
+                    (
+                        r
+                        for r in rows
+                        if r.registration_number.upper()
+                        == app.registration_number.upper()
+                    ),
+                    None,
+                )
+            if live is None:
+                raise RTIError(
+                    f"{app.registration_number} is no longer in the {bucket} "
+                    f"{category} list -- its status may have changed. "
+                    "Re-run with refresh=true."
+                )
+            url = getattr(live, attr)
+            if not url:
+                raise RTIError(
+                    f"The portal offers no detail link for {app.registration_number}."
+                )
+            html, page_url = self._get_html(url)
+            return self._parse_fields(html, page_url)
+
+        try:
+            parsed = attempt()
+        except Blocked:
+            # Poisoned session: reset through the seed and walk again.
+            self.dashboard(refresh=True)
+            parsed = attempt()
+
+        if not parsed.get("fields"):
+            raise RTIError(
+                f"The portal returned no details for {app.registration_number}."
+            )
+        return parsed
+
+    @staticmethod
+    def _parse_fields(html: str, page_url: str) -> dict[str, Any]:
+        """Pull label/value pairs and document links off a detail page.
+
+        Detail pages are nested label/value tables, so two-cell rows carry the
+        data. The application text is the exception: it sits alone in a
+        single-cell row just after a 'Description of Information Sought'
+        heading.
+        """
+        soup = BeautifulSoup(html, "html.parser")
+        fields: dict[str, str] = {}
+        documents: dict[str, str] = {}
+        expecting_text = False
+        app_text = ""
+
+        for tr in soup.find_all("tr"):
+            cells = tr.find_all(["td", "th"])
+            texts = [_cell_text(c) for c in cells]
+
+            if len(cells) == 1:
+                label = texts[0]
+                if not label:
+                    continue
+                # The portal prints the heading twice ("(Description of
+                # Information sought (upto 500 characters)" then "Description
+                # of Information Sought") before the body, so keep waiting
+                # until a row that is not itself a heading turns up.
+                if re.search(r"Description of Information sought", label, re.I):
+                    expecting_text = True
+                elif expecting_text and not app_text:
+                    app_text = label
+                    expecting_text = False
+                continue
+
+            if len(cells) != 2:
+                continue
+            label, value = texts[0].rstrip(":").strip(), texts[1]
+            if not label or label.lower() == "s.no.":
+                continue
+            fields.setdefault(label, value)
+
+        for a in soup.find_all("a", href=True):
+            href = a["href"]
+            if "regId=" not in href:
+                continue
+            if "viewPDF.php" in href:
+                documents.setdefault("reply", urljoin(page_url, href.replace("&amp;", "&")))
+            elif "pdfDocument.php" in href:
+                documents.setdefault(
+                    "request", urljoin(page_url, href.replace("&amp;", "&"))
+                )
+
+        if app_text:
+            fields.setdefault("Description of Information Sought", app_text)
+        return {"fields": fields, "documents": documents}
+
+    def status_detail(self, app: Application) -> dict[str, Any]:
+        return self._fetch_detail(app, "status_url")
+
+    def request_detail(self, app: Application) -> dict[str, Any]:
+        return self._fetch_detail(app, "details_url")
+
+    # ------------------------------------------------------------ documents
+
+    def download(self, app: Application, kind: str, dest) -> dict[str, Any]:
+        """Save a request/reply PDF. The link has to be taken live, like details."""
+        kind = kind.lower()
+        if kind not in ("reply", "request"):
+            raise RTIError("kind must be 'reply' or 'request'")
+
+        attr = "status_url" if kind == "reply" else "details_url"
+        parsed = self._fetch_detail(app, attr)
+        url = parsed["documents"].get(kind)
+        if not url:
+            # The reply PDF is sometimes linked from the details page instead.
+            other = self._fetch_detail(
+                app, "details_url" if attr == "status_url" else "status_url"
+            )
+            url = other["documents"].get(kind)
+        if not url:
+            raise RTIError(
+                f"No {kind} document is attached to {app.registration_number}."
+            )
+
+        resp = self._get(url, timeout=240)
+        body = resp.content
+        content_type = resp.headers.get("Content-Type", "")
+        if b"%PDF" not in body[:1024] and "pdf" not in content_type.lower():
+            snippet = _clean(BeautifulSoup(resp.text[:4000], "html.parser").get_text(" "))
+            raise RTIError(
+                f"The portal did not return a PDF for the {kind} document "
+                f"(Content-Type: {content_type or 'unknown'}). It said: {snippet[:200]}"
+            )
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_bytes(body)
+        return {"path": str(dest), "bytes": len(body), "content_type": content_type}