law-cn-cli 0.2.0__py3-none-any.whl

cnlaw/__init__.py

@@ -0,0 +1,6 @@
+"""Unified CLI and adapter SDK for official Chinese legal sources."""
+
+from .models import LawRecord, SearchQuery
+
+__all__ = ["LawRecord", "SearchQuery"]
+

cnlaw/__main__.py

@@ -0,0 +1,5 @@
+from .cli import entrypoint
+
+
+entrypoint()
+

cnlaw/adapters/__init__.py

@@ -0,0 +1,6 @@
+"""Official source adapters."""
+
+from .base import SourceAdapter
+
+__all__ = ["SourceAdapter"]
+

cnlaw/adapters/base.py

@@ -0,0 +1,212 @@
+"""Common adapter contract and full-pagination collector."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+import httpx
+
+from cnlaw.errors import UnsupportedCapabilityError
+from cnlaw.transport import DocumentFileCache, RateLimiter, ResponseCache
+from cnlaw.models import (
+    ArticleLookup,
+    ArticleSearchResult,
+    DocumentInfo,
+    DocumentPreview,
+    LawRecord,
+    SearchManifest,
+    SearchPage,
+    SearchQuery,
+    SourceCapabilities,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class DownloadResult:
+    """Result of an explicit file download from an official source."""
+
+    source: str
+    document_id: str
+    file_path: str
+    file_size: int
+    content_hash: str
+    format: str
+    official_url: str
+    downloaded_at: str = field(
+        default_factory=lambda: datetime.now(UTC).isoformat()
+    )
+    cache_hit: bool = False
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass(frozen=True, slots=True)
+class BatchDownloadFailure:
+    document_id: str
+    error: str
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass(frozen=True, slots=True)
+class BatchDownloadResult:
+    source: str
+    requested: int
+    downloaded: tuple[DownloadResult, ...]
+    failures: tuple[BatchDownloadFailure, ...]
+
+    def to_dict(self) -> dict:
+        return {
+            "source": self.source,
+            "requested": self.requested,
+            "downloaded": [result.to_dict() for result in self.downloaded],
+            "failures": [failure.to_dict() for failure in self.failures],
+        }
+
+
+class SourceAdapter(ABC):
+    code: str
+    name: str
+
+    def __init__(
+        self,
+        client: httpx.Client | None = None,
+        *,
+        cache: ResponseCache | None = None,
+        file_cache: DocumentFileCache | None = None,
+        rate_limiter: RateLimiter | None = None,
+    ) -> None:
+        self.client = client or httpx.Client(
+            follow_redirects=True,
+            timeout=30,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                    "AppleWebKit/537.36 Chrome/125 Safari/537.36"
+                )
+            },
+        )
+        self._cache = cache
+        self._file_cache = file_cache
+        self._rate_limiter = rate_limiter
+
+    @abstractmethod
+    def capabilities(self) -> SourceCapabilities:
+        """Describe which unified filters this source can honor."""
+
+    def _cached_get(self, url: str, *, headers: dict | None = None, params: dict | None = None, ttl: float = 3600.0):
+        """GET with optional caching and rate limiting."""
+        cache_key_body = str(sorted((params or {}).items())).encode() if params else None
+        if self._cache is not None:
+            cached = self._cache.get("GET", url, cache_key_body)
+            if cached is not None:
+                return cached
+        if self._rate_limiter is not None:
+            self._rate_limiter.acquire()
+        response = self.client.get(url, headers=headers or {}, params=params or {})
+        if self._rate_limiter is not None and response.status_code == 429:
+            self._rate_limiter.record_429()
+        elif self._rate_limiter is not None:
+            self._rate_limiter.record_success()
+        if self._cache is not None and response.is_success:
+            self._cache.set("GET", url, response, ttl=ttl, body=cache_key_body)
+        return response
+
+    @abstractmethod
+    def search_pages(self, query: SearchQuery) -> Iterator[SearchPage]:
+        """Yield every result page reported by the official source."""
+
+    def info(self, document_id: str) -> DocumentInfo:
+        """Fetch one official document detail page by source-native ID."""
+        raise UnsupportedCapabilityError(
+            f"{self.code} does not support document info"
+        )
+
+    def article(
+        self,
+        document_id: str,
+        *,
+        article_number: str | None = None,
+        grep: str | None = None,
+        refresh: bool = False,
+    ) -> ArticleLookup:
+        """Extract article text from one official source document."""
+        raise UnsupportedCapabilityError(
+            f"{self.code} does not support article extraction"
+        )
+
+    def preview(
+        self, document_id: str, *, refresh: bool = False
+    ) -> DocumentPreview:
+        """Show regulation structure: TOC, article count, sample article numbers."""
+        raise UnsupportedCapabilityError(
+            f"{self.code} does not support document preview"
+        )
+
+    def article_search(
+        self,
+        keyword: str,
+        *,
+        max_laws: int | None = None,
+        statuses: tuple[str, ...] = (),
+        offset: int = 0,
+        context: int = 0,
+        refresh: bool = False,
+    ) -> ArticleSearchResult:
+        """Search across multiple laws for articles containing a keyword."""
+        raise UnsupportedCapabilityError(
+            f"{self.code} does not support cross-law article search"
+        )
+
+    def download(
+        self,
+        document_id: str,
+        *,
+        format: str = "docx",
+        output: Path | None = None,
+        refresh: bool = False,
+    ) -> "DownloadResult":
+        """Download an official document file to disk."""
+        raise UnsupportedCapabilityError(
+            f"{self.code} does not support file download"
+        )
+
+    def search(self, query: SearchQuery) -> tuple[list[LawRecord], SearchManifest]:
+        records: list[LawRecord] = []
+        pages_fetched = 0
+        total_reported: int | None = None
+        truncated = False
+
+        for page in self.search_pages(query):
+            pages_fetched += 1
+            if page.total is not None:
+                total_reported = page.total
+            for record in page.records:
+                if query.limit is not None and len(records) >= query.limit:
+                    truncated = True
+                    break
+                records.append(record)
+            if truncated:
+                break
+
+        if (
+            query.limit is not None
+            and total_reported is not None
+            and total_reported > len(records)
+        ):
+            truncated = True
+
+        return records, SearchManifest(
+            source=self.code,
+            keyword=query.keyword,
+            pages_fetched=pages_fetched,
+            records_written=len(records),
+            total_reported=total_reported,
+            explicit_limit=query.limit,
+            truncated=truncated,
+        )

cnlaw/adapters/bse.py

@@ -0,0 +1,232 @@
+"""北京证券交易所业务规则 JSONP search adapter."""
+
+from __future__ import annotations
+
+from bs4 import BeautifulSoup
+
+import json
+import math
+import re
+from collections.abc import Iterator
+from urllib.parse import urlparse, urlencode
+
+import httpx
+
+from cnlaw.errors import SourceParseError
+from cnlaw.models import (
+    CapabilityLevel,
+    DocumentInfo,
+    LawRecord,
+    SearchParameter,
+    SearchPage,
+    SearchQuery,
+    SourceCapabilities,
+)
+
+from .base import SourceAdapter
+from .utils import canonical_https, clean_html
+
+
+class BSEAdapter(SourceAdapter):
+    code = "bse"
+    name = "北京证券交易所规则"
+    search_url = "https://www.bse.cn/infoEsController/searchDetails.do"
+    base_url = "https://www.bse.cn"
+    search_referer = "https://www.bse.cn/node/latestRule.html"
+    site_id = "6"
+    issuing_authority = "北京证券交易所"
+    page_size = 20
+    business_rule_node_ids = (
+        "1302",
+        "1303",
+        "1304",
+        "3130",
+        "3131",
+        "3132",
+        "1306",
+    )
+
+    def __init__(self, client: httpx.Client | None = None) -> None:
+        # The CDN currently sends one same-URL challenge redirect and a short-
+        # lived C3VK cookie. It remains only in this in-memory client.
+        self.client = client or httpx.Client(
+            follow_redirects=False,
+            timeout=30,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                    "AppleWebKit/537.36 Chrome/125 Safari/537.36"
+                )
+            },
+        )
+
+    def capabilities(self) -> SourceCapabilities:
+        return SourceCapabilities(
+            source=self.code,
+            transport="frontend_jsonp_api",
+            filters={
+                "scope:all": CapabilityLevel.NATIVE,
+                "publish_date": CapabilityLevel.NATIVE,
+                "sort:newest": CapabilityLevel.NATIVE,
+            },
+            source_parameters=(
+                SearchParameter(
+                    "rule_channel", ("nodeIds[]",), "规则栏目叶节点 ID；默认全部业务规则栏目",
+                    repeatable=True,
+                ),
+            ),
+        )
+
+    def _body(self, query: SearchQuery, page: int) -> str:
+        pairs = [
+            ("page", str(page)),
+            ("pageSize", str(self.page_size)),
+            ("keywords", query.keyword),
+            ("startTime", query.publish_from or ""),
+            ("endTime", query.publish_to or ""),
+        ]
+        node_ids = query.source_param_values("rule_channel") or self.business_rule_node_ids
+        pairs.extend(("nodeIds[]", node_id) for node_id in node_ids)
+        pairs.extend(
+            [
+                ("siteId", self.site_id),
+                ("relevancy", "1" if query.sort == "newest" else "0"),
+            ]
+        )
+        return urlencode(pairs)
+
+    def _post(self, query: SearchQuery, page: int) -> httpx.Response:
+        response = self.client.post(
+            self.search_url,
+            params={"callback": "cnlaw"},
+            content=self._body(query, page),
+            headers={
+                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+                "Referer": self.search_referer,
+            },
+        )
+        if response.status_code in {301, 302, 307, 308} and response.headers.get(
+            "set-cookie"
+        ):
+            response = self.client.post(
+                self.search_url,
+                params={"callback": "cnlaw"},
+                content=self._body(query, page),
+                headers={
+                    "Content-Type": (
+                        "application/x-www-form-urlencoded; charset=UTF-8"
+                    ),
+                    "Referer": self.search_referer,
+                },
+            )
+        return response
+
+    @staticmethod
+    def _parse_jsonp(text: str) -> dict:
+        match = re.fullmatch(r"\s*[^(]+\((.*)\)\s*;?\s*", text, re.DOTALL)
+        if not match:
+            raise SourceParseError("bse search response is not JSONP")
+        try:
+            outer = json.loads(match.group(1))
+        except json.JSONDecodeError as exc:
+            raise SourceParseError("bse search JSONP payload is invalid") from exc
+        if (
+            not isinstance(outer, list)
+            or not outer
+            or outer[0].get("result") is not True
+            or not isinstance(outer[0].get("data"), dict)
+        ):
+            raise SourceParseError("bse search response contract changed")
+        return outer[0]["data"]
+
+    @staticmethod
+    def _document_number(text: str) -> str | None:
+        match = re.search(r"北证[^\s〔]{0,8}〔\d{4}〕\d+\s*号", text)
+        return match.group(0).replace(" ", "") if match else None
+
+    def _official_url(self, value: str) -> str:
+        if value.startswith("//"):
+            return f"https:{value}"
+        if value.startswith("/"):
+            return f"{self.base_url}{value}"
+        return value
+
+    def search_pages(self, query: SearchQuery) -> Iterator[SearchPage]:
+        page = 0
+        page_count = 1
+        rank = 0
+        while page < page_count:
+            response = self._post(query, page)
+            response.raise_for_status()
+            data = self._parse_jsonp(response.text)
+            items = data.get("content")
+            if not isinstance(items, list):
+                raise SourceParseError("bse search content is not a list")
+            total = int(data.get("totalElements") or 0)
+            size = int(data.get("size") or self.page_size)
+            page_count = max(1, math.ceil(total / size))
+            records = []
+            for item in items:
+                rank += 1
+                summary = clean_html(item.get("description"))
+                relative_url = item.get("linkUrl") or item.get("url") or ""
+                records.append(
+                    LawRecord(
+                        source=self.code,
+                        source_name=self.name,
+                        source_document_id=str(item.get("id") or "") or None,
+                        title=clean_html(item.get("title")),
+                        official_url=self._official_url(relative_url),
+                        document_type="交易所规则",
+                        issuing_authority=self.issuing_authority,
+                        document_number=self._document_number(summary),
+                        publish_date=item.get("publishDate") or None,
+                        summary=summary or None,
+                        source_rank=rank,
+                        raw_metadata=item,
+                    )
+                )
+            yield SearchPage(
+                source=self.code,
+                page=page,
+                total=total,
+                records=tuple(records),
+            )
+            page += 1
+
+
+    _INFO_ALLOWED_DOMAINS = frozenset({"www.bse.cn", "bse.cn"})
+
+    def info(self, document_id: str) -> DocumentInfo:
+        """Fetch one document detail page."""
+        document_id = document_id.strip()
+        if not document_id:
+            raise ValueError("document id must not be empty")
+        url = document_id if document_id.startswith("http") else canonical_https(document_id)
+        parsed = urlparse(url)
+        if parsed.hostname not in self._INFO_ALLOWED_DOMAINS:
+            raise ValueError(f"bse info URL must be on an official domain, got: {parsed.hostname}")
+        response = self.client.get(url)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.text, "html.parser")
+        title_node = soup.find("h1") or soup.find("title")
+        title = clean_html(str(title_node)) if title_node else url
+        body_container = (
+            soup.select_one(".in_main")
+            or soup.select_one(".article-content")
+            or soup.select_one(".detail-content")
+            or soup.select_one(".news-content")
+            or soup.select_one(".content")
+        )
+        body = clean_html(str(body_container)) if body_container else None
+        return DocumentInfo(
+            source=self.code,
+            source_name=self.name,
+            source_document_id=document_id,
+            title=title,
+            official_url=url,
+            body=body,
+            body_availability="full" if body else "unavailable",
+            body_note=None if body else "详情页未找到正文容器。",
+            raw_metadata={"fetched_url": url},
+        )