recent-state-summarizer 0.0.8__tar.gz

recent_state_summarizer-0.0.8/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 nikkie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

recent_state_summarizer-0.0.8/PKG-INFO

@@ -0,0 +1,112 @@
+Metadata-Version: 2.4
+Name: recent-state-summarizer
+Version: 0.0.8
+Summary: Summarize a list of entry titles using LLM
+Author-email: nikkie <takuyafjp+develop@gmail.com>
+License: MIT
+Classifier: Development Status :: 1 - Planning
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: beautifulsoup4
+Requires-Dist: feedparser
+Requires-Dist: httpx
+Requires-Dist: openai<1
+Provides-Extra: testing
+Requires-Dist: pytest; extra == "testing"
+Requires-Dist: responses; extra == "testing"
+Requires-Dist: respx; extra == "testing"
+Provides-Extra: lint
+Requires-Dist: flake8; extra == "lint"
+Requires-Dist: black; extra == "lint"
+Requires-Dist: isort; extra == "lint"
+Provides-Extra: dev
+Requires-Dist: wheel; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: license-file
+
+# recent-state-summarizer
+
+Summarize blog article titles with the OpenAI API
+
+a.k.a. _RSS_ 😃
+
+## Setup
+
+```
+$ pip install recent-state-summarizer
+```
+
+⚠️ Set `OPENAI_API_KEY` environment variable.  
+ref: https://platform.openai.com/account/api-keys
+
+## Usage
+
+```
+$ omae-douyo https://nikkie-ftnext.hatenablog.com/archive/2023/4
+
+この人物は最近、プログラミングに関することを中心にして活動しています。
+
+（略）
+
+最近は、株式会社はてなに入社したようです。
+```
+
+Currently support:
+
+- はてなブログ（Hatena blog）
+- はてなブックマークRSS
+- Adventar
+- Qiita Advent Calendar
+
+To see help, type `omae-douyo -h`.
+
+### Fetch only (save to file)
+
+Fetch titles and URLs of articles, and save them to a file without summarization:
+
+```
+# Save as JSON format (default)
+$ omae-douyo fetch https://nikkie-ftnext.hatenablog.com/archive/2023/4 articles.jsonl
+
+# Save as bullet list
+$ omae-douyo fetch https://nikkie-ftnext.hatenablog.com/archive/2023/4 titles.txt --as-title-list
+```
+
+## Development
+
+### Sub commands
+
+Fetch only (same as `omae-douyo fetch`):
+
+```
+python -m recent_state_summarizer.fetch -h
+```
+
+Summarize only:  
+It's convenient to omit fetching in tuning the prompt.
+
+```
+python -m recent_state_summarizer.summarize -h
+```
+
+### Environment
+
+```
+$ git clone https://github.com/ftnext/recent-state-summarizer.git
+$ cd recent-state-summarizer
+
+$ python -m venv venv
+$ source venv/bin/activate
+(venv) $ pip install -r requirements.lock
+(venv) $ pip install -e '.'
+```

recent_state_summarizer-0.0.8/README.md

@@ -0,0 +1,76 @@
+# recent-state-summarizer
+
+Summarize blog article titles with the OpenAI API
+
+a.k.a. _RSS_ 😃
+
+## Setup
+
+```
+$ pip install recent-state-summarizer
+```
+
+⚠️ Set `OPENAI_API_KEY` environment variable.  
+ref: https://platform.openai.com/account/api-keys
+
+## Usage
+
+```
+$ omae-douyo https://nikkie-ftnext.hatenablog.com/archive/2023/4
+
+この人物は最近、プログラミングに関することを中心にして活動しています。
+
+（略）
+
+最近は、株式会社はてなに入社したようです。
+```
+
+Currently support:
+
+- はてなブログ（Hatena blog）
+- はてなブックマークRSS
+- Adventar
+- Qiita Advent Calendar
+
+To see help, type `omae-douyo -h`.
+
+### Fetch only (save to file)
+
+Fetch titles and URLs of articles, and save them to a file without summarization:
+
+```
+# Save as JSON format (default)
+$ omae-douyo fetch https://nikkie-ftnext.hatenablog.com/archive/2023/4 articles.jsonl
+
+# Save as bullet list
+$ omae-douyo fetch https://nikkie-ftnext.hatenablog.com/archive/2023/4 titles.txt --as-title-list
+```
+
+## Development
+
+### Sub commands
+
+Fetch only (same as `omae-douyo fetch`):
+
+```
+python -m recent_state_summarizer.fetch -h
+```
+
+Summarize only:  
+It's convenient to omit fetching in tuning the prompt.
+
+```
+python -m recent_state_summarizer.summarize -h
+```
+
+### Environment
+
+```
+$ git clone https://github.com/ftnext/recent-state-summarizer.git
+$ cd recent-state-summarizer
+
+$ python -m venv venv
+$ source venv/bin/activate
+(venv) $ pip install -r requirements.lock
+(venv) $ pip install -e '.'
+```

recent_state_summarizer-0.0.8/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "recent-state-summarizer"
+authors = [
+    {name = "nikkie", email = "takuyafjp+develop@gmail.com"},
+]
+description = "Summarize a list of entry titles using LLM"
+readme = {file = "README.md", content-type = "text/markdown"}
+requires-python = ">=3.10"
+license = {text = "MIT"}
+classifiers = [
+    "Development Status :: 1 - Planning",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+]
+dependencies = [
+    "beautifulsoup4",
+    "feedparser",
+    "httpx",
+    "openai<1",
+]
+dynamic = ["version"]
+
+[project.optional-dependencies]
+testing = ["pytest", "responses", "respx"]
+lint = ["flake8", "black", "isort"]
+dev = ["wheel", "build", "twine"]
+
+[project.scripts]
+omae-douyo = "recent_state_summarizer.__main__:main"
+
+[tool.setuptools.packages.find]
+exclude = ["tests"]
+
+[tool.setuptools.dynamic]
+version = {attr = "recent_state_summarizer.__version__"}
+
+[tool.pytest]
+testpaths = ["tests"]

recent_state_summarizer-0.0.8/recent_state_summarizer/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.8"

recent_state_summarizer-0.0.8/recent_state_summarizer/__main__.py

@@ -0,0 +1,93 @@
+import argparse
+import sys
+import tempfile
+from textwrap import dedent
+
+from recent_state_summarizer.fetch import _main as fetch_main
+from recent_state_summarizer.fetch import build_parser as build_fetch_parser
+from recent_state_summarizer.summarize import summarize_titles
+
+
+def build_parser():
+    help_message = """
+    Summarize blog article titles with the OpenAI API.
+
+    ⚠️ Set `OPENAI_API_KEY` environment variable.
+
+    Example:
+        omae-douyo https://awesome.hatenablog.com/archive/2023
+
+    Retrieve the titles of articles from a specified URL.
+    After summarization, prints the summary.
+    """
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=dedent(help_message),
+    )
+    subparsers = parser.add_subparsers(dest="subcommand")
+
+    run_parser = subparsers.add_parser(
+        "run", help="Fetch article titles and generate summary (default)"
+    )
+    run_parser.add_argument("url", help="URL of archive page")
+    run_parser.set_defaults(func=run_cli)
+
+    fetch_help_message = """
+    Retrieve the titles and URLs of articles from a web page specified by URL
+    and save them as JSON Lines format.
+
+    Support:
+        - はてなブログ（Hatena blog）
+        - はてなブックマークRSS
+        - Adventar
+        - Qiita Advent Calendar
+
+    Example:
+        omae-douyo fetch https://awesome.hatenablog.com/archive/2023 articles.jsonl
+    """
+    fetch_parser = subparsers.add_parser(
+        "fetch",
+        parents=[build_fetch_parser(add_help=False)],
+        help="Fetch article titles only and save to file",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=dedent(fetch_help_message),
+    )
+    fetch_parser.set_defaults(func=fetch_cli)
+
+    return parser
+
+
+def run_cli(args):
+    with tempfile.NamedTemporaryFile(mode="w+") as tempf:
+        fetch_main(args.url, tempf.name, save_as_title_list=True)
+        tempf.seek(0)
+        titles = tempf.read()
+    summary = summarize_titles(titles)
+    print(summary)
+
+
+def fetch_cli(args):
+    fetch_main(args.url, args.save_path, save_as_title_list=args.as_title_list)
+
+
+def normalize_argv() -> list[str]:
+    argv = sys.argv[1:]
+    if len(argv) == 0:
+        return ["--help"]
+
+    help_flags = {"-h", "--help"}
+    if argv[0] in help_flags:
+        return argv
+
+    known_subcommands = {"run", "fetch"}
+    if argv[0] not in known_subcommands:
+        return ["run"] + argv
+
+    return argv
+
+
+def main():
+    parser = build_parser()
+    argv = normalize_argv()
+    args = parser.parse_args(argv)
+    args.func(args)

recent_state_summarizer-0.0.8/recent_state_summarizer/fetch/__init__.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import textwrap
+from collections.abc import Iterable
+from enum import Enum
+from pathlib import Path
+from urllib.parse import urlparse
+
+from recent_state_summarizer.fetch.adventar import (
+    TitleTag,
+    fetch_adventar_calendar,
+)
+from recent_state_summarizer.fetch.hatena_blog import _fetch_titles
+from recent_state_summarizer.fetch.hatena_bookmark import (
+    fetch_hatena_bookmark_rss,
+)
+from recent_state_summarizer.fetch.qiita_advent_calendar import (
+    fetch_qiita_advent_calendar,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class URLType(Enum):
+    """Type of URL for fetching."""
+
+    HATENA_BLOG = "hatena_blog"
+    HATENA_BOOKMARK_RSS = "hatena_bookmark_rss"
+    ADVENTAR = "adventar"
+    QIITA_ADVENT_CALENDAR = "qiita_advent_calendar"
+    UNKNOWN = "unknown"
+
+
+def _detect_url_type(url: str) -> URLType:
+    """Detect the type of URL to determine fetch strategy.
+
+    Args:
+        url: URL to analyze
+
+    Returns:
+        URLType indicating the fetch strategy to use
+    """
+    parsed = urlparse(url)
+    if (
+        parsed.netloc == "b.hatena.ne.jp"
+        and parsed.path.startswith("/entrylist/")
+        and parsed.path.endswith(".rss")
+    ):
+        return URLType.HATENA_BOOKMARK_RSS
+
+    if parsed.netloc == "qiita.com" and "/advent-calendar/" in parsed.path:
+        return URLType.QIITA_ADVENT_CALENDAR
+
+    if "/calendars/" in parsed.path or "adventar.org" in parsed.netloc:
+        return URLType.ADVENTAR
+
+    if (
+        "hatenablog.com" in url
+        or "hateblo.jp" in url
+        or "/archive/" in parsed.path
+    ):
+        return URLType.HATENA_BLOG
+
+    return URLType.UNKNOWN
+
+
+def _select_fetcher(url_type):
+    match url_type:
+        case URLType.HATENA_BOOKMARK_RSS:
+            return fetch_hatena_bookmark_rss
+        case URLType.HATENA_BLOG:
+            return _fetch_titles
+        case URLType.ADVENTAR:
+            return fetch_adventar_calendar
+        case URLType.QIITA_ADVENT_CALENDAR:
+            return fetch_qiita_advent_calendar
+        case _:
+            raise ValueError(f"Unsupported URL type: {url_type}")
+
+
+def _main(
+    url: str, save_path: str | Path, *, save_as_title_list: bool
+) -> None:
+    url_type = _detect_url_type(url)
+    fetcher = _select_fetcher(url_type)
+    title_tags = fetcher(url)
+    if save_as_title_list:
+        contents = _as_bullet_list(
+            title_tag["title"] for title_tag in title_tags
+        )
+    else:
+        contents = _as_json(title_tags)
+    _save(save_path, contents)
+
+
+def _as_bullet_list(titles: Iterable[str]) -> str:
+    return "\n".join(f"- {title}" for title in titles)
+
+
+def _as_json(title_tags: Iterable[TitleTag]) -> str:
+    return "\n".join(
+        json.dumps(title_tag, ensure_ascii=False) for title_tag in title_tags
+    )
+
+
+def _save(path: str | Path, contents: str) -> None:
+    with open(path, "w", encoding="utf8", newline="") as f:
+        f.write(contents)
+
+
+def build_parser(add_help: bool = True) -> argparse.ArgumentParser:
+    help_message = """
+    Retrieve the titles and URLs of articles from a web page specified by URL
+    and save them as JSON Lines format.
+
+    Support:
+        - はてなブログ（Hatena blog）
+        - はてなブックマークRSS
+        - Adventar
+        - Qiita Advent Calendar
+
+    Example:
+        python -m recent_state_summarizer.fetch \\
+          https://awesome.hatenablog.com/archive/2023 articles.jsonl
+    """
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=textwrap.dedent(help_message),
+        add_help=add_help,
+    )
+    parser.add_argument("url", help="URL of archive page")
+    parser.add_argument("save_path", help="Local file path")
+    parser.add_argument(
+        "--as-title-list",
+        action="store_true",
+        default=False,
+        help="Save as title-only bullet list instead of JSON Lines",
+    )
+    return parser
+
+
+def cli():
+    parser = build_parser()
+    args = parser.parse_args()
+
+    _main(args.url, args.save_path, save_as_title_list=args.as_title_list)
+
+
+if __name__ == "__main__":
+    cli()

recent_state_summarizer-0.0.8/recent_state_summarizer/fetch/adventar.py

@@ -0,0 +1,59 @@
+from collections.abc import Generator
+from typing import TypedDict
+
+import httpx
+from bs4 import BeautifulSoup
+
+
+class TitleTag(TypedDict):
+    title: str
+    url: str
+
+
+def _fetch(url: str) -> str:
+    response = httpx.get(url)
+    response.raise_for_status()
+    return response.text
+
+
+def fetch_adventar_calendar(url: str) -> Generator[TitleTag, None, None]:
+    """Fetch article titles and URLs from Adventar calendar.
+
+    Args:
+        url: Adventar calendar URL (e.g., https://adventar.org/calendars/11474)
+
+    Yields:
+        TitleTag dictionaries containing title and url
+    """
+    raw_html = _fetch(url)
+    yield from _parse_titles(raw_html)
+
+
+def _parse_titles(raw_html: str) -> Generator[TitleTag, None, None]:
+    """Parse titles from Adventar calendar HTML."""
+    soup = BeautifulSoup(raw_html, "html.parser")
+    entry_list = soup.find("ul", class_="EntryList")
+    if not entry_list:
+        return
+
+    items = entry_list.find_all("li", class_="item")
+    for item in items:
+        article = item.find("div", class_="article")
+        if not article:
+            continue
+
+        link_div = article.find("div", class_="link")
+        if not link_div:
+            continue
+
+        link = link_div.find("a")
+        if not link or "href" not in link.attrs:
+            continue
+
+        title_div = link_div.find_next_sibling("div")
+        if title_div and title_div.text.strip():
+            title = title_div.text.strip()
+        else:
+            title = link.text.strip()
+
+        yield {"title": title, "url": link["href"]}

recent_state_summarizer-0.0.8/recent_state_summarizer/fetch/hatena_blog.py

@@ -0,0 +1,39 @@
+from collections.abc import Generator
+from typing import TypedDict
+
+import httpx
+from bs4 import BeautifulSoup
+
+PARSE_HATENABLOG_KWARGS = {"name": "a", "attrs": {"class": "entry-title-link"}}
+
+
+class TitleTag(TypedDict):
+    title: str
+    url: str
+
+
+def _fetch(url: str) -> str:
+    with httpx.Client() as client:
+        response = client.get(url)
+        response.raise_for_status()
+        return response.text
+
+
+def _fetch_titles(url: str) -> Generator[TitleTag, None, None]:
+    raw_html = _fetch(url)
+    yield from _parse_titles(raw_html)
+
+    soup = BeautifulSoup(raw_html, "html.parser")
+    next_link = soup.find("a", class_="test-pager-next")
+    if next_link and "href" in next_link.attrs:
+        next_url = next_link["href"]
+        print(f"Next page found, fetching... {next_url}")
+        yield from _fetch_titles(next_url)
+
+
+def _parse_titles(raw_html: str) -> Generator[TitleTag, None, None]:
+    soup = BeautifulSoup(raw_html, "html.parser")
+    body = soup.body
+    title_tags = body.find_all(**PARSE_HATENABLOG_KWARGS)
+    for title_tag in title_tags:
+        yield {"title": title_tag.text, "url": title_tag["href"]}

recent_state_summarizer-0.0.8/recent_state_summarizer/fetch/hatena_bookmark.py

@@ -0,0 +1,34 @@
+from typing import Generator, TypedDict
+
+import feedparser
+import httpx
+
+
+class BookmarkEntry(TypedDict):
+    title: str
+    url: str
+    description: str
+
+
+def fetch_hatena_bookmark_rss(
+    url: str,
+) -> Generator[BookmarkEntry, None, None]:
+    """Fetch entries from Hatena Bookmark RSS feed.
+
+    Args:
+        url: URL of the Hatena Bookmark RSS feed
+
+    Yields:
+        Bookmark entries with title, url, and description
+    """
+    response = httpx.get(url)
+    response.raise_for_status()
+
+    feed = feedparser.parse(response.content)
+
+    for entry in feed.entries:
+        yield {
+            "title": entry.title,
+            "url": entry.link,
+            "description": entry.description,
+        }

recent_state_summarizer-0.0.8/recent_state_summarizer/fetch/qiita_advent_calendar.py

@@ -0,0 +1,58 @@
+import json
+from collections.abc import Generator
+from typing import TypedDict
+
+import httpx
+from bs4 import BeautifulSoup
+
+
+class TitleTag(TypedDict):
+    title: str
+    url: str
+
+
+def _fetch(url: str) -> str:
+    response = httpx.get(url)
+    response.raise_for_status()
+    return response.text
+
+
+def fetch_qiita_advent_calendar(url: str) -> Generator[TitleTag, None, None]:
+    """Fetch article titles and URLs from Qiita Advent Calendar.
+
+    Args:
+        url: Qiita Advent Calendar URL (e.g., https://qiita.com/advent-calendar/2025/python-type-hints)
+
+    Yields:
+        TitleTag dictionaries containing title and url
+    """
+    raw_html = _fetch(url)
+    yield from _parse_titles(raw_html)
+
+
+def _parse_titles(raw_html: str) -> Generator[TitleTag, None, None]:
+    """Parse titles from Qiita Advent Calendar HTML by extracting JSON data."""
+    soup = BeautifulSoup(raw_html, "html.parser")
+    script_tag = soup.find(
+        "script",
+        attrs={"data-js-react-on-rails-store": "AppStoreWithReactOnRails"},
+    )
+    if not script_tag or not script_tag.string:
+        return
+
+    data = json.loads(script_tag.string)
+    advent_calendars = data.get("adventCalendars", {})
+    table_calendars = advent_calendars.get("tableAdventCalendars", [])
+    if not table_calendars:
+        return
+    items = table_calendars[0].get("items", [])
+
+    for item in items:
+        if not item.get("isRevealed", False):
+            continue
+
+        title = item.get("comment")
+        article_url = item.get("url")
+
+        if title and article_url:
+            yield {"title": title, "url": article_url}