recent-state-summarizer 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

recent_state_summarizer/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.1"
+__version__ = "0.0.3"

recent_state_summarizer/fetch.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import json
 from collections.abc import Generator, Iterable
 from pathlib import Path
+from typing import TypedDict
 from urllib.request import urlopen
 
 from bs4 import BeautifulSoup
@@ -9,37 +11,57 @@ from bs4 import BeautifulSoup
 PARSE_HATENABLOG_KWARGS = {"name": "a", "attrs": {"class": "entry-title-link"}}
 
 
-def _main(url: str, save_path: str | Path) -> None:
-    contents = fetch_titles_as_bullet_list(url)
-    _save(save_path, contents)
+class TitleTag(TypedDict):
+    title: str
+    url: str
 
 
-def fetch_titles_as_bullet_list(url: str) -> str:
-    return _as_bullet_list(_fetch_titles(url))
+def _main(url: str, save_path: str | Path, save_as_json: bool) -> None:
+    title_tags = _fetch_titles(url)
+    if not save_as_json:
+        contents = _as_bullet_list(
+            title_tag["title"] for title_tag in title_tags
+        )
+    else:
+        contents = _as_json(title_tags)
+    _save(save_path, contents)
 
 
-def _fetch_titles(url: str) -> Generator[str, None, None]:
+def _fetch_titles(url: str) -> Generator[TitleTag, None, None]:
     raw_html = _fetch(url)
     yield from _parse_titles(raw_html)
 
+    soup = BeautifulSoup(raw_html, "html.parser")
+    next_link = soup.find("a", class_="test-pager-next")
+    if next_link and "href" in next_link.attrs:
+        next_url = next_link["href"]
+        print(f"Next page found, fetching... {next_url}")
+        yield from _fetch_titles(next_url)
+
 
 def _fetch(url: str) -> str:
     with urlopen(url) as res:
         return res.read()
 
 
-def _parse_titles(raw_html: str) -> Generator[str, None, None]:
+def _parse_titles(raw_html: str) -> Generator[TitleTag, None, None]:
     soup = BeautifulSoup(raw_html, "html.parser")
     body = soup.body
     title_tags = body.find_all(**PARSE_HATENABLOG_KWARGS)
     for title_tag in title_tags:
-        yield title_tag.text
+        yield {"title": title_tag.text, "url": title_tag["href"]}
 
 
 def _as_bullet_list(titles: Iterable[str]) -> str:
     return "\n".join(f"- {title}" for title in titles)
 
 
+def _as_json(title_tags: Iterable[TitleTag]) -> str:
+    return "\n".join(
+        json.dumps(title_tag, ensure_ascii=False) for title_tag in title_tags
+    )
+
+
 def _save(path: str | Path, contents: str) -> None:
     with open(path, "w", encoding="utf8", newline="") as f:
         f.write(contents)
@@ -66,6 +88,12 @@ if __name__ == "__main__":
     )
     parser.add_argument("url", help="URL of archive page")
     parser.add_argument("save_path", help="Local file path")
+    parser.add_argument(
+        "--as-json",
+        action="store_true",
+        default=False,
+        help="Save as JSON format instead of bullet list",
+    )
     args = parser.parse_args()
 
-    _main(args.url, args.save_path)
+    _main(args.url, args.save_path, args.as_json)

recent_state_summarizer/summarize.py

@@ -25,17 +25,19 @@ def _build_prompts(titles: str):
 
 def _build_summarize_prompt_text(titles_as_list: str) -> str:
     return f"""\
-以下は同一人物が最近書いたブログ記事のタイトルの一覧です。
+3つのバッククォートで囲まれた以下は、同一人物が最近書いたブログ記事のタイトルの一覧です。
 それを読み、この人物が最近何をやっているかを詳しく教えてください。
 応答は文ごとに改行して区切ってください。
 
+```
 {titles_as_list}
+```
 """
 
 
-def _complete_chat(prompts):
+def _complete_chat(prompts, temperature=0.0):
     return openai.ChatCompletion.create(
-        model=MODEL, messages=prompts, temperature=0.8
+        model=MODEL, messages=prompts, temperature=temperature
     )
 
 

{recent_state_summarizer-0.0.1.dist-info → recent_state_summarizer-0.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: recent-state-summarizer
-Version: 0.0.1
+Version: 0.0.3
 Summary: Summarize a list of entry titles using LLM
 Author-email: nikkie <takuyafjp+develop@gmail.com>
 License: MIT
@@ -8,25 +8,28 @@ Classifier: Development Status :: 1 - Planning
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.8
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: beautifulsoup4
-Requires-Dist: openai
-Provides-Extra: dev
-Requires-Dist: wheel ; extra == 'dev'
-Requires-Dist: build ; extra == 'dev'
-Requires-Dist: twine ; extra == 'dev'
-Provides-Extra: lint
-Requires-Dist: flake8 ; extra == 'lint'
-Requires-Dist: black ; extra == 'lint'
-Requires-Dist: isort ; extra == 'lint'
+Requires-Dist: openai<1
 Provides-Extra: testing
-Requires-Dist: pytest ; extra == 'testing'
+Requires-Dist: pytest; extra == "testing"
+Requires-Dist: pytest_httpserver; extra == "testing"
+Provides-Extra: lint
+Requires-Dist: flake8; extra == "lint"
+Requires-Dist: black; extra == "lint"
+Requires-Dist: isort; extra == "lint"
+Provides-Extra: dev
+Requires-Dist: wheel; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: license-file
 
 # recent-state-summarizer
 

recent_state_summarizer-0.0.3.dist-info/RECORD

@@ -0,0 +1,10 @@
+recent_state_summarizer/__init__.py,sha256=4GZKi13lDTD25YBkGakhZyEQZWTER_OWQMNPoH_UM2c,22
+recent_state_summarizer/__main__.py,sha256=mpPX-XZ6Ggrs41Lilc657by7ori5JwjRU4ek_eKK49k,966
+recent_state_summarizer/fetch.py,sha256=GtlmJvHsxgqtgqi258nOxdzrc7zl1xIYYBzY8CEVcUE,2808
+recent_state_summarizer/summarize.py,sha256=0CgVNkY2m8qN0n1G9V2zvqghO3abUYlLIYEpJErdUTw,1989
+recent_state_summarizer-0.0.3.dist-info/licenses/LICENSE,sha256=kB7ZyyVGuCxC6lFG2Yts8vznxcztjtI_0dsKVa08Tg8,1063
+recent_state_summarizer-0.0.3.dist-info/METADATA,sha256=NvQqKKvZySn0uWGluPad-JcVvOdl3FyCSL5IfoU0h9o,2262
+recent_state_summarizer-0.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+recent_state_summarizer-0.0.3.dist-info/entry_points.txt,sha256=DSCCRVear3M5V-kOSQi0O08zsyb4MRm8lft6yPUz5Yc,69
+recent_state_summarizer-0.0.3.dist-info/top_level.txt,sha256=ZDx-fDfnpDSCeaxWI4rJq0otLE56owkWHOHKmj7zfZg,24
+recent_state_summarizer-0.0.3.dist-info/RECORD,,

{recent_state_summarizer-0.0.1.dist-info → recent_state_summarizer-0.0.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

recent_state_summarizer-0.0.1.dist-info/RECORD

@@ -1,10 +0,0 @@
-recent_state_summarizer/__init__.py,sha256=sXLh7g3KC4QCFxcZGBTpG2scR7hmmBsMjq6LqRptkRg,22
-recent_state_summarizer/__main__.py,sha256=mpPX-XZ6Ggrs41Lilc657by7ori5JwjRU4ek_eKK49k,966
-recent_state_summarizer/fetch.py,sha256=g4kQN9x3qXF8_p4y-nJNNiu3mcuuDJ4WbsV8b-D02KQ,1942
-recent_state_summarizer/summarize.py,sha256=8Adyo_qhJLrOxkYIWZWlEVJ1WACpBGf-37rvhqu_BJk,1910
-recent_state_summarizer-0.0.1.dist-info/LICENSE,sha256=kB7ZyyVGuCxC6lFG2Yts8vznxcztjtI_0dsKVa08Tg8,1063
-recent_state_summarizer-0.0.1.dist-info/METADATA,sha256=kDsfcAiVp-y_uRNkyUe1zXh8HFM9M6aetSC4QsizlYI,2140
-recent_state_summarizer-0.0.1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-recent_state_summarizer-0.0.1.dist-info/entry_points.txt,sha256=DSCCRVear3M5V-kOSQi0O08zsyb4MRm8lft6yPUz5Yc,69
-recent_state_summarizer-0.0.1.dist-info/top_level.txt,sha256=ZDx-fDfnpDSCeaxWI4rJq0otLE56owkWHOHKmj7zfZg,24
-recent_state_summarizer-0.0.1.dist-info/RECORD,,