anki-addons-dataset 1.3.0__tar.gz

anki_addons_dataset-1.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Aleksey Yablokov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

anki_addons_dataset-1.3.0/PKG-INFO

@@ -0,0 +1,76 @@
+Metadata-Version: 2.4
+Name: anki-addons-dataset
+Version: 1.3.0
+Summary: Collects, processes, and publishes structured data about Anki flashcard addons to HuggingFace.
+Author-email: Aleksey Yablokov <alex_ya@mailbox.org>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Aleks-Ya/anki-addons-dataset
+Project-URL: Repository, https://github.com/Aleks-Ya/anki-addons-dataset
+Project-URL: Dataset, https://huggingface.co/datasets/Ya-Alex/anki-addons
+Project-URL: Visualizations, https://huggingface.co/spaces/Ya-Alex/anki-addons
+Keywords: anki,addons,dataset,ankiweb,huggingface
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Education
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: bs4
+Requires-Dist: mdutils
+Requires-Dist: pyyaml
+Requires-Dist: XlsxWriter
+Requires-Dist: jsonschema
+Requires-Dist: selenium
+Requires-Dist: huggingface-hub
+Requires-Dist: pandas
+Requires-Dist: pyarrow
+Requires-Dist: seedir
+Requires-Dist: openpyxl
+Requires-Dist: pydiscourse
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Requires-Dist: freezegun; extra == "dev"
+Requires-Dist: bump-my-version; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: license-file
+
+# Anki Addons Dataset
+
+A HuggingFace dataset of addons for the [Anki](https://apps.ankiweb.net) flashcard program.
+
+## Install
+
+```bash
+pip install anki-addons-dataset
+```
+
+This installs the `anki-addons-dataset` command. The pipeline runs as a sequence of operations:
+
+```bash
+anki-addons-dataset init
+anki-addons-dataset download -d 2026-01-01
+anki-addons-dataset parse
+anki-addons-dataset report
+anki-addons-dataset bundle
+anki-addons-dataset upload
+```
+
+## Links
+- [Visualizations](https://huggingface.co/spaces/Ya-Alex/anki-addons) in HuggingFace Spaces
+- [HuggingFace Dataset](https://huggingface.co/datasets/Ya-Alex/anki-addons)
+- [Developer Guide](README-DEV.md)
+- [Sonar Qube](https://sonarcloud.io/project/overview?id=Aleks-Ya_anki-addons-dataset)
+- Anki
+    - [Anki home page](https://apps.ankiweb.net)
+    - [Anki Addons catalog](https://ankiweb.net/shared/addons)
+
+[![Unit-tests](https://github.com/Aleks-Ya/anki-addons-dataset/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/Aleks-Ya/anki-addons-dataset/actions/workflows/unit-tests.yml)
+[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=Aleks-Ya_anki-addons-dataset&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=Aleks-Ya_anki-addons-dataset)
+[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=Aleks-Ya_anki-addons-dataset&metric=coverage)](https://sonarcloud.io/summary/new_code?id=Aleks-Ya_anki-addons-dataset)

anki_addons_dataset-1.3.0/README.md

@@ -0,0 +1,33 @@
+# Anki Addons Dataset
+
+A HuggingFace dataset of addons for the [Anki](https://apps.ankiweb.net) flashcard program.
+
+## Install
+
+```bash
+pip install anki-addons-dataset
+```
+
+This installs the `anki-addons-dataset` command. The pipeline runs as a sequence of operations:
+
+```bash
+anki-addons-dataset init
+anki-addons-dataset download -d 2026-01-01
+anki-addons-dataset parse
+anki-addons-dataset report
+anki-addons-dataset bundle
+anki-addons-dataset upload
+```
+
+## Links
+- [Visualizations](https://huggingface.co/spaces/Ya-Alex/anki-addons) in HuggingFace Spaces
+- [HuggingFace Dataset](https://huggingface.co/datasets/Ya-Alex/anki-addons)
+- [Developer Guide](README-DEV.md)
+- [Sonar Qube](https://sonarcloud.io/project/overview?id=Aleks-Ya_anki-addons-dataset)
+- Anki
+    - [Anki home page](https://apps.ankiweb.net)
+    - [Anki Addons catalog](https://ankiweb.net/shared/addons)
+
+[![Unit-tests](https://github.com/Aleks-Ya/anki-addons-dataset/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/Aleks-Ya/anki-addons-dataset/actions/workflows/unit-tests.yml)
+[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=Aleks-Ya_anki-addons-dataset&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=Aleks-Ya_anki-addons-dataset)
+[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=Aleks-Ya_anki-addons-dataset&metric=coverage)](https://sonarcloud.io/summary/new_code?id=Aleks-Ya_anki-addons-dataset)

anki_addons_dataset-1.3.0/pyproject.toml

@@ -0,0 +1,70 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "anki-addons-dataset"
+dynamic = ["version"]
+description = "Collects, processes, and publishes structured data about Anki flashcard addons to HuggingFace."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Aleksey Yablokov", email = "alex_ya@mailbox.org" }]
+keywords = ["anki", "addons", "dataset", "ankiweb", "huggingface"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Topic :: Education",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "requests",
+    "bs4",
+    "mdutils",
+    "pyyaml",
+    "XlsxWriter",
+    "jsonschema",
+    "selenium",
+    "huggingface-hub",
+    "pandas",
+    "pyarrow",
+    "seedir",
+    "openpyxl",
+    "pydiscourse",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest",
+    "pytest-cov",
+    "pytest-mock",
+    "freezegun",
+    "bump-my-version",
+    "build",
+    "twine",
+]
+
+[project.urls]
+Homepage = "https://github.com/Aleks-Ya/anki-addons-dataset"
+Repository = "https://github.com/Aleks-Ya/anki-addons-dataset"
+Dataset = "https://huggingface.co/datasets/Ya-Alex/anki-addons"
+Visualizations = "https://huggingface.co/spaces/Ya-Alex/anki-addons"
+
+[project.scripts]
+anki-addons-dataset = "anki_addons_dataset.addon_catalog:main"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+namespaces = true
+
+[tool.setuptools.package-data]
+"*" = ["*.txt", "*.yaml", "*.json", "*.md"]
+
+[tool.setuptools.dynamic]
+version = { attr = "anki_addons_dataset.__version__" }

anki_addons_dataset-1.3.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

anki_addons_dataset-1.3.0/src/anki_addons_dataset/__init__.py

@@ -0,0 +1,10 @@
+"""Anki Addons Dataset package.
+
+Exposes ``__version__`` read verbatim from ``version.txt`` (the single source of truth,
+kept in sync by ``bump-my-version``). The value is already PEP 440-compliant: development
+builds use the ``.dev0`` suffix (e.g. ``1.3.0.dev0``) and releases drop it (``1.3.0``).
+"""
+
+from pathlib import Path
+
+__version__: str = (Path(__file__).parent / "version.txt").read_text().strip()

anki_addons_dataset-1.3.0/src/anki_addons_dataset/addon_catalog.py

@@ -0,0 +1,38 @@
+import logging
+from datetime import datetime
+from logging import Logger
+from pathlib import Path
+from typing import Optional
+
+from huggingface_hub import HfApi
+
+from anki_addons_dataset.argument.script_arguments import ScriptArguments, Operation
+from anki_addons_dataset.common.data_types import SnapshotDate, ReportDate
+from anki_addons_dataset.common.working_dir import WorkingDir
+from anki_addons_dataset.facade.facade import Facade
+from anki_addons_dataset.huggingface.hugging_face_client import HuggingFaceClient
+from anki_addons_dataset.common.log import Log
+
+log: Logger = logging.getLogger(__name__)
+
+def main() -> None:
+    Log.configure_logging()
+
+    arguments: ScriptArguments = ScriptArguments()
+
+    Log.set_log_level(arguments.get_log_level())
+    operation: Operation = arguments.get_operation()
+    snapshot_date: Optional[SnapshotDate] = arguments.get_snapshot_date()
+    log.info(f"Snapshot date: {snapshot_date}")
+    report_date: ReportDate = ReportDate(datetime.now().replace(microsecond=0))
+    log.info(f"Report date: {report_date}")
+
+    hf_api: HfApi = HfApi()
+    hugging_face_client: HuggingFaceClient = HuggingFaceClient(hf_api)
+    working_dir: WorkingDir = WorkingDir(Path.home() / "anki-addons-dataset")
+    facade: Facade = Facade(working_dir, hugging_face_client)
+    facade.process(operation, snapshot_date, report_date)
+
+
+if __name__ == "__main__":
+    main()

anki_addons_dataset-1.3.0/src/anki_addons_dataset/argument/script_arguments.py

@@ -0,0 +1,52 @@
+from argparse import ArgumentParser, Namespace, ArgumentTypeError
+from datetime import date, datetime
+from enum import Enum
+from typing import Optional
+import logging
+
+from anki_addons_dataset.common.data_types import SnapshotDate
+
+
+class Operation(Enum):
+    INIT = "init"
+    DOWNLOAD = "download"
+    PARSE = "parse"
+    REPORT = "report"
+    BUNDLE = "bundle"
+    UPLOAD = "upload"
+
+
+class ScriptArguments:
+    def __init__(self):
+        parser: ArgumentParser = ArgumentParser()
+        parser.add_argument('operation')
+        parser.add_argument('-d', '--snapshot-date', type=self.__valid_date)
+        parser.add_argument('-l', '--log-level', type=self.__valid_log_level, default='INFO')
+        self.namespace: Namespace = parser.parse_args()
+
+    def get_snapshot_date(self) -> Optional[SnapshotDate]:
+        return self.namespace.snapshot_date
+
+    def get_operation(self) -> Operation:
+        return Operation[self.namespace.operation.upper()]
+
+    def get_log_level(self) -> int:
+        return self.namespace.log_level
+
+    @staticmethod
+    def __valid_date(s: str) -> date:
+        try:
+            return datetime.strptime(s, "%Y-%m-%d").date()
+        except ValueError:
+            msg: str = f"Not a valid date: '{s}'. Expected format: YYYY-MM-DD."
+            raise ArgumentTypeError(msg)
+
+    @staticmethod
+    def __valid_log_level(s: str) -> int:
+        level_name: str = s.upper()
+        level_mapping: dict[str, int] = logging.getLevelNamesMapping()
+        if level_name not in level_mapping:
+            valid_levels: list[str] = list(level_mapping.keys())
+            msg: str = f"Not a valid log level: '{s}'. Expected one of: {', '.join(valid_levels)}."
+            raise ArgumentTypeError(msg)
+        return level_mapping[level_name]

anki_addons_dataset-1.3.0/src/anki_addons_dataset/bundle/dataset_bundle.py

@@ -0,0 +1,60 @@
+import logging
+import shutil
+from logging import Logger
+from pathlib import Path
+from typing import Optional
+
+from anki_addons_dataset.common.data_types import SnapshotDate
+from anki_addons_dataset.common.working_dir import WorkingDir, SnapshotDir
+from anki_addons_dataset.huggingface.hugging_face import HuggingFace
+
+log: Logger = logging.getLogger(__name__)
+
+
+class DatasetBundle:
+    def __init__(self, working_dir: WorkingDir):
+        self.__working_dir: WorkingDir = working_dir
+
+    def create_bundle(self) -> None:
+        bundle_dir: Path = self.__working_dir.get_bundle_dir()
+        log.info(f"Creating bundle in {bundle_dir}")
+        shutil.rmtree(bundle_dir, ignore_errors=True)
+        self.__copy_snapshots(bundle_dir)
+        self.__copy_latest_snapshot(bundle_dir)
+        HuggingFace.create_dataset_card(bundle_dir)
+
+    def __copy_snapshots(self, bundle_dir: Path):
+        bundle_history_dir: Path = bundle_dir / "history"
+        bundle_history_dir.mkdir(parents=True, exist_ok=True)
+        for snapshot_dir in self.__working_dir.list_snapshot_dirs():
+            snapshot_date: SnapshotDate = snapshot_dir.snapshot_dir_to_snapshot_date()
+            base_name: str = f"{snapshot_date}"
+            output_dir: Path = bundle_history_dir / base_name
+            self.__create_zip(snapshot_dir.get_raw_dir(), output_dir, "raw")
+            self.__create_zip(snapshot_dir.get_stage_dir(), output_dir, "stage")
+            log.info(f"Copying {snapshot_dir.get_final_dir()} to {output_dir}")
+            shutil.copytree(snapshot_dir.get_final_dir(), output_dir, dirs_exist_ok=True)
+            src_metadata_file: Path = snapshot_dir.get_metadata_json()
+            dest_metadata_file: Path = output_dir / src_metadata_file.name
+            log.info(f"Copying {src_metadata_file} to {dest_metadata_file}")
+            shutil.copyfile(src_metadata_file, dest_metadata_file)
+
+    def __copy_latest_snapshot(self, bundle_dir: Path):
+        latest_dir: Path = bundle_dir / "latest"
+        log.info(f"Copying the latest snapshot: {latest_dir}")
+        latest_snapshot_dir: Optional[SnapshotDir] = self.__working_dir.get_latest_snapshot_dir()
+        if not latest_snapshot_dir:
+            raise ValueError("No snapshots found")
+        final_dir: Path = latest_snapshot_dir.get_final_dir()
+        shutil.copytree(final_dir, latest_dir)
+        src_metadata_file: Path = latest_snapshot_dir.get_metadata_json()
+        dest_metadata_file: Path = latest_dir / src_metadata_file.name
+        log.info(f"Copying {src_metadata_file} to {dest_metadata_file}")
+        shutil.copyfile(src_metadata_file, dest_metadata_file)
+
+    @staticmethod
+    def __create_zip(source_dir: Path, output_dir: Path, output_base_name: str) -> None:
+        zip_base_name: str = str(output_dir / f"{output_base_name}")
+        archive_format: str = "zip"
+        log.info(f"Creating zip: {zip_base_name}.{archive_format}")
+        shutil.make_archive(zip_base_name, archive_format, source_dir)

anki_addons_dataset-1.3.0/src/anki_addons_dataset/collector/addon_infos_collector.py

@@ -0,0 +1,46 @@
+import logging
+from logging import Logger
+
+from anki_addons_dataset.collector.ankiforum.ankiforum_enricher import AnkiForumEnricher
+from anki_addons_dataset.collector.github.github_enricher import GithubEnricher
+from anki_addons_dataset.collector.overrider.overrider import Overrider
+from anki_addons_dataset.common.data_types import AddonInfo, AddonHeader, AddonInfos
+from anki_addons_dataset.collector.ankiweb.ankiweb_service import AnkiWebService
+
+log: Logger = logging.getLogger(__name__)
+
+
+class AddonInfosCollector:
+    def __init__(self, ankiweb_service: AnkiWebService, github_enricher: GithubEnricher,
+                 anki_forum_enricher: AnkiForumEnricher, overrider: Overrider):
+        self.__ankiweb_service: AnkiWebService = ankiweb_service
+        self.__github_enricher: GithubEnricher = github_enricher
+        self.__anki_forum_enricher: AnkiForumEnricher = anki_forum_enricher
+        self.__overrider: Overrider = overrider
+
+    def collect_addons(self) -> AddonInfos:
+        self.__github_enricher.start()
+        self.__anki_forum_enricher.start()
+
+        addon_headers: list[AddonHeader] = self.__ankiweb_service.get_headers()
+        log.info(f"Addon number: {len(addon_headers)}")
+        addons_infos: AddonInfos = self.__get_addon_infos(addon_headers)
+        log.info("All addons are added to queue")
+        self.__github_enricher.wait_download_finish()
+        self.__anki_forum_enricher.wait_download_finish()
+        github_enriched_addon_infos: AddonInfos = self.__github_enricher.enrich(addons_infos)
+        anki_forum_enriched_addon_infos: AddonInfos = self.__anki_forum_enricher.enrich(github_enriched_addon_infos)
+        log.info("All addons are enriched")
+
+        overridden_addon_infos: AddonInfos = self.__overrider.override(anki_forum_enriched_addon_infos)
+        return overridden_addon_infos
+
+    def __get_addon_infos(self, addon_headers: list[AddonHeader]) -> AddonInfos:
+        addon_infos: list[AddonInfo] = []
+        for i, addon_header in enumerate(addon_headers):
+            log.info(f"Parsing addon page: {addon_header.id} ({i}/{len(addon_headers)})")
+            addon_info: AddonInfo = self.__ankiweb_service.get_addon_info(addon_header)
+            self.__github_enricher.download_in_background(addon_info)
+            self.__anki_forum_enricher.download_in_background(addon_info)
+            addon_infos.append(addon_info)
+        return AddonInfos(addon_infos)

anki_addons_dataset-1.3.0/src/anki_addons_dataset/collector/aggregator.py

@@ -0,0 +1,17 @@
+from typing import cast
+
+from anki_addons_dataset.common.data_types import Aggregation, AddonInfos, AddonInfo
+
+
+class Aggregator:
+
+    @staticmethod
+    def aggregate(addon_infos: AddonInfos) -> Aggregation:
+        addon_number: int = len(cast(list[AddonInfo], addon_infos))
+        addon_with_github_number: int = len([addon for addon in addon_infos if addon.github.github_repo])
+        addon_with_anki_forum_page_number: int = len([addon for addon in addon_infos
+                                                      if addon.forum and addon.forum.anki_forum_url])
+        addon_with_unit_tests_number: int = len([addon for addon in addon_infos
+                                                 if addon.github.tests_count and addon.github.tests_count > 0])
+        return Aggregation(addon_number, addon_with_github_number, addon_with_anki_forum_page_number,
+                           addon_with_unit_tests_number)

anki_addons_dataset-1.3.0/src/anki_addons_dataset/collector/ankiforum/ankiforum_enricher.py

@@ -0,0 +1,52 @@
+from pathlib import Path
+from typing import Optional
+import logging
+from logging import Logger
+
+from anki_addons_dataset.collector.ankiforum.ankiforum_service import AnkiForumService
+from anki_addons_dataset.collector.ankiforum.ankiforum_topic import AnkiForumTopic
+from anki_addons_dataset.collector.enricher import Enricher
+from anki_addons_dataset.common.data_types import AddonInfo, AddonId, \
+    AnkiForumInfo, TopicSlug, TopicId, LastPostedAt, AddonInfos, PostsCount, URL
+from anki_addons_dataset.common.json_helper import JsonHelper
+from anki_addons_dataset.common.working_dir import SnapshotDir
+
+log: Logger = logging.getLogger(__name__)
+
+
+class AnkiForumEnricher(Enricher):
+    __name: str = "AnkiForum"
+
+    def __init__(self, snapshot_dir: SnapshotDir, anki_forum_service: AnkiForumService):
+        super().__init__(name=self.__name)
+        self.__stage_dir: Path = snapshot_dir.get_stage_dir() / "3-enricher" / "forum"
+        self.__anki_forum_service: AnkiForumService = anki_forum_service
+        self.__anki_forum_infos: dict[AddonId, Optional[AnkiForumInfo]] = {}
+
+    def enrich(self, addon_infos: AddonInfos) -> AddonInfos:
+        return AddonInfos([self.__enrich(addon_info, self.__anki_forum_infos[addon_info.header.id])
+                           for addon_info in addon_infos])
+
+    def _download(self, addon_info: AddonInfo) -> None:
+        anki_forum_url: Optional[URL] = addon_info.forum.anki_forum_url
+        if anki_forum_url:
+            topic_slug: Optional[TopicSlug] = AnkiForumTopic.extract_topic_slug(anki_forum_url)
+            topic_id: Optional[TopicId] = AnkiForumTopic.extract_topic_id(anki_forum_url)
+            last_posted_at: Optional[LastPostedAt] = self.__anki_forum_service.get_last_posted_at(topic_slug, topic_id)
+            posts_count: Optional[PostsCount] = self.__anki_forum_service.get_posts_count(topic_slug, topic_id)
+            anki_forum: Optional[AnkiForumInfo] = AnkiForumInfo(anki_forum_url, topic_slug, topic_id, last_posted_at,
+                                                                posts_count)
+        else:
+            anki_forum: Optional[AnkiForumInfo] = None
+        self.__anki_forum_infos[addon_info.header.id] = anki_forum
+
+    def _done(self) -> int:
+        return len(self.__anki_forum_infos)
+
+    def __enrich(self, addon_info: AddonInfo, anki_forum_info: Optional[AnkiForumInfo]) -> AddonInfo:
+        enriched_addon_info: AddonInfo = AddonInfo(
+            addon_info.header, addon_info.page, addon_info.github, anki_forum_info)
+        addon_json_file: Path = self.__stage_dir / f"{addon_info.header.id}.json"
+        JsonHelper.write_addon_info_to_file(addon_info, addon_json_file)
+        log.info(f"Enriched ({self.__name}): {addon_info.header.id}")
+        return enriched_addon_info

anki_addons_dataset-1.3.0/src/anki_addons_dataset/collector/ankiforum/ankiforum_service.py

@@ -0,0 +1,60 @@
+import json
+import logging
+from datetime import datetime, timezone
+from logging import Logger
+from pathlib import Path
+from typing import Optional
+
+from pydiscourse import DiscourseClient
+
+from anki_addons_dataset.common.data_types import TopicId, TopicSlug, LastPostedAt, PostsCount
+from anki_addons_dataset.common.working_dir import SnapshotDir
+
+log: Logger = logging.getLogger(__name__)
+
+
+class AnkiForumService:
+
+    def __init__(self, discourse_client: DiscourseClient, snapshot_dir: SnapshotDir, offline: bool):
+        self.__discourse_client: DiscourseClient = discourse_client
+        raw_dir: Path = snapshot_dir.get_raw_dir() / "3-forum"
+        self.__topic = raw_dir / "topic"
+        self.__topic.mkdir(parents=True, exist_ok=True)
+        self.__offline: bool = offline
+
+    def get_last_posted_at(self, topic_slug: Optional[TopicSlug], topic_id: Optional[TopicId]) -> Optional[
+        LastPostedAt]:
+        topic_dict: Optional[dict] = self.__read_topic_json(topic_slug, topic_id)
+        if not topic_dict:
+            return None
+        last_posted_at_str: str = topic_dict['last_posted_at']
+        last_posted_at: datetime = datetime.strptime(last_posted_at_str, "%Y-%m-%dT%H:%M:%S.%fZ").replace(
+            tzinfo=timezone.utc)
+        return LastPostedAt(last_posted_at)
+
+    def get_posts_count(self, topic_slug: Optional[TopicSlug], topic_id: Optional[TopicId]) -> Optional[PostsCount]:
+        topic_dict: Optional[dict] = self.__read_topic_json(topic_slug, topic_id)
+        if not topic_dict:
+            return None
+        posts_count_str: str = topic_dict['posts_count']
+        return PostsCount(int(posts_count_str))
+
+    def __read_topic_json(self, topic_slug: Optional[TopicSlug], topic_id: Optional[TopicId]) -> Optional[dict]:
+        json_file: Path = self.__topic / f"{topic_id}.json"
+        if json_file.exists():
+            json_str: str = json_file.read_text()
+            if json_str == "None":
+                return None
+            topic_dict: dict = json.loads(json_str)
+        else:
+            if self.__offline:
+                log.debug(f"Offline mode is enabled. Skip fetching last_posted_at for topic '{topic_slug}/{topic_id}'")
+                return None
+            topic_dict: dict = self.__discourse_client.topic(
+                topic_slug, topic_id, override_request_kwargs={"allow_redirects": True})
+            if topic_dict is None:
+                json_file.write_text("None")
+                return None
+            topic_json: str = json.dumps(topic_dict, indent=2)
+            json_file.write_text(topic_json)
+        return topic_dict

anki_addons_dataset-1.3.0/src/anki_addons_dataset/collector/ankiforum/ankiforum_topic.py

@@ -0,0 +1,31 @@
+import re
+from re import Match
+from typing import Optional
+
+from anki_addons_dataset.common.data_types import URL, TopicId, TopicSlug
+
+
+class AnkiForumTopic:
+    __url_pattern: re.Pattern = re.compile(r'^https://forums\.ankiweb\.net/t/([^/]+)/(\d+)')
+
+    @staticmethod
+    def extract_topic_slug(topic_url: Optional[URL]) -> Optional[TopicSlug]:
+        if topic_url is None:
+            return None
+        match: Optional[Match[str]] = re.search(AnkiForumTopic.__url_pattern, topic_url)
+        if match:
+            topic_slug_str: str = match.group(1)
+            return TopicSlug(topic_slug_str)
+        else:
+            raise ValueError(f"Cannot extract Topic Slug from Anki Forum Topic URL: '{topic_url}'")
+
+    @staticmethod
+    def extract_topic_id(topic_url: Optional[URL]) -> Optional[TopicId]:
+        if topic_url is None:
+            return None
+        match: Optional[Match[str]] = re.search(AnkiForumTopic.__url_pattern, topic_url)
+        if match:
+            topic_id_str: str = match.group(2)
+            return TopicId(int(topic_id_str))
+        else:
+            raise ValueError(f"Cannot extract Topic ID from Anki Forum Topic URL: '{topic_url}'")

anki_addons_dataset-1.3.0/src/anki_addons_dataset/collector/ankiweb/addon_branch_parser.py

@@ -0,0 +1,27 @@
+import datetime
+import re
+from datetime import date
+from re import Match
+from typing import Optional
+
+from anki_addons_dataset.common.data_types import AddonBranch, AnkiVersion
+
+
+class AddonBranchParser:
+    __addon_branch_re: re.Pattern[str] = re.compile(
+        r'^(?P<min>\d+(?:\.\d+)*)'
+        r'(?:-(?P<max>\d+(?:\.\d+)*\+?)|(?P<plus>\+))?'
+        r' \(Updated (?P<updated>\d{4}-\d{2}-\d{2})\)$'
+    )
+
+    @staticmethod
+    def extract_addon_branch(addon_branch_str: str) -> AddonBranch:
+        match: Optional[Match[str]] = AddonBranchParser.__addon_branch_re.fullmatch(addon_branch_str)
+        if not match:
+            raise ValueError(f"Cannot parse version string: {addon_branch_str!r}")
+        min_addon_version: AnkiVersion = AnkiVersion(match.group("min"))
+        max_addon_version: AnkiVersion = AnkiVersion(match.group("max"))
+        if max_addon_version is None and match.group("plus") is not None:
+            max_addon_version = AnkiVersion("+")
+        updated: date = datetime.date.fromisoformat(match.group("updated"))
+        return AddonBranch(min_anki_version=min_addon_version, max_anki_version=max_addon_version, updated=updated)

anki_addons_dataset-1.3.0/src/anki_addons_dataset/collector/ankiweb/addon_page_downloader.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+import logging
+from logging import Logger
+
+from anki_addons_dataset.collector.ankiweb.addon_page_parser import AddonPageParser
+from anki_addons_dataset.collector.ankiweb.page_downloader import PageDownloader
+from anki_addons_dataset.common.data_types import AddonId, AddonInfo, AddonHeader, HtmlStr
+from anki_addons_dataset.common.json_helper import JsonHelper
+from anki_addons_dataset.common.working_dir import SnapshotDir
+
+log: Logger = logging.getLogger(__name__)
+
+
+class AddonPageDownloader:
+    def __init__(self, page_downloader: PageDownloader, snapshot_dir: SnapshotDir, addon_page_parser: AddonPageParser,
+                 offline: bool) -> None:
+        self.__addon_page_parser: AddonPageParser = addon_page_parser
+        self.__page_downloader: PageDownloader = page_downloader
+        self.__raw_dir: Path = snapshot_dir.get_raw_dir() / "1-anki-web"
+        self.__stage_dir: Path = snapshot_dir.get_stage_dir() / "1-anki-web"
+        self.__offline: bool = offline
+
+    def get_addon_info(self, addon_header: AddonHeader) -> AddonInfo:
+        try:
+            html: HtmlStr = self.__load_addon_page(addon_header.id)
+            addon_info: AddonInfo = self.__addon_page_parser.parse_addon_page(addon_header, html)
+            addon_json_file: Path = self.__stage_dir / "addon" / f"{addon_header.id}.json"
+            JsonHelper.write_addon_info_to_file(addon_info, addon_json_file)
+            return addon_info
+        except Exception as e:
+            raise RuntimeError(f"Cannot get addon info: {addon_header.id}") from e
+
+    def __load_addon_page(self, addon_id: AddonId) -> HtmlStr:
+        raw_file: Path = self.__raw_dir / "addon" / f"{addon_id}.html"
+        if raw_file.exists() and raw_file.stat().st_size == 0:
+            raw_file.unlink()
+            log.info(f"Removed empty file: {raw_file}")
+        if not raw_file.exists():
+            log.debug(f"Downloading addon page to {raw_file}")
+            if self.__offline:
+                raise RuntimeError("Offline mode is enabled")
+            raw_file.parent.mkdir(parents=True, exist_ok=True)
+            html: HtmlStr = self.__page_downloader.load_page(f"https://ankiweb.net/shared/info/{addon_id}")
+            raw_file.write_text(html)
+        return HtmlStr(raw_file.read_text())