tumblrbot 1.9.5__py3-none-any.whl → 1.9.7__py3-none-any.whl

tumblrbot/__main__.py

@@ -1,3 +1,5 @@
+from sys import exit as sys_exit
+
 from openai import OpenAI
 from rich.prompt import Confirm
 from rich.traceback import install
@@ -17,16 +19,16 @@ def main() -> None:
     tokens = Tokens.load()
     with OpenAI(api_key=tokens.openai_api_key) as openai, TumblrSession(tokens) as tumblr:
         if Confirm.ask("Download latest posts?", default=False):
-            PostDownloader(openai=openai, tumblr=tumblr).main()
+            PostDownloader(openai, tumblr).main()
 
-        examples_writer = ExamplesWriter(openai=openai, tumblr=tumblr)
+        examples_writer = ExamplesWriter(openai, tumblr)
         if Confirm.ask("Create training data?", default=False):
             examples_writer.main()
 
         if Confirm.ask("Remove training data flagged by the OpenAI moderation? [bold]This can sometimes resolve errors with fine-tuning validation, but is slow.", default=False):
             examples_writer.filter_examples()
 
-        fine_tuner = FineTuner(openai=openai, tumblr=tumblr)
+        fine_tuner = FineTuner(openai, tumblr)
         fine_tuner.print_estimates()
 
         message = "Resume monitoring the previous fine-tuning process?" if FlowClass.config.job_id else "Upload data to OpenAI for fine-tuning?"
@@ -34,4 +36,8 @@ def main() -> None:
             fine_tuner.main()
 
         if Confirm.ask("Generate drafts?", default=False):
-            DraftGenerator(openai=openai, tumblr=tumblr).main()
+            DraftGenerator(openai, tumblr).main()
+
+
+if __name__ == "__main__":
+    sys_exit(main())

tumblrbot/flow/examples.py

@@ -1,52 +1,45 @@
-import re
+from collections.abc import Generator
 from itertools import batched
 from json import loads
 from math import ceil
-from re import search
-from typing import IO, TYPE_CHECKING, override
+from re import IGNORECASE
+from re import compile as re_compile
+from typing import TYPE_CHECKING, override
 
-import rich
-from openai import BadRequestError
+from openai import RateLimitError
+from rich import print as rich_print
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random_exponential
 
 from tumblrbot.utils.common import FlowClass, PreviewLive
-from tumblrbot.utils.models import Example, Post
+from tumblrbot.utils.models import Example, Message, Post
 
 if TYPE_CHECKING:
-    from collections.abc import Generator
+    from collections.abc import Generator, Iterable
     from pathlib import Path
 
+    from openai._types import SequenceNotStr
+    from openai.types import ModerationCreateResponse, ModerationMultiModalInputParam
+
 
 class ExamplesWriter(FlowClass):
     @override
     def main(self) -> None:
         self.config.examples_file.parent.mkdir(parents=True, exist_ok=True)
 
-        with self.config.examples_file.open("w", encoding="utf_8") as fp:
-            for user_message, assistant_response in self.get_custom_prompts():
-                self.write_example(
-                    user_message,
-                    assistant_response,
-                    fp,
-                )
-
-            for post in self.get_valid_posts():
-                self.write_example(
-                    self.config.user_message,
-                    str(post),
-                    fp,
-                )
-
-        rich.print(f"[bold]The examples file can be found at: '{self.config.examples_file}'\n")
-
-    def write_example(self, user_message: str, assistant_message: str, fp: IO[str]) -> None:
-        example = Example(
+        examples = [self.create_example(*prompt) for prompt in self.get_custom_prompts()]
+        examples.extend(self.create_example(self.config.user_message, str(post)) for post in self.get_valid_posts())
+        self.write_examples(examples)
+
+        rich_print(f"[bold]The examples file can be found at: '{self.config.examples_file}'\n")
+
+    def create_example(self, user_message: str, assistant_message: str) -> Example:
+        return Example(
             messages=[
-                Example.Message(role="developer", content=self.config.developer_message),
-                Example.Message(role="user", content=user_message),
-                Example.Message(role="assistant", content=assistant_message),
+                Message(role="developer", content=self.config.developer_message),
+                Message(role="user", content=user_message),
+                Message(role="assistant", content=assistant_message),
             ],
         )
-        fp.write(f"{example.model_dump_json()}\n")
 
     def get_custom_prompts(self) -> Generator[tuple[str, str]]:
         self.config.custom_prompts_file.parent.mkdir(parents=True, exist_ok=True)
@@ -57,13 +50,19 @@ class ExamplesWriter(FlowClass):
                 data: dict[str, str] = loads(line)
                 yield from data.items()
 
+    # This function mostly exists to make writing examples atomic.
+    def write_examples(self, examples: Iterable[Example]) -> None:
+        with self.config.examples_file.open("w", encoding="utf_8") as fp:
+            for example in examples:
+                fp.write(f"{example.model_dump_json()}\n")
+
     def get_valid_posts(self) -> Generator[Post]:
         for path in self.get_data_paths():
             posts = list(self.get_valid_posts_from_path(path))
             yield from posts[-self.config.post_limit :]
 
     def get_valid_posts_from_path(self, path: Path) -> Generator[Post]:
-        pattern = re.compile("|".join(self.config.filtered_words), re.IGNORECASE)
+        pattern = re_compile("|".join(self.config.filtered_words), IGNORECASE)
         with path.open("rb") as fp:
             for line in fp:
                 post = Post.model_validate_json(line)
@@ -71,30 +70,28 @@ class ExamplesWriter(FlowClass):
                     yield post
 
     def filter_examples(self) -> None:
-        examples = self.config.examples_file.read_text("utf_8").splitlines()
-        with self.config.examples_file.open("w", encoding="utf_8") as fp:
-            batch_size = self.get_moderation_batch_size()
-            removed = 0
-
-            with PreviewLive() as live:
-                for batch in live.progress.track(
-                    batched(examples, batch_size, strict=False),
-                    ceil(len(examples) / batch_size),
-                    description="Removing flagged posts...",
-                ):
-                    response = self.openai.moderations.create(input=list(batch))
-                    for example, moderation in zip(batch, response.results, strict=True):
-                        if moderation.flagged:
-                            removed += 1
-                        else:
-                            fp.write(f"{example}\n")
-            rich.print(f"[red]Removed {removed} posts.\n")
-
-    def get_moderation_batch_size(self) -> int:
-        try:
-            self.openai.moderations.create(input=[""] * self.config.max_moderation_batch_size)
-        except BadRequestError as error:
-            message = error.response.json()["error"]["message"]
-            if match := search(r"(\d+)\.", message):
-                return int(match.group(1))
-        return self.config.max_moderation_batch_size
+        raw_examples = self.config.examples_file.read_bytes().splitlines()
+        old_examples = map(Example.model_validate_json, raw_examples)
+        new_examples: list[Example] = []
+        with PreviewLive() as live:
+            for batch in live.progress.track(
+                batched(old_examples, self.config.moderation_batch_size, strict=False),
+                ceil(len(raw_examples) / self.config.moderation_batch_size),
+                description="Removing flagged posts...",
+            ):
+                response = self.create_moderation_batch(tuple(map(Example.get_assistant_message, batch)))
+                new_examples.extend(example for example, moderation in zip(batch, response.results, strict=True) if not moderation.flagged)
+
+        self.write_examples(new_examples)
+
+        rich_print(f"[red]Removed {len(raw_examples) - len(new_examples)} posts.\n")
+
+    @retry(
+        stop=stop_after_attempt(10),
+        wait=wait_random_exponential(),
+        retry=retry_if_exception_type(RateLimitError),
+        before_sleep=lambda state: rich_print(f"[yellow]OpenAI rate limit exceeded. Waiting for {state.idle_for} seconds..."),
+        reraise=True,
+    )
+    def create_moderation_batch(self, api_input: str | SequenceNotStr[str] | Iterable[ModerationMultiModalInputParam]) -> ModerationCreateResponse:
+        return self.openai.moderations.create(input=api_input)

tumblrbot/flow/fine_tune.py

@@ -3,9 +3,9 @@ from textwrap import dedent
 from time import sleep
 from typing import TYPE_CHECKING, override
 
-import rich
-from rich import progress
+from rich import print as rich_print
 from rich.console import Console
+from rich.progress import open as progress_open
 from rich.prompt import Confirm
 from tiktoken import encoding_for_model, get_encoding
 
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
 class FineTuner(FlowClass):
     @staticmethod
     def dedent_print(text: str) -> None:
-        rich.print(dedent(text).lstrip())
+        rich_print(dedent(text).lstrip())
 
     @override
     def main(self) -> None:
@@ -55,12 +55,12 @@ class FineTuner(FlowClass):
         if self.config.job_id:
             return self.poll_job_status()
 
-        with progress.open(self.config.examples_file, "rb", description=f"Uploading [purple]{self.config.examples_file}[/]...") as fp:
+        with progress_open(self.config.examples_file, "rb", description=f"Uploading [purple]{self.config.examples_file}[/]...") as fp:
             file = self.openai.files.create(
                 file=fp,
                 purpose="fine-tune",
             )
-        rich.print()
+        rich_print()
 
         job = self.openai.fine_tuning.jobs.create(
             model=self.config.base_model,
@@ -96,7 +96,7 @@ class FineTuner(FlowClass):
         if job.status != "succeeded":
             if Confirm.ask("[gray62]Delete uploaded examples file?", default=False):
                 self.openai.files.delete(job.training_file)
-                rich.print()
+                rich_print()
 
             if job.status == "failed" and job.error is not None:
                 raise RuntimeError(job.error.message)

tumblrbot/flow/generate.py

@@ -1,21 +1,20 @@
+from dataclasses import dataclass
 from functools import cache
 from random import choice, random, sample
 from typing import TYPE_CHECKING, override
 
-import rich
-from pydantic import ConfigDict
+from rich import print as rich_print
 from rich.prompt import IntPrompt
 
 from tumblrbot.utils.common import FlowClass, PreviewLive
-from tumblrbot.utils.models import Post
+from tumblrbot.utils.models import Block, Post
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
 
+@dataclass(frozen=True)
 class DraftGenerator(FlowClass):
-    model_config = ConfigDict(frozen=True)  # Makes this class hashable.
-
     @override
     def main(self) -> None:
         self.config.draft_count = IntPrompt.ask("How many drafts should be generated?", default=self.config.draft_count)
@@ -32,7 +31,7 @@ class DraftGenerator(FlowClass):
                     exception.add_note(f"📉 An error occurred! Generated {i} draft(s) before failing. {message}")
                     raise
 
-        rich.print(f":chart_increasing: [bold green]Generated {self.config.draft_count} draft(s).[/] {message}")
+        rich_print(f":chart_increasing: [bold green]Generated {self.config.draft_count} draft(s).[/] {message}")
 
     def generate_post(self) -> Post:
         if original := self.get_random_post():
@@ -48,7 +47,7 @@ class DraftGenerator(FlowClass):
             tags = tags.tags
 
         return Post(
-            content=[Post.Block(type="text", text=text)],
+            content=[Block(text=text)],
             tags=tags or [],
             parent_tumblelog_uuid=original.blog.uuid,
             parent_post_id=original.id,

tumblrbot/utils/common.py

@@ -1,23 +1,25 @@
 from abc import abstractmethod
-from pathlib import Path
+from dataclasses import dataclass
 from random import choice
-from typing import ClassVar, Self, override
+from typing import TYPE_CHECKING, ClassVar
 
-from openai import OpenAI
-from pydantic import ConfigDict
+from openai import OpenAI  # noqa: TC002
 from rich._spinners import SPINNERS
-from rich.console import RenderableType
 from rich.live import Live
 from rich.progress import MofNCompleteColumn, Progress, SpinnerColumn, TimeElapsedColumn
 from rich.table import Table
 
-from tumblrbot.utils.models import Config, FullyValidatedModel
-from tumblrbot.utils.tumblr import TumblrSession
+from tumblrbot.utils.models import Config
+from tumblrbot.utils.tumblr import TumblrSession  # noqa: TC001
 
+if TYPE_CHECKING:
+    from pathlib import Path
 
-class FlowClass(FullyValidatedModel):
-    model_config = ConfigDict(arbitrary_types_allowed=True)
+    from rich.console import RenderableType
 
+
+@dataclass(frozen=True)
+class FlowClass:
     config: ClassVar = Config.load()
 
     openai: OpenAI
@@ -48,11 +50,6 @@ class PreviewLive(Live):
 
         self.custom_update()
 
-    @override
-    def __enter__(self) -> Self:
-        super().__enter__()
-        return self
-
     def custom_update(self, *renderables: RenderableType | None) -> None:
         table = Table.grid()
         table.add_row(self.progress)

tumblrbot/utils/models.py

@@ -1,17 +1,19 @@
-from collections.abc import Generator
 from getpass import getpass
 from pathlib import Path
-from typing import Annotated, Any, Literal, Self, override
+from tomllib import loads
+from typing import TYPE_CHECKING, Annotated, Any, Literal, Self, override
 
-import rich
-from openai.types import ChatModel
+from openai.types import ChatModel  # noqa: TC002
 from pydantic import BaseModel, ConfigDict, Field, NonNegativeFloat, NonNegativeInt, PlainSerializer, PositiveFloat, PositiveInt, model_validator
-from pydantic.json_schema import SkipJsonSchema
+from pydantic.json_schema import SkipJsonSchema  # noqa: TC002
 from requests_oauthlib import OAuth1Session
+from rich import print as rich_print
 from rich.panel import Panel
 from rich.prompt import Prompt
 from tomlkit import comment, document, dumps  # pyright: ignore[reportUnknownVariableType]
-from tomllib import loads
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
 
 
 class FullyValidatedModel(BaseModel):
@@ -58,7 +60,7 @@ class Config(FileSyncSettings):
 
     # Writing Examples
     post_limit: NonNegativeInt = Field(0, description="The number of the most recent posts from each blog that should be included in the training data.")
-    max_moderation_batch_size: PositiveInt = Field(100, description="The number of posts, at most, to submit to the OpenAI moderation API. This is also capped by the API.")
+    moderation_batch_size: PositiveInt = Field(25, description="The number of posts at a time to submit to the OpenAI moderation API.")
     custom_prompts_file: Path = Field(Path("custom_prompts.jsonl"), description="Where to read in custom prompts from.")
     filtered_words: list[str] = Field([], description="A case-insensitive list of disallowed words used to filter out training data. Regular expressions are allowed, but must be escaped.")
 
@@ -80,7 +82,7 @@ class Config(FileSyncSettings):
 
     # Generating
     upload_blog_identifier: str = Field("", description="The identifier of the blog which generated drafts will be uploaded to. This must be a blog associated with the same account as the configured Tumblr secret tokens.")
-    draft_count: PositiveInt = Field(150, description="The number of drafts to process. This will affect the number of tokens used with OpenAI")
+    draft_count: PositiveInt = Field(100, description="The number of drafts to process. This will affect the number of tokens used with OpenAI")
     tags_chance: NonNegativeFloat = Field(0.1, description="The chance to generate tags for any given post. This will use more OpenAI tokens.")
     tags_developer_message: str = Field("You will be provided with a block of text, and your task is to extract a very short list of the most important subjects from it.", description="The developer message used to generate tags.")
     reblog_blog_identifiers: list[str] = Field([], description="The identifiers of blogs that can be reblogged from when generating drafts.")
@@ -88,13 +90,15 @@ class Config(FileSyncSettings):
     reblog_user_message: str = Field("Please write a comical Tumblr post in response to the following post:\n\n{}", description="The format string for the user message used to reblog posts.")
 
     @override
-    def model_post_init(self, _: object) -> None:
+    def model_post_init(self, context: object) -> None:
+        super().model_post_init(context)
+
         if not self.download_blog_identifiers:
-            rich.print("Enter the [cyan]identifiers of your blogs[/] that data should be [bold purple]downloaded[/] from, separated by commas.")
+            rich_print("Enter the [cyan]identifiers of your blogs[/] that data should be [bold purple]downloaded[/] from, separated by commas.")
             self.download_blog_identifiers = list(map(str.strip, Prompt.ask("[bold][Example] [dim]staff.tumblr.com,changes").split(",")))
 
         if not self.upload_blog_identifier:
-            rich.print("Enter the [cyan]identifier of your blog[/] that drafts should be [bold purple]uploaded[/] to.")
+            rich_print("Enter the [cyan]identifier of your blog[/] that drafts should be [bold purple]uploaded[/] to.")
             self.upload_blog_identifier = Prompt.ask("[bold][Example] [dim]staff.tumblr.com or changes").strip()
 
 
@@ -108,8 +112,20 @@ class Tokens(FileSyncSettings):
     openai_api_key: str = ""
     tumblr: Tumblr = Tumblr()
 
+    @staticmethod
+    def online_token_prompt(url: str, *tokens: str) -> Generator[str]:
+        formatted_token_string = " and ".join(f"[cyan]{token}[/]" for token in tokens)
+
+        rich_print(f"Retrieve your {formatted_token_string} from: {url}")
+        for token in tokens:
+            yield getpass(f"Enter your {token} (masked): ", echo_char="*").strip()
+
+        rich_print()
+
     @override
-    def model_post_init(self, _: object) -> None:
+    def model_post_init(self, context: object) -> None:
+        super().model_post_init(context)
+
         # Check if any tokens are missing or if the user wants to reset them, then set tokens if necessary.
         if not self.openai_api_key:
             (self.openai_api_key,) = self.online_token_prompt("https://platform.openai.com/api-keys", "API key")
@@ -120,38 +136,18 @@ class Tokens(FileSyncSettings):
             # This is the whole OAuth 1.0 process.
             # https://requests-oauthlib.readthedocs.io/en/latest/examples/tumblr.html
             # We tried setting up OAuth 2.0, but the token refresh process is far too unreliable for this sort of program.
-            with OAuth1Session(
-                self.tumblr.client_key,
-                self.tumblr.client_secret,
-            ) as oauth_session:
-                fetch_response = oauth_session.fetch_request_token("http://tumblr.com/oauth/request_token")
-                full_authorize_url = oauth_session.authorization_url("http://tumblr.com/oauth/authorize")
-                (redirect_response,) = self.online_token_prompt(full_authorize_url, "full redirect URL")
-                oauth_response = oauth_session.parse_authorization_response(redirect_response)
-
-            with OAuth1Session(
-                self.tumblr.client_key,
-                self.tumblr.client_secret,
-                *self.get_oauth_tokens(fetch_response),
-                verifier=oauth_response["oauth_verifier"],
-            ) as oauth_session:
-                oauth_tokens = oauth_session.fetch_access_token("http://tumblr.com/oauth/access_token")
-
-            self.tumblr.resource_owner_key, self.tumblr.resource_owner_secret = self.get_oauth_tokens(oauth_tokens)
+            with OAuth1Session(**self.tumblr.model_dump()) as session:
+                session.fetch_request_token("http://tumblr.com/oauth/request_token")  # pyright: ignore[reportUnknownMemberType]
 
-    @staticmethod
-    def online_token_prompt(url: str, *tokens: str) -> Generator[str]:
-        formatted_token_string = " and ".join(f"[cyan]{token}[/]" for token in tokens)
+                rich_print("Open the link below in your browser, and authorize this application.\nAfter authorizing, copy and paste the URL of the page you are redirected to below.")
+                authorization_url = session.authorization_url("http://tumblr.com/oauth/authorize")  # pyright: ignore[reportUnknownMemberType]
+                (authorization_response,) = self.online_token_prompt(authorization_url, "full redirect URL")
+                session.parse_authorization_response(authorization_response)
 
-        rich.print(f"Retrieve your {formatted_token_string} from: {url}")
-        for token in tokens:
-            yield getpass(f"Enter your {token} (masked): ", echo_char="*").strip()
+                access_token = session.fetch_access_token("http://tumblr.com/oauth/access_token")  # pyright: ignore[reportUnknownMemberType]
 
-        rich.print()
-
-    @staticmethod
-    def get_oauth_tokens(token: dict[str, str]) -> tuple[str, str]:
-        return token["oauth_token"], token["oauth_token_secret"]
+            self.tumblr.resource_owner_key = access_token["oauth_token"]
+            self.tumblr.resource_owner_secret = access_token["oauth_token_secret"]
 
 
 class Blog(FullyValidatedModel):
@@ -168,25 +164,26 @@ class ResponseModel(FullyValidatedModel):
     response: Response
 
 
+class Block(FullyValidatedModel):
+    type: str = "text"
+    text: str = ""
+    blocks: list[int] = []
+
+
 class Post(FullyValidatedModel):
-    class Block(FullyValidatedModel):
-        type: str = ""
-        text: str = ""
-        blocks: list[int] = []
-
-    blog: SkipJsonSchema[Blog] = Blog()
-    id: SkipJsonSchema[int] = 0
-    parent_tumblelog_uuid: SkipJsonSchema[str] = ""
-    parent_post_id: SkipJsonSchema[int] = 0
-    reblog_key: SkipJsonSchema[str] = ""
-
-    timestamp: SkipJsonSchema[int] = 0
+    blog: Blog = Blog()
+    id: int = 0
+    parent_tumblelog_uuid: str = ""
+    parent_post_id: int = 0
+    reblog_key: str = ""
+
+    timestamp: int = 0
     tags: Annotated[list[str], PlainSerializer(",".join)] = []
-    state: SkipJsonSchema[Literal["published", "queued", "draft", "private", "unapproved"]] = "draft"
+    state: Literal["published", "queued", "draft", "private", "unapproved"] = "draft"
 
-    content: SkipJsonSchema[list[Block]] = []
-    layout: SkipJsonSchema[list[Block]] = []
-    trail: SkipJsonSchema[list[Self]] = []
+    content: list[Block] = []
+    layout: list[Block] = []
+    trail: list[Self] = []
 
     is_submission: SkipJsonSchema[bool] = False
 
@@ -212,9 +209,17 @@ class Post(FullyValidatedModel):
         return bool(self.content) and all(block.type == "text" for block in self.content) and not (self.is_submission or any(block.type == "ask" for block in self.layout))
 
 
-class Example(FullyValidatedModel):
-    class Message(FullyValidatedModel):
-        role: Literal["developer", "user", "assistant"]
-        content: str
+class Message(FullyValidatedModel):
+    role: Literal["developer", "user", "assistant"]
+    content: str
 
+
+class Example(FullyValidatedModel):
     messages: list[Message]
+
+    def get_assistant_message(self) -> str:
+        for message in self.messages:
+            if message.role == "assistant":
+                return message.content
+        msg = "Assistant message not found!"
+        raise ValueError(msg)

tumblrbot/utils/tumblr.py

@@ -1,31 +1,46 @@
-from typing import Self
-
-from requests import HTTPError, Response
-from requests_oauthlib import OAuth1Session
+from requests import HTTPError, Response, Session
+from requests_oauthlib import OAuth1
+from rich import print as rich_print
+from tenacity import retry, retry_if_exception_message, stop_after_attempt, wait_random_exponential
 
 from tumblrbot.utils.models import Post, ResponseModel, Tokens
 
+rate_limit_retry = retry(
+    stop=stop_after_attempt(10),
+    wait=wait_random_exponential(min=60),
+    retry=retry_if_exception_message(match="429 Client Error: Limit Exceeded for url: .+"),
+    before_sleep=lambda state: rich_print(f"[yellow]Tumblr rate limit exceeded. Waiting for {state.idle_for} seconds..."),
+    reraise=True,
+)
+
 
-class TumblrSession(OAuth1Session):
+class TumblrSession(Session):
     def __init__(self, tokens: Tokens) -> None:
-        super().__init__(**tokens.tumblr.model_dump())
+        super().__init__()
+        self.auth = OAuth1(**tokens.tumblr.model_dump())
         self.hooks["response"].append(self.response_hook)
 
-    def __enter__(self) -> Self:
-        super().__enter__()
-        return self
+        self.api_key = tokens.tumblr.client_key
 
     def response_hook(self, response: Response, *_args: object, **_kwargs: object) -> None:
         try:
             response.raise_for_status()
         except HTTPError as error:
-            error.add_note(response.text)
+            for error_msg in response.json()["errors"]:
+                error.add_note(f"{error_msg['code']}: {error_msg['detail']}")
             raise
 
+    @rate_limit_retry
     def retrieve_blog_info(self, blog_identifier: str) -> ResponseModel:
-        response = self.get(f"https://api.tumblr.com/v2/blog/{blog_identifier}/info")
+        response = self.get(
+            f"https://api.tumblr.com/v2/blog/{blog_identifier}/info",
+            params={
+                "api_key": self.api_key,
+            },
+        )
         return ResponseModel.model_validate_json(response.text)
 
+    @rate_limit_retry
     def retrieve_published_posts(
         self,
         blog_identifier: str,
@@ -35,6 +50,7 @@ class TumblrSession(OAuth1Session):
         response = self.get(
             f"https://api.tumblr.com/v2/blog/{blog_identifier}/posts",
             params={
+                "api_key": self.api_key,
                 "offset": offset,
                 "after": after,
                 "sort": "asc",
@@ -43,6 +59,7 @@ class TumblrSession(OAuth1Session):
         )
         return ResponseModel.model_validate_json(response.text)
 
+    @rate_limit_retry
     def create_post(self, blog_identifier: str, post: Post) -> ResponseModel:
         response = self.post(
             f"https://api.tumblr.com/v2/blog/{blog_identifier}/posts",

{tumblrbot-1.9.5.dist-info → tumblrbot-1.9.7.dist-info}/METADATA

@@ -1,19 +1,21 @@
 Metadata-Version: 2.4
 Name: tumblrbot
-Version: 1.9.5
+Version: 1.9.7
 Summary: An updated bot that posts to Tumblr, based on your very own blog!
 Requires-Python: >= 3.14
 Description-Content-Type: text/markdown
-Requires-Dist: click
 Requires-Dist: openai
 Requires-Dist: pydantic
 Requires-Dist: requests
 Requires-Dist: requests-oauthlib
 Requires-Dist: rich
+Requires-Dist: tenacity
 Requires-Dist: tiktoken
 Requires-Dist: tomlkit
+Requires-Dist: pyinstaller ; extra == "dev"
 Project-URL: Funding, https://ko-fi.com/maidscientistizutsumimarin
 Project-URL: Source, https://github.com/MaidScientistIzutsumiMarin/tumblrbot
+Provides-Extra: dev
 
 # tumblrbot
 
@@ -38,7 +40,6 @@ Project-URL: Source, https://github.com/MaidScientistIzutsumiMarin/tumblrbot
 [Tumblr]: https://tumblr.com
 [Tumblr Tokens]: https://tumblr.com/oauth/apps
 [Tumblr API Documentation on Blog Identifiers]: https://tumblr.com/docs/en/api/v2#blog-identifiers
-[Tumblr API Documentation on Rate Limits]: https://tumblr.com/docs/en/api/v2#rate-limits
 
 [Format String]: https://docs.python.org/3/library/string.html#format-string-syntax
 
@@ -84,10 +85,6 @@ Features:
 - Colorful output, progress bars, and post previews using [rich].
 - Automatically keeps the [config][configurable] file up-to-date and recreates it if missing (without overriding user settings).
 
-**To-Do:**
-
-- Add retry logic for rate limiting.
-
 **Known Issues:**
 
 - Fine-tuning can fail after the validation phase due to the examples file not passing [OpenAI] moderation checks. There are a few workarounds for this that can be tried in combination:
@@ -96,31 +93,39 @@ Features:
   - You can use regular expressions to filter out training data in the [config][configurable]. This is more of a brute-force solution, but it can work if the other solutions do not.
   - You can try limiting your dataset by specifying fewer blogs to download from or limiting the number of posts taken from each one in the [config][configurable].
   - If all else fails, you can manually remove data from the examples file until it passes. It is unfortunately not a definitive resource, but it can help to read about what the [OpenAI moderation API flags][Flags].
-- Sometimes, you will get an error about the training file not being found when starting fine-tuning. We do not currently have a fix or workaround for this. You should instead use the online portal for fine-tuning if this continues to happen. Read more in [fine-tuning].
-- Post counts are incorrect when downloading posts. We are not certain what the cause of this is, but our tests suggest this is a [Tumblr] API problem that is giving inaccurate numbers.
-- During post downloading or post generation, you may receive a “Limit Exceeded” error message from the [Tumblr] API. This is caused by server-side rate-limiting by [Tumblr]. The only workaround is trying again or waiting for a period of time before retrying. In most cases, you either have to wait for a minute or an hour for the limits to reset. You can read more about the limits in the [Tumblr API documentation on rate limits].
-- Similar to the above issue, you may sometimes get a message saying your IP is blocked. This block is temporary and probably follows the same rules as previously described.
+- Sometimes, you will get an error about the training file not being found when starting fine-tuning. We do not currently have a fix or workaround for this. You should instead use the online portal for fine-tuning if this continues to happen. Read more in [fine-tuning]
+  - *We are unsure if this is still happening.*
+- Post counts are incorrect when downloading posts. Our tests suggest this is a [Tumblr] API problem that is giving inaccurate numbers, so treat them as estimates.
 
 **Please submit an issue or contact us for features you want added/reimplemented.**
 
-## Installation
+## Installation & Usage
+
+### Downloadable Binary
+
+| Pros | Cons |
+| --- | --- |
+| Easier to install | Harder to update |
+| No risk of dependencies breaking | Dependencies may be older |
+
+1. Download the latest release's [tumblrbot.exe].
+1. Launch `tumblrbot.exe` in the install location.
+
+### PyPi
+
+| Pros | Cons |
+| --- | --- |
+| Easier to update | Harder to install |
+| Dependencies may be newer | Dependencies may break |
 
 1. Install the latest version of [Python]:
    - Windows: `winget install python3`
    - Linux (apt): `apt install python-pip`
    - Linux (pacman): `pacman install python-pip`
 1. Install the [pip] package: `pip install tumblrbot`
-   - Alternatively, you can install from this repository: `pip install git+https://github.com/MaidThatPrograms/tumblrbot.git`
+   - Alternatively, you can install from this repository: `pip install git+https://github.com/MaidScientistIzutsumiMarin/tumblrbot.git`
    - On Linux, you will have to make a virtual environment or use the flag to install packages system-wide.
-
-### Alternative Installation for Windows
-
-1. Download the latest release's [tumblrbot.exe].
-1. Run the file directly, or add it to your path, and use it as normal.
-
-## Usage
-
-Run `tumblrbot` from anywhere. Run `tumblrbot --help` for command-line options. Every command-line option corresponds to a value from the [config][configurable].
+1. Run `tumblrbot` from anywhere. Run `tumblrbot --help` for command-line options. Every command-line option corresponds to a value from the [config][configurable].
 
 ## Obtaining Tokens
 
@@ -177,6 +182,7 @@ Specific Options:
    To be specific, it should follow the [JSON Lines] file format with one collection of name/value pairs (a dictionary) per line. You can validate your file using the [JSON Lines Validator].
 
 - **`post_limit`** - At most, this many valid posts will be included in the training data. This effectively is a filter to select the `N` most recent valid posts from each blog. `0` will use every available valid post.
+- **`moderation_batch_size`** - This controls the batch size when submitting posts to the OpenAI moderation. There is no limit, but higher numbers will cause you to be rate-limited more, which can overall be slower. Low numbers reduce rate-limiting, but can sometimes take longer due to needing more requests. The best value will depend on your computer, internet connection, and any number of factors on OpenAI's side. The default value is just what worked best for our computer.
 - **`filtered_words`** - During training data generation, any posts with the specified words will be removed. Word boundaries are not checked by default, so “the” will also filter out posts with “them” or “thematic”. This setting supports regular expressions, so you can explicitly look for word boundaries by surrounding an entry with “\\\b”, i.e., “\\\bthe\\\b”. Regular expressions have to be escaped like so due to how JSON data is read in. If you are familiar with regular expressions, it could be useful for you to know that every entry is joined with a “|” which is then used to search the post content for any matches.
 - **`developer_message`** - This message is used in for fine-tuning the AI as well as generating prompts. If you change this, you will need to run the fine-tuning again with the new value before generating posts.
 - **`user_message`** - This setting is used and works in the same way as `developer_message`.

tumblrbot-1.9.7.dist-info/RECORD

@@ -0,0 +1,15 @@
+tumblrbot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tumblrbot/__main__.py,sha256=DYHfOs5oyOIT4c0cdEd4o0p7UPo39EnTMJHyOrsVWBg,1753
+tumblrbot/flow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tumblrbot/flow/download.py,sha256=r_4Bc_SNnIDbilT18YypBjJ4xhrg0AJAfvOezUL-p2c,2052
+tumblrbot/flow/examples.py,sha256=30FVJQomjwpUBKeeAJDfdkr8ZcQlykoXZ6uL3tqxgOg,4465
+tumblrbot/flow/fine_tune.py,sha256=KaC37_RNsbfAh1_ZdsiAELvRbYkJrrH8QPJPGuZtzOM,5483
+tumblrbot/flow/generate.py,sha256=WQEG_85_nuRp91Le6lvZ0FeIWnBA1a012MlI3_YMq-8,4302
+tumblrbot/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tumblrbot/utils/common.py,sha256=4n3kcdgTP_HHjCg5Z7GjNyBZzBLUGrTu4fwxaOFYGt4,1676
+tumblrbot/utils/models.py,sha256=s5Hm1AEEtnz4mi_apUlu2x4NuXeZtMAOH8DHSYgbv1E,11050
+tumblrbot/utils/tumblr.py,sha256=HfaoRr0YikP0uJdnwCxdhyK5wUBOuvaC9unNWa1Erk8,2526
+tumblrbot-1.9.7.dist-info/entry_points.txt,sha256=lTiN7PxAbyGY1fpCWApEw6NUIUgobfcOKhvn6cu3IQA,53
+tumblrbot-1.9.7.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+tumblrbot-1.9.7.dist-info/METADATA,sha256=s21DxgfmBLtlK03NSZktVB0UZWc6MUCZ8YHpFGM-MkU,14195
+tumblrbot-1.9.7.dist-info/RECORD,,

tumblrbot-1.9.5.dist-info/RECORD

@@ -1,15 +0,0 @@
-tumblrbot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tumblrbot/__main__.py,sha256=XWSbOmI_y2MJVU9xpkgA-0zaF3HNwR5uF6_BZqtCQWY,1719
-tumblrbot/flow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tumblrbot/flow/download.py,sha256=r_4Bc_SNnIDbilT18YypBjJ4xhrg0AJAfvOezUL-p2c,2052
-tumblrbot/flow/examples.py,sha256=fSgAat4VpTw_ALt_Ytp9TBzl6-RFeMQaK_jcMXF0rcQ,4124
-tumblrbot/flow/fine_tune.py,sha256=LHpaoqHc9JnXNQzbJIjYO36hwy2l8A_e4IuLx7ke2MQ,5436
-tumblrbot/flow/generate.py,sha256=-Q5ZSbfRGk3jQdE_73DjlI-iICIUxbJtIP463eChsHg,4337
-tumblrbot/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tumblrbot/utils/common.py,sha256=RvICPORtBSqsN7VWADgStogJ8w4owzBfR1E2XbCQrfA,1795
-tumblrbot/utils/models.py,sha256=iBCY1NCnUvEgfsSR7Tes6ht7UYY6UIvnWFnZEtkB69I,11018
-tumblrbot/utils/tumblr.py,sha256=vpLRX5cdhqmCaJdQbiGMOo9uOVhiC2CQF67BJ5u4fwU,1769
-tumblrbot-1.9.5.dist-info/entry_points.txt,sha256=lTiN7PxAbyGY1fpCWApEw6NUIUgobfcOKhvn6cu3IQA,53
-tumblrbot-1.9.5.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-tumblrbot-1.9.5.dist-info/METADATA,sha256=OB1Ke61MPRdtsGxbd2Zi6pnwxqpSI61-Lai4vrVssaE,14126
-tumblrbot-1.9.5.dist-info/RECORD,,