tumblrbot 1.9.4__py3-none-any.whl → 1.9.5__py3-none-any.whl

tumblrbot/flow/download.py

@@ -1,10 +1,12 @@
-from io import TextIOBase
 from json import dump
-from typing import override
+from typing import TYPE_CHECKING, override
 
 from tumblrbot.utils.common import FlowClass, PreviewLive
 from tumblrbot.utils.models import Post
 
+if TYPE_CHECKING:
+    from io import TextIOBase
+
 
 class PostDownloader(FlowClass):
     @override

tumblrbot/flow/examples.py

@@ -1,11 +1,9 @@
 import re
-from collections.abc import Generator
 from itertools import batched
 from json import loads
 from math import ceil
-from pathlib import Path
 from re import search
-from typing import IO, override
+from typing import IO, TYPE_CHECKING, override
 
 import rich
 from openai import BadRequestError
@@ -13,6 +11,10 @@ from openai import BadRequestError
 from tumblrbot.utils.common import FlowClass, PreviewLive
 from tumblrbot.utils.models import Example, Post
 
+if TYPE_CHECKING:
+    from collections.abc import Generator
+    from pathlib import Path
+
 
 class ExamplesWriter(FlowClass):
     @override

tumblrbot/flow/fine_tune.py

@@ -1,11 +1,9 @@
-from collections.abc import Generator
 from datetime import datetime
 from textwrap import dedent
 from time import sleep
-from typing import override
+from typing import TYPE_CHECKING, override
 
 import rich
-from openai.types.fine_tuning import FineTuningJob
 from rich import progress
 from rich.console import Console
 from rich.prompt import Confirm
@@ -14,6 +12,11 @@ from tiktoken import encoding_for_model, get_encoding
 from tumblrbot.utils.common import FlowClass, PreviewLive
 from tumblrbot.utils.models import Example
 
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from openai.types.fine_tuning import FineTuningJob
+
 
 class FineTuner(FlowClass):
     @staticmethod
@@ -131,4 +134,4 @@ class FineTuner(FlowClass):
                     yield 4 + len(encoding.encode(message.content))
 
     def get_cost_string(self, total_tokens: int) -> str:
-        return f"${self.config.token_price / 1000000 * total_tokens:.2f}"
+        return f"{self.config.token_price / 1000000 * total_tokens:.2f} USD"

tumblrbot/flow/generate.py

@@ -1,7 +1,6 @@
-from collections.abc import Iterable
 from functools import cache
 from random import choice, random, sample
-from typing import override
+from typing import TYPE_CHECKING, override
 
 import rich
 from pydantic import ConfigDict
@@ -10,6 +9,9 @@ from rich.prompt import IntPrompt
 from tumblrbot.utils.common import FlowClass, PreviewLive
 from tumblrbot.utils.models import Post
 
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
 
 class DraftGenerator(FlowClass):
     model_config = ConfigDict(frozen=True)  # Makes this class hashable.

tumblrbot/utils/models.py

@@ -1,17 +1,17 @@
 from collections.abc import Generator
+from getpass import getpass
 from pathlib import Path
-from tomllib import loads
 from typing import Annotated, Any, Literal, Self, override
 
 import rich
 from openai.types import ChatModel
-from pwinput import pwinput
 from pydantic import BaseModel, ConfigDict, Field, NonNegativeFloat, NonNegativeInt, PlainSerializer, PositiveFloat, PositiveInt, model_validator
 from pydantic.json_schema import SkipJsonSchema
 from requests_oauthlib import OAuth1Session
 from rich.panel import Panel
 from rich.prompt import Prompt
 from tomlkit import comment, document, dumps  # pyright: ignore[reportUnknownVariableType]
+from tomllib import loads
 
 
 class FullyValidatedModel(BaseModel):
@@ -145,7 +145,7 @@ class Tokens(FileSyncSettings):
 
         rich.print(f"Retrieve your {formatted_token_string} from: {url}")
         for token in tokens:
-            yield pwinput(f"Enter your {token} (masked): ").strip()
+            yield getpass(f"Enter your {token} (masked): ", echo_char="*").strip()
 
         rich.print()
 

{tumblrbot-1.9.4.dist-info → tumblrbot-1.9.5.dist-info}/METADATA

@@ -1,12 +1,11 @@
 Metadata-Version: 2.4
 Name: tumblrbot
-Version: 1.9.4
+Version: 1.9.5
 Summary: An updated bot that posts to Tumblr, based on your very own blog!
-Requires-Python: >= 3.13
+Requires-Python: >= 3.14
 Description-Content-Type: text/markdown
 Requires-Dist: click
 Requires-Dist: openai
-Requires-Dist: pwinput
 Requires-Dist: pydantic
 Requires-Dist: requests
 Requires-Dist: requests-oauthlib
@@ -18,6 +17,8 @@ Project-URL: Source, https://github.com/MaidScientistIzutsumiMarin/tumblrbot
 
 # tumblrbot
 
+[tumblrbot.exe]: https://github.com/MaidScientistIzutsumiMarin/tumblrbot/releases/latest/download/tumblrbot.exe
+
 [OAuth]: https://oauth.net/1
 [Python]: https://python.org/download
 
@@ -31,6 +32,7 @@ Project-URL: Source, https://github.com/MaidScientistIzutsumiMarin/tumblrbot
 [OpenAI Pricing]: https://platform.openai.com/docs/pricing#fine-tuning
 [OpenAI Tokens]: https://platform.openai.com/settings/organization/api-keys
 [OpenAI Moderation API]: https://platform.openai.com/docs/guides/moderation
+[Flags]: https://platform.openai.com/docs/guides/moderation/over#content-classifications
 [Fine-Tuning Portal]: https://platform.openai.com/finetune
 
 [Tumblr]: https://tumblr.com
@@ -82,11 +84,21 @@ Features:
 - Colorful output, progress bars, and post previews using [rich].
 - Automatically keeps the [config][configurable] file up-to-date and recreates it if missing (without overriding user settings).
 
+**To-Do:**
+
+- Add retry logic for rate limiting.
+
 **Known Issues:**
 
+- Fine-tuning can fail after the validation phase due to the examples file not passing [OpenAI] moderation checks. There are a few workarounds for this that can be tried in combination:
+  - You can retry with the same examples file. This has, on rare occasions, worked.
+  - You can submit the examples file to the [OpenAI] moderation API with this program's guided prompts. This has worked consistently for our dataset, but others have reported it not being thorough enough.
+  - You can use regular expressions to filter out training data in the [config][configurable]. This is more of a brute-force solution, but it can work if the other solutions do not.
+  - You can try limiting your dataset by specifying fewer blogs to download from or limiting the number of posts taken from each one in the [config][configurable].
+  - If all else fails, you can manually remove data from the examples file until it passes. It is unfortunately not a definitive resource, but it can help to read about what the [OpenAI moderation API flags][Flags].
 - Sometimes, you will get an error about the training file not being found when starting fine-tuning. We do not currently have a fix or workaround for this. You should instead use the online portal for fine-tuning if this continues to happen. Read more in [fine-tuning].
 - Post counts are incorrect when downloading posts. We are not certain what the cause of this is, but our tests suggest this is a [Tumblr] API problem that is giving inaccurate numbers.
-- During post downloading or post generation, you may receive a "Limit Exceeded" error message from the [Tumblr] API. This is caused by server-side rate-limiting by [Tumblr]. The only workaround is trying again or waiting for a period of time before retrying. In most cases, you either have to wait for a minute or an hour for the limits to reset. You can read more about the limits in the [Tumblr API documentation on rate limits].
+- During post downloading or post generation, you may receive a “Limit Exceeded” error message from the [Tumblr] API. This is caused by server-side rate-limiting by [Tumblr]. The only workaround is trying again or waiting for a period of time before retrying. In most cases, you either have to wait for a minute or an hour for the limits to reset. You can read more about the limits in the [Tumblr API documentation on rate limits].
 - Similar to the above issue, you may sometimes get a message saying your IP is blocked. This block is temporary and probably follows the same rules as previously described.
 
 **Please submit an issue or contact us for features you want added/reimplemented.**
@@ -101,6 +113,11 @@ Features:
    - Alternatively, you can install from this repository: `pip install git+https://github.com/MaidThatPrograms/tumblrbot.git`
    - On Linux, you will have to make a virtual environment or use the flag to install packages system-wide.
 
+### Alternative Installation for Windows
+
+1. Download the latest release's [tumblrbot.exe].
+1. Run the file directly, or add it to your path, and use it as normal.
+
 ## Usage
 
 Run `tumblrbot` from anywhere. Run `tumblrbot --help` for command-line options. Every command-line option corresponds to a value from the [config][configurable].
@@ -160,7 +177,7 @@ Specific Options:
    To be specific, it should follow the [JSON Lines] file format with one collection of name/value pairs (a dictionary) per line. You can validate your file using the [JSON Lines Validator].
 
 - **`post_limit`** - At most, this many valid posts will be included in the training data. This effectively is a filter to select the `N` most recent valid posts from each blog. `0` will use every available valid post.
-- **`filtered_words`** - During training data generation, any posts with the specified words will be removed. Word boundaries are not checked by default, so "the" will also filter out posts with "them" or "thematic". This setting supports regular expressions, so you can explicitly look for word boundaries by surrounding an entry with "\\\b", i.e. "\\\bthe\\\b". Regular expressions have to be escaped like so due to how JSON data is read in. If you are familiar with regular expressions, it could be useful for you to know that every entry is joined with a "|" which is then used to search the post content for any matches.
+- **`filtered_words`** - During training data generation, any posts with the specified words will be removed. Word boundaries are not checked by default, so “the” will also filter out posts with “them” or “thematic”. This setting supports regular expressions, so you can explicitly look for word boundaries by surrounding an entry with “\\\b”, i.e., “\\\bthe\\\b”. Regular expressions have to be escaped like so due to how JSON data is read in. If you are familiar with regular expressions, it could be useful for you to know that every entry is joined with a “|” which is then used to search the post content for any matches.
 - **`developer_message`** - This message is used in for fine-tuning the AI as well as generating prompts. If you change this, you will need to run the fine-tuning again with the new value before generating posts.
 - **`user_message`** - This setting is used and works in the same way as `developer_message`.
 - **`expected_epochs`** - The default value here is the default number of epochs for `base_model`. You may have to change this value if you change `base_model`. After running fine-tuning once, you will see the number of epochs used in the [fine-tuning portal] under *Hyperparameters*. This value will also be updated automatically if you run fine-tuning through this program.

tumblrbot-1.9.5.dist-info/RECORD

@@ -0,0 +1,15 @@
+tumblrbot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tumblrbot/__main__.py,sha256=XWSbOmI_y2MJVU9xpkgA-0zaF3HNwR5uF6_BZqtCQWY,1719
+tumblrbot/flow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tumblrbot/flow/download.py,sha256=r_4Bc_SNnIDbilT18YypBjJ4xhrg0AJAfvOezUL-p2c,2052
+tumblrbot/flow/examples.py,sha256=fSgAat4VpTw_ALt_Ytp9TBzl6-RFeMQaK_jcMXF0rcQ,4124
+tumblrbot/flow/fine_tune.py,sha256=LHpaoqHc9JnXNQzbJIjYO36hwy2l8A_e4IuLx7ke2MQ,5436
+tumblrbot/flow/generate.py,sha256=-Q5ZSbfRGk3jQdE_73DjlI-iICIUxbJtIP463eChsHg,4337
+tumblrbot/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tumblrbot/utils/common.py,sha256=RvICPORtBSqsN7VWADgStogJ8w4owzBfR1E2XbCQrfA,1795
+tumblrbot/utils/models.py,sha256=iBCY1NCnUvEgfsSR7Tes6ht7UYY6UIvnWFnZEtkB69I,11018
+tumblrbot/utils/tumblr.py,sha256=vpLRX5cdhqmCaJdQbiGMOo9uOVhiC2CQF67BJ5u4fwU,1769
+tumblrbot-1.9.5.dist-info/entry_points.txt,sha256=lTiN7PxAbyGY1fpCWApEw6NUIUgobfcOKhvn6cu3IQA,53
+tumblrbot-1.9.5.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+tumblrbot-1.9.5.dist-info/METADATA,sha256=OB1Ke61MPRdtsGxbd2Zi6pnwxqpSI61-Lai4vrVssaE,14126
+tumblrbot-1.9.5.dist-info/RECORD,,

tumblrbot-1.9.4.dist-info/RECORD

@@ -1,15 +0,0 @@
-tumblrbot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tumblrbot/__main__.py,sha256=XWSbOmI_y2MJVU9xpkgA-0zaF3HNwR5uF6_BZqtCQWY,1719
-tumblrbot/flow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tumblrbot/flow/download.py,sha256=PUs7eM-1kGOb2RKijy3lW0zyvfFDwbxzTGhVghrWIhc,2012
-tumblrbot/flow/examples.py,sha256=MLj51kEZ3AWvP47V4mR9UKUUciKoNQJEvXhr5jxSgfs,4080
-tumblrbot/flow/fine_tune.py,sha256=YDukEwZNw3GveEAH4ORv6oylka5MQNLK_4iSmuAVPtg,5387
-tumblrbot/flow/generate.py,sha256=b5Yfo_k9LoQWvHEJEAWWARshpxqY1rsInFrAuqJ8Qbw,4297
-tumblrbot/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tumblrbot/utils/common.py,sha256=RvICPORtBSqsN7VWADgStogJ8w4owzBfR1E2XbCQrfA,1795
-tumblrbot/utils/models.py,sha256=WdLqX-og8h4vZurtuyZ05ph4sssP16eny4GXvFkkijM,11003
-tumblrbot/utils/tumblr.py,sha256=vpLRX5cdhqmCaJdQbiGMOo9uOVhiC2CQF67BJ5u4fwU,1769
-tumblrbot-1.9.4.dist-info/entry_points.txt,sha256=lTiN7PxAbyGY1fpCWApEw6NUIUgobfcOKhvn6cu3IQA,53
-tumblrbot-1.9.4.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-tumblrbot-1.9.4.dist-info/METADATA,sha256=MBrQIyt48nWGQL4tAgV7a9hPr_sRw3DiGptTGkX-DHM,12666
-tumblrbot-1.9.4.dist-info/RECORD,,