tumblrbot 1.1.0__tar.gz → 1.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tumblrbot-1.1.2/.github/dependabot.yml +11 -0
- tumblrbot-1.1.2/.gitignore +215 -0
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/PKG-INFO +11 -17
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/README.md +8 -12
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/UNLICENSE +24 -24
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/pyproject.toml +4 -7
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/__main__.py +1 -7
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/flow/download.py +12 -17
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/flow/fine_tune.py +2 -2
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/flow/generate.py +5 -2
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/utils/models.py +5 -6
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/utils/settings.py +1 -1
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/utils/tumblr.py +13 -12
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/__init__.py +0 -0
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/flow/__init__.py +0 -0
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/flow/examples.py +0 -0
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/utils/__init__.py +0 -0
- {tumblrbot-1.1.0 → tumblrbot-1.1.2}/src/tumblrbot/utils/common.py +0 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# To get started with Dependabot version updates, you'll need to specify which
|
|
2
|
+
# package ecosystems to update and where the package manifests are located.
|
|
3
|
+
# Please see the documentation for all configuration options:
|
|
4
|
+
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
|
5
|
+
|
|
6
|
+
version: 2
|
|
7
|
+
updates:
|
|
8
|
+
- package-ecosystem: "pip" # See documentation for possible values
|
|
9
|
+
directory: "/" # Location of package manifests
|
|
10
|
+
schedule:
|
|
11
|
+
interval: "weekly"
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
#Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
#uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
#poetry.lock
|
|
109
|
+
#poetry.toml
|
|
110
|
+
|
|
111
|
+
# pdm
|
|
112
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
113
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
114
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
115
|
+
#pdm.lock
|
|
116
|
+
#pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# pixi
|
|
121
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
122
|
+
#pixi.lock
|
|
123
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
124
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
125
|
+
.pixi
|
|
126
|
+
|
|
127
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
128
|
+
__pypackages__/
|
|
129
|
+
|
|
130
|
+
# Celery stuff
|
|
131
|
+
celerybeat-schedule
|
|
132
|
+
celerybeat.pid
|
|
133
|
+
|
|
134
|
+
# SageMath parsed files
|
|
135
|
+
*.sage.py
|
|
136
|
+
|
|
137
|
+
# Environments
|
|
138
|
+
.env
|
|
139
|
+
.envrc
|
|
140
|
+
.venv
|
|
141
|
+
env/
|
|
142
|
+
venv/
|
|
143
|
+
ENV/
|
|
144
|
+
env.bak/
|
|
145
|
+
venv.bak/
|
|
146
|
+
|
|
147
|
+
# Spyder project settings
|
|
148
|
+
.spyderproject
|
|
149
|
+
.spyproject
|
|
150
|
+
|
|
151
|
+
# Rope project settings
|
|
152
|
+
.ropeproject
|
|
153
|
+
|
|
154
|
+
# mkdocs documentation
|
|
155
|
+
/site
|
|
156
|
+
|
|
157
|
+
# mypy
|
|
158
|
+
.mypy_cache/
|
|
159
|
+
.dmypy.json
|
|
160
|
+
dmypy.json
|
|
161
|
+
|
|
162
|
+
# Pyre type checker
|
|
163
|
+
.pyre/
|
|
164
|
+
|
|
165
|
+
# pytype static type analyzer
|
|
166
|
+
.pytype/
|
|
167
|
+
|
|
168
|
+
# Cython debug symbols
|
|
169
|
+
cython_debug/
|
|
170
|
+
|
|
171
|
+
# PyCharm
|
|
172
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
173
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
174
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
175
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
176
|
+
#.idea/
|
|
177
|
+
|
|
178
|
+
# Abstra
|
|
179
|
+
# Abstra is an AI-powered process automation framework.
|
|
180
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
181
|
+
# Learn more at https://abstra.io/docs
|
|
182
|
+
.abstra/
|
|
183
|
+
|
|
184
|
+
# Visual Studio Code
|
|
185
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
186
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
187
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
188
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
189
|
+
# .vscode/
|
|
190
|
+
|
|
191
|
+
# Ruff stuff:
|
|
192
|
+
.ruff_cache/
|
|
193
|
+
|
|
194
|
+
# PyPI configuration file
|
|
195
|
+
.pypirc
|
|
196
|
+
|
|
197
|
+
# Cursor
|
|
198
|
+
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
|
|
199
|
+
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
|
|
200
|
+
# refer to https://docs.cursor.com/context/ignore-files
|
|
201
|
+
.cursorignore
|
|
202
|
+
.cursorindexingignore
|
|
203
|
+
|
|
204
|
+
# Marimo
|
|
205
|
+
marimo/_static/
|
|
206
|
+
marimo/_lsp/
|
|
207
|
+
__marimo__/
|
|
208
|
+
|
|
209
|
+
# Streamlit
|
|
210
|
+
.streamlit/secrets.toml
|
|
211
|
+
|
|
212
|
+
# Custom
|
|
213
|
+
data
|
|
214
|
+
*.toml
|
|
215
|
+
*.jsonl
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tumblrbot
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.2
|
|
4
4
|
Summary: An updated bot that posts to Tumblr, based on your very own blog!
|
|
5
5
|
Requires-Python: >= 3.13
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
|
-
License-Expression: Unlicense
|
|
8
|
-
License-File: UNLICENSE
|
|
9
7
|
Requires-Dist: more-itertools
|
|
10
8
|
Requires-Dist: openai
|
|
11
9
|
Requires-Dist: pydantic
|
|
@@ -15,8 +13,7 @@ Requires-Dist: requests-oauthlib
|
|
|
15
13
|
Requires-Dist: rich
|
|
16
14
|
Requires-Dist: tiktoken
|
|
17
15
|
Requires-Dist: tomlkit
|
|
18
|
-
Project-URL:
|
|
19
|
-
Project-URL: Repository, https://github.com/MaidThatPrograms/tumblrbot
|
|
16
|
+
Project-URL: Source, https://github.com/MaidThatPrograms/tumblrbot
|
|
20
17
|
|
|
21
18
|
[OpenAI]: https://pypi.org/project/openai
|
|
22
19
|
[Python]: https://python.org/download
|
|
@@ -32,15 +29,12 @@ Project-URL: Repository, https://github.com/MaidThatPrograms/tumblrbot
|
|
|
32
29
|
[Examples]: tumblrbot/flow/examples.py
|
|
33
30
|
[Fine-Tune]: tumblrbot/flow/fine_tune.py
|
|
34
31
|
[Generate]: tumblrbot/flow/generate.py
|
|
35
|
-
[Utils]: tumblrbot/utils/common.py
|
|
36
|
-
[Models]: tumblrbot/utils/models.py
|
|
37
32
|
[Settings]: tumblrbot/utils/settings.py
|
|
38
|
-
[Tumblr]: tumblrbot/utils/tumblr.py
|
|
39
33
|
[Main]: __main__.py
|
|
40
34
|
[README.md]: README.md
|
|
41
35
|
|
|
42
36
|
# tumblrbot
|
|
43
|
-

|
|
37
|
+
[](https://python.org/pypi/tumblrbot)
|
|
44
38
|
|
|
45
39
|
Description of original project:
|
|
46
40
|
> 4tv-tumblrbot was a collaborative project I embarked on with my close friend Dima, who goes by @smoqueen on Tumblr. The aim of this endeavor was straightforward yet silly: to develop a Tumblr bot powered by a machine-learning model. This bot would be specifically trained on the content from a particular Tumblr blog or a selected set of blogs, allowing it to mimic the style, tone, and thematic essence of the original posts.
|
|
@@ -89,6 +83,7 @@ To-Do:
|
|
|
89
83
|
- Add documentation.
|
|
90
84
|
- Finish updating [README.md].
|
|
91
85
|
- Look into places more-itertools can help.
|
|
86
|
+
- Change the differences list to instead just be a list of features.
|
|
92
87
|
|
|
93
88
|
|
|
94
89
|
**Please submit an issue or contact us for features you want to added/reimplemented.**
|
|
@@ -99,16 +94,15 @@ To-Do:
|
|
|
99
94
|
- Linux (apt): `apt install python-pip`
|
|
100
95
|
- Linux (pacman): `pacman install python-pip`
|
|
101
96
|
1. Install the [pip] package: `pip install tumblrbot`
|
|
102
|
-
- On Linux, you will have to make a virtual environment.
|
|
103
97
|
- Alternatively, you can install from this repository: `pip install git+https://github.com/MaidThatPrograms/tumblrbot.git`
|
|
98
|
+
- On Linux, you will have to make a virtual environment.
|
|
104
99
|
|
|
105
100
|
## Usage
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
101
|
+
Run `tumblrbot` from anywhere. Run `tumblrbot --help` for command-line options.
|
|
102
|
+
|
|
103
|
+
## Obtaining Tokens
|
|
104
|
+
> WIP
|
|
109
105
|
|
|
110
|
-
|
|
111
|
-
|
|
106
|
+
## Configuration
|
|
107
|
+
> WIP
|
|
112
108
|
|
|
113
|
-
## More Information
|
|
114
|
-
- WIP
|
|
@@ -12,15 +12,12 @@
|
|
|
12
12
|
[Examples]: tumblrbot/flow/examples.py
|
|
13
13
|
[Fine-Tune]: tumblrbot/flow/fine_tune.py
|
|
14
14
|
[Generate]: tumblrbot/flow/generate.py
|
|
15
|
-
[Utils]: tumblrbot/utils/common.py
|
|
16
|
-
[Models]: tumblrbot/utils/models.py
|
|
17
15
|
[Settings]: tumblrbot/utils/settings.py
|
|
18
|
-
[Tumblr]: tumblrbot/utils/tumblr.py
|
|
19
16
|
[Main]: __main__.py
|
|
20
17
|
[README.md]: README.md
|
|
21
18
|
|
|
22
19
|
# tumblrbot
|
|
23
|
-

|
|
20
|
+
[](https://python.org/pypi/tumblrbot)
|
|
24
21
|
|
|
25
22
|
Description of original project:
|
|
26
23
|
> 4tv-tumblrbot was a collaborative project I embarked on with my close friend Dima, who goes by @smoqueen on Tumblr. The aim of this endeavor was straightforward yet silly: to develop a Tumblr bot powered by a machine-learning model. This bot would be specifically trained on the content from a particular Tumblr blog or a selected set of blogs, allowing it to mimic the style, tone, and thematic essence of the original posts.
|
|
@@ -69,6 +66,7 @@ To-Do:
|
|
|
69
66
|
- Add documentation.
|
|
70
67
|
- Finish updating [README.md].
|
|
71
68
|
- Look into places more-itertools can help.
|
|
69
|
+
- Change the differences list to instead just be a list of features.
|
|
72
70
|
|
|
73
71
|
|
|
74
72
|
**Please submit an issue or contact us for features you want to added/reimplemented.**
|
|
@@ -79,16 +77,14 @@ To-Do:
|
|
|
79
77
|
- Linux (apt): `apt install python-pip`
|
|
80
78
|
- Linux (pacman): `pacman install python-pip`
|
|
81
79
|
1. Install the [pip] package: `pip install tumblrbot`
|
|
82
|
-
- On Linux, you will have to make a virtual environment.
|
|
83
80
|
- Alternatively, you can install from this repository: `pip install git+https://github.com/MaidThatPrograms/tumblrbot.git`
|
|
81
|
+
- On Linux, you will have to make a virtual environment.
|
|
84
82
|
|
|
85
83
|
## Usage
|
|
86
|
-
|
|
87
|
-
- Double-click `__main__.py` in the root directory.
|
|
88
|
-
- Run `python __main__.py` in the root directory.
|
|
84
|
+
Run `tumblrbot` from anywhere. Run `tumblrbot --help` for command-line options.
|
|
89
85
|
|
|
90
|
-
|
|
91
|
-
|
|
86
|
+
## Obtaining Tokens
|
|
87
|
+
> WIP
|
|
92
88
|
|
|
93
|
-
##
|
|
94
|
-
|
|
89
|
+
## Configuration
|
|
90
|
+
> WIP
|
|
@@ -1,24 +1,24 @@
|
|
|
1
|
-
This is free and unencumbered software released into the public domain.
|
|
2
|
-
|
|
3
|
-
Anyone is free to copy, modify, publish, use, compile, sell, or
|
|
4
|
-
distribute this software, either in source code form or as a compiled
|
|
5
|
-
binary, for any purpose, commercial or non-commercial, and by any
|
|
6
|
-
means.
|
|
7
|
-
|
|
8
|
-
In jurisdictions that recognize copyright laws, the author or authors
|
|
9
|
-
of this software dedicate any and all copyright interest in the
|
|
10
|
-
software to the public domain. We make this dedication for the benefit
|
|
11
|
-
of the public at large and to the detriment of our heirs and
|
|
12
|
-
successors. We intend this dedication to be an overt act of
|
|
13
|
-
relinquishment in perpetuity of all present and future rights to this
|
|
14
|
-
software under copyright law.
|
|
15
|
-
|
|
16
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
17
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
18
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
19
|
-
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
|
20
|
-
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
21
|
-
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
22
|
-
OTHER DEALINGS IN THE SOFTWARE.
|
|
23
|
-
|
|
24
|
-
For more information, please refer to <https://unlicense.org
|
|
1
|
+
This is free and unencumbered software released into the public domain.
|
|
2
|
+
|
|
3
|
+
Anyone is free to copy, modify, publish, use, compile, sell, or
|
|
4
|
+
distribute this software, either in source code form or as a compiled
|
|
5
|
+
binary, for any purpose, commercial or non-commercial, and by any
|
|
6
|
+
means.
|
|
7
|
+
|
|
8
|
+
In jurisdictions that recognize copyright laws, the author or authors
|
|
9
|
+
of this software dedicate any and all copyright interest in the
|
|
10
|
+
software to the public domain. We make this dedication for the benefit
|
|
11
|
+
of the public at large and to the detriment of our heirs and
|
|
12
|
+
successors. We intend this dedication to be an overt act of
|
|
13
|
+
relinquishment in perpetuity of all present and future rights to this
|
|
14
|
+
software under copyright law.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
19
|
+
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
|
20
|
+
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
21
|
+
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
22
|
+
OTHER DEALINGS IN THE SOFTWARE.
|
|
23
|
+
|
|
24
|
+
For more information, please refer to <https://unlicense.org>
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tumblrbot"
|
|
3
|
-
version = "1.1.
|
|
3
|
+
version = "1.1.2"
|
|
4
4
|
description = "An updated bot that posts to Tumblr, based on your very own blog!"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">= 3.13"
|
|
7
|
-
license = "Unlicense"
|
|
8
|
-
license-files = ["UNLICENSE"]
|
|
9
7
|
dependencies = [
|
|
10
8
|
"more-itertools",
|
|
11
9
|
"openai",
|
|
@@ -19,12 +17,11 @@ dependencies = [
|
|
|
19
17
|
]
|
|
20
18
|
|
|
21
19
|
[project.urls]
|
|
22
|
-
|
|
23
|
-
Repository = "https://github.com/MaidThatPrograms/tumblrbot"
|
|
20
|
+
Source = "https://github.com/MaidThatPrograms/tumblrbot"
|
|
24
21
|
|
|
25
22
|
[project.scripts]
|
|
26
23
|
tumblrbot = "tumblrbot.__main__:main"
|
|
27
24
|
|
|
28
25
|
[build-system]
|
|
29
|
-
requires = ["
|
|
30
|
-
build-backend = "
|
|
26
|
+
requires = ["hatchling"]
|
|
27
|
+
build-backend = "hatchling.build"
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import sys
|
|
2
|
-
|
|
3
1
|
from openai import OpenAI
|
|
4
2
|
from rich.prompt import Confirm
|
|
5
3
|
from rich.traceback import install
|
|
@@ -8,8 +6,8 @@ from tumblrbot.flow.download import PostDownloader
|
|
|
8
6
|
from tumblrbot.flow.examples import ExamplesWriter
|
|
9
7
|
from tumblrbot.flow.fine_tune import FineTuner
|
|
10
8
|
from tumblrbot.flow.generate import DraftGenerator
|
|
9
|
+
from tumblrbot.utils.common import TumblrClient
|
|
11
10
|
from tumblrbot.utils.settings import Tokens
|
|
12
|
-
from tumblrbot.utils.tumblr import TumblrClient
|
|
13
11
|
|
|
14
12
|
|
|
15
13
|
def main() -> None:
|
|
@@ -33,7 +31,3 @@ def main() -> None:
|
|
|
33
31
|
|
|
34
32
|
if Confirm.ask("Generate drafts?", default=False):
|
|
35
33
|
DraftGenerator(openai, tumblr).create_drafts()
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if __name__ == "__main__":
|
|
39
|
-
sys.exit(main())
|
|
@@ -2,19 +2,17 @@ from io import TextIOBase
|
|
|
2
2
|
from json import dump
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
|
|
5
|
-
from more_itertools import last
|
|
6
|
-
|
|
7
5
|
from tumblrbot.utils.common import PreviewLive, UtilClass
|
|
8
6
|
from tumblrbot.utils.models import Post
|
|
9
7
|
|
|
10
8
|
|
|
11
9
|
class PostDownloader(UtilClass):
|
|
12
|
-
def paginate_posts(self,
|
|
13
|
-
task_id = live.progress.add_task(f"Downloading posts from '{
|
|
10
|
+
def paginate_posts(self, blog_identifier: str, offset: int, fp: TextIOBase, live: PreviewLive) -> None:
|
|
11
|
+
task_id = live.progress.add_task(f"Downloading posts from '{blog_identifier}'...", total=None, completed=offset)
|
|
14
12
|
|
|
15
13
|
while True:
|
|
16
|
-
response = self.tumblr.retrieve_published_posts(
|
|
17
|
-
live.progress.update(task_id, total=response["blog"]["posts"])
|
|
14
|
+
response = self.tumblr.retrieve_published_posts(blog_identifier, offset).json()["response"]
|
|
15
|
+
live.progress.update(task_id, total=response["blog"]["posts"], completed=offset)
|
|
18
16
|
|
|
19
17
|
if posts := response["posts"]:
|
|
20
18
|
for post in posts:
|
|
@@ -22,15 +20,14 @@ class PostDownloader(UtilClass):
|
|
|
22
20
|
fp.write("\n")
|
|
23
21
|
|
|
24
22
|
model = Post.model_validate(post)
|
|
25
|
-
before = model.timestamp
|
|
26
|
-
|
|
27
|
-
live.progress.update(task_id, advance=1)
|
|
28
23
|
live.custom_update(model)
|
|
24
|
+
|
|
25
|
+
offset += len(posts)
|
|
29
26
|
else:
|
|
30
27
|
break
|
|
31
28
|
|
|
32
|
-
def get_data_path(self,
|
|
33
|
-
return (self.config.data_directory /
|
|
29
|
+
def get_data_path(self, blog_identifier: str) -> Path:
|
|
30
|
+
return (self.config.data_directory / blog_identifier).with_suffix(".jsonl")
|
|
34
31
|
|
|
35
32
|
def get_data_paths(self) -> list[Path]:
|
|
36
33
|
return list(map(self.get_data_path, self.config.download_blog_identifiers))
|
|
@@ -39,15 +36,13 @@ class PostDownloader(UtilClass):
|
|
|
39
36
|
self.config.data_directory.mkdir(parents=True, exist_ok=True)
|
|
40
37
|
|
|
41
38
|
with PreviewLive() as live:
|
|
42
|
-
for
|
|
43
|
-
data_path = self.get_data_path(
|
|
44
|
-
lines = data_path.read_text("utf_8").splitlines() if data_path.exists() else []
|
|
39
|
+
for blog_identifier in self.config.download_blog_identifiers:
|
|
40
|
+
data_path = self.get_data_path(blog_identifier)
|
|
45
41
|
|
|
46
42
|
with data_path.open("a", encoding="utf_8") as fp:
|
|
47
43
|
self.paginate_posts(
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
len(lines),
|
|
44
|
+
blog_identifier,
|
|
45
|
+
len(data_path.read_text("utf_8").splitlines()) if data_path.exists() else 0,
|
|
51
46
|
fp,
|
|
52
47
|
live,
|
|
53
48
|
)
|
|
@@ -82,7 +82,7 @@ class FineTuner(UtilClass):
|
|
|
82
82
|
|
|
83
83
|
live.progress.update(
|
|
84
84
|
task_id,
|
|
85
|
-
description=f"Fine-tuning: {job.status}...",
|
|
85
|
+
description=f"Fine-tuning: [italic]{job.status.replace('_', ' ').title()}[/]...",
|
|
86
86
|
)
|
|
87
87
|
|
|
88
88
|
sleep(1)
|
|
@@ -101,5 +101,5 @@ class FineTuner(UtilClass):
|
|
|
101
101
|
Total tokens for [bold orange1]{self.config.expected_epochs}[/] epoch(s): {total_tokens:,}
|
|
102
102
|
Expected cost when trained with [bold purple]{self.config.base_model}[/]: {cost_string}
|
|
103
103
|
NOTE: Token values are approximate and may not be 100% accurate, please be aware of this when using the data.
|
|
104
|
-
[italic red]
|
|
104
|
+
[italic red]Amelia, Mutsumi, and Marin are not responsible for any inaccuracies in the token count or estimated price.[/]
|
|
105
105
|
""")
|
|
@@ -30,7 +30,10 @@ class DraftGenerator(UtilClass):
|
|
|
30
30
|
|
|
31
31
|
def generate_post(self) -> Post:
|
|
32
32
|
content = self.generate_content()
|
|
33
|
-
post = Post(
|
|
33
|
+
post = Post(
|
|
34
|
+
content=[content],
|
|
35
|
+
state="draft",
|
|
36
|
+
)
|
|
34
37
|
if tags := self.generate_tags(content):
|
|
35
38
|
post.tags = tags.tags
|
|
36
39
|
return post
|
|
@@ -42,7 +45,7 @@ class DraftGenerator(UtilClass):
|
|
|
42
45
|
for i in live.progress.track(range(self.config.draft_count), description="Generating drafts..."):
|
|
43
46
|
try:
|
|
44
47
|
post = self.generate_post()
|
|
45
|
-
self.tumblr.
|
|
48
|
+
self.tumblr.create_post(self.config.upload_blog_identifier, post)
|
|
46
49
|
live.custom_update(post)
|
|
47
50
|
except BaseException as exc:
|
|
48
51
|
exc.add_note(f"📉 An error occurred! Generated {i} draft(s) before failing. {message}")
|
|
@@ -18,16 +18,15 @@ class FullyValidatedModel(BaseModel):
|
|
|
18
18
|
|
|
19
19
|
class Post(FullyValidatedModel):
|
|
20
20
|
class Block(FullyValidatedModel):
|
|
21
|
-
type: str = "
|
|
21
|
+
type: str = ""
|
|
22
22
|
text: str = ""
|
|
23
|
-
blocks:
|
|
23
|
+
blocks: list[int] = [] # noqa: RUF012
|
|
24
24
|
|
|
25
25
|
tags: Annotated[list[str], PlainSerializer(",".join)] = [] # noqa: RUF012
|
|
26
26
|
content: SkipJsonSchema[list[Block]] = [] # noqa: RUF012
|
|
27
27
|
layout: SkipJsonSchema[list[Block]] = [] # noqa: RUF012
|
|
28
28
|
trail: SkipJsonSchema[list[Any]] = [] # noqa: RUF012
|
|
29
|
-
state: SkipJsonSchema[Literal["published", "queued", "draft", "private", "unapproved"]] = "
|
|
30
|
-
timestamp: SkipJsonSchema[int] = 0
|
|
29
|
+
state: SkipJsonSchema[Literal["published", "queued", "draft", "private", "unapproved"]] = "published"
|
|
31
30
|
is_submission: SkipJsonSchema[bool] = False
|
|
32
31
|
|
|
33
32
|
def __rich__(self) -> Panel:
|
|
@@ -45,7 +44,7 @@ class Post(FullyValidatedModel):
|
|
|
45
44
|
indices: set[int] = set()
|
|
46
45
|
for block in self.layout:
|
|
47
46
|
if block.type == "ask":
|
|
48
|
-
indices
|
|
47
|
+
indices.update(block.blocks)
|
|
49
48
|
|
|
50
49
|
self.content = [block for i, block in enumerate(self.content) if i not in indices and block.type == "text"]
|
|
51
50
|
|
|
@@ -55,7 +54,7 @@ class Post(FullyValidatedModel):
|
|
|
55
54
|
|
|
56
55
|
class Example(FullyValidatedModel):
|
|
57
56
|
class Message(FullyValidatedModel):
|
|
58
|
-
role:
|
|
57
|
+
role: Literal["developer", "user", "assistant"]
|
|
59
58
|
content: str
|
|
60
59
|
|
|
61
60
|
messages: list[Message]
|
|
@@ -7,7 +7,7 @@ from openai.types import ChatModel
|
|
|
7
7
|
from pydantic import Field, PositiveFloat, PositiveInt, Secret, model_validator
|
|
8
8
|
from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict, TomlConfigSettingsSource
|
|
9
9
|
from rich.prompt import Prompt
|
|
10
|
-
from tomlkit import comment, document, dumps
|
|
10
|
+
from tomlkit import comment, document, dumps # pyright: ignore[reportUnknownVariableType]
|
|
11
11
|
|
|
12
12
|
if TYPE_CHECKING:
|
|
13
13
|
from _typeshed import StrPath
|
|
@@ -16,7 +16,7 @@ class TumblrClient(OAuth2Session):
|
|
|
16
16
|
tokens: Tokens
|
|
17
17
|
|
|
18
18
|
def __post_init__(self) -> None:
|
|
19
|
-
super().__init__(
|
|
19
|
+
super().__init__( # pyright: ignore[reportUnknownMemberType]
|
|
20
20
|
self.tokens.tumblr_client_id.get_secret_value(),
|
|
21
21
|
auto_refresh_url="https://api.tumblr.com/v2/oauth2/token",
|
|
22
22
|
auto_refresh_kwargs={
|
|
@@ -35,14 +35,14 @@ class TumblrClient(OAuth2Session):
|
|
|
35
35
|
super().__enter__()
|
|
36
36
|
|
|
37
37
|
if not self.tokens.tumblr_token.get_secret_value():
|
|
38
|
-
authorization_url, _ = self.authorization_url("https://tumblr.com/oauth2/authorize")
|
|
38
|
+
authorization_url, _ = self.authorization_url("https://tumblr.com/oauth2/authorize") # pyright: ignore[reportUnknownMemberType]
|
|
39
39
|
|
|
40
40
|
rich.print(f"Please go to {authorization_url} and authorize access.")
|
|
41
41
|
authorization_response = Prompt.ask("Enter the full callback URL")
|
|
42
42
|
rich.print("\n")
|
|
43
43
|
|
|
44
44
|
self.token_saver(
|
|
45
|
-
self.fetch_token(
|
|
45
|
+
self.fetch_token( # pyright: ignore[reportUnknownMemberType]
|
|
46
46
|
"https://api.tumblr.com/v2/oauth2/token",
|
|
47
47
|
authorization_response=authorization_response,
|
|
48
48
|
client_secret=self.tokens.tumblr_client_secret.get_secret_value(),
|
|
@@ -68,17 +68,18 @@ class TumblrClient(OAuth2Session):
|
|
|
68
68
|
error.add_note(str(json))
|
|
69
69
|
raise
|
|
70
70
|
|
|
71
|
-
def
|
|
72
|
-
return self.post(
|
|
73
|
-
f"https://api.tumblr.com/v2/blog/{blog_name}/posts",
|
|
74
|
-
json=post.model_dump(mode="json"),
|
|
75
|
-
)
|
|
76
|
-
|
|
77
|
-
def retrieve_published_posts(self, blog_name: str, before: int) -> Response:
|
|
71
|
+
def retrieve_published_posts(self, blog_identifier: str, offset: int) -> Response:
|
|
78
72
|
return self.get(
|
|
79
|
-
f"https://api.tumblr.com/v2/blog/{
|
|
73
|
+
f"https://api.tumblr.com/v2/blog/{blog_identifier}/posts",
|
|
80
74
|
params={
|
|
81
|
-
"
|
|
75
|
+
"offset": offset,
|
|
76
|
+
"sort": "asc",
|
|
82
77
|
"npf": True,
|
|
83
78
|
},
|
|
84
79
|
)
|
|
80
|
+
|
|
81
|
+
def create_post(self, blog_identifier: str, post: Post) -> Response:
|
|
82
|
+
return self.post(
|
|
83
|
+
f"https://api.tumblr.com/v2/blog/{blog_identifier}/posts",
|
|
84
|
+
json=post.model_dump(mode="json"),
|
|
85
|
+
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|