ofscraper 3.7.0.dev11__tar.gz → 3.8.0.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/PKG-INFO +1 -1
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/actions/process.py +8 -37
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/actions/scraper.py +1 -1
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/archive.py +21 -23
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/messages.py +19 -4
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/paid.py +14 -6
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/timeline.py +20 -5
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/media.py +53 -9
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/placeholder.py +6 -4
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/commands/scraper.py +1 -1
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/other_url.py +1 -1
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/common.py +5 -8
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/media/helpers.py +4 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/media/main.py +19 -1
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/models/retriver.py +12 -1
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/models/selector.py +5 -15
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/parse.py +14 -6
- ofscraper-3.8.0.dev0/ofscraper/utils/args/quality.py +13 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/read.py +4 -2
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/pyproject.toml +1 -1
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/LICENSE +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/README.md +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/__main__.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/__version__.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/__version__.pye +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/actions/__init__.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/actions/like.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/highlights.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/init.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/labels.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/me.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/pinned.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/profile.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/subscriptions/helpers.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/subscriptions/individual.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/subscriptions/lists.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/subscriptions/subscriptions.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/labels.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/models.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/multiprocessprogress.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/posts.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/semaphoreDelayed.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/sessionbuilder.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/table.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/commands/check.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/commands/manual.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/commands/picker.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/config/paths.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/config.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/constants.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/download.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/files.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/general.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/logger.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/prompts.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/req.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/test_constants.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/time.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/url.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/db/__init__.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/db/operations.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/db/queries.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/alt_download.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/alt_downloadbatch.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/download.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/downloadbatch.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/downloadnormal.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/keyhelpers.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/main_download.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/main_downloadbatch.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/date.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/flags.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/other.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/price.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/sort.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/subtype.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/keybindings.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/model_helpers.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/promptConvert.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/actions.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/area.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/auth.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/binary.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/config.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/menu.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/model.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/profile.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_strings.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_validators.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompts.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/runner/exit.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/runner/load.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/runner/run.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/__init__.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/actions.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/areas.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/globals.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/helpers.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/write.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/auth.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/binaries.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/cache.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/checkers.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/config.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/context.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/custom.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/data.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/file.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/menu.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/schema.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/wrapper.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/console.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/constants.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/context/exit.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/context/run_async.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/context/stdout.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/dates.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/encoding.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/classes.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/close.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/globals.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/helpers.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/logger.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/logs.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/other.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/stdout.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/manager.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/me.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/menu.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/paths/check.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/paths/common.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/paths/manage.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/paths/paths.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/profiles/data.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/profiles/manage.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/profiles/tools.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/run.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/separate.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/system/free.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/system/network.py +0 -0
- {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/system/system.py +0 -0
|
@@ -23,11 +23,13 @@ import ofscraper.actions.like as like
|
|
|
23
23
|
import ofscraper.actions.scraper as OF
|
|
24
24
|
import ofscraper.api.init as init
|
|
25
25
|
import ofscraper.api.profile as profile
|
|
26
|
+
import ofscraper.classes.models as models
|
|
26
27
|
import ofscraper.classes.placeholder as placeholder
|
|
27
28
|
import ofscraper.db.operations as operations
|
|
28
29
|
import ofscraper.download.download as download
|
|
29
30
|
import ofscraper.filters.media.main as filters
|
|
30
31
|
import ofscraper.models.selector as userselector
|
|
32
|
+
import ofscraper.models.selector as selector
|
|
31
33
|
import ofscraper.utils.actions as actions
|
|
32
34
|
import ofscraper.utils.args.areas as areas
|
|
33
35
|
import ofscraper.utils.args.read as read_args
|
|
@@ -119,10 +121,13 @@ def scrape_paid(user_dict=None):
|
|
|
119
121
|
)
|
|
120
122
|
for ele in output
|
|
121
123
|
]
|
|
124
|
+
oldUsers = selector.get_ALL_SUBS_DICT()
|
|
122
125
|
for value in user_dict.values():
|
|
123
126
|
model_id = value[0].post.model_id
|
|
124
127
|
username = value[0].post.username
|
|
125
|
-
|
|
128
|
+
selector.set_ALL_SUBS_DICTVManger(
|
|
129
|
+
{username: models.Model(profile.scrape_profile(model_id))}
|
|
130
|
+
)
|
|
126
131
|
operations.create_tables(model_id=model_id, username=username)
|
|
127
132
|
operations.create_backup(model_id, username)
|
|
128
133
|
operations.write_profile_table(model_id=model_id, username=username)
|
|
@@ -131,6 +136,8 @@ def scrape_paid(user_dict=None):
|
|
|
131
136
|
model_id,
|
|
132
137
|
value,
|
|
133
138
|
)
|
|
139
|
+
# restore og users
|
|
140
|
+
selector.set_ALL_SUBS_DICT(oldUsers)
|
|
134
141
|
|
|
135
142
|
|
|
136
143
|
@exit.exit_wrapper
|
|
@@ -169,42 +176,6 @@ def normal_post_process():
|
|
|
169
176
|
log.traceback_(f"failed with exception: {e}")
|
|
170
177
|
log.traceback_(traceback.format_exc())
|
|
171
178
|
|
|
172
|
-
if read_args.retriveArgs().scrape_paid:
|
|
173
|
-
user_dict = {}
|
|
174
|
-
[
|
|
175
|
-
user_dict.update(
|
|
176
|
-
{ele.post.model_id: user_dict.get(ele.post.model_id, []) + [ele]}
|
|
177
|
-
)
|
|
178
|
-
for ele in OF.process_all_paid()
|
|
179
|
-
]
|
|
180
|
-
for value in user_dict.values():
|
|
181
|
-
try:
|
|
182
|
-
model_id = value[0].post.model_id
|
|
183
|
-
username = value[0].post.username
|
|
184
|
-
log.info(
|
|
185
|
-
f"inserting {len(value)} items into into media table for {username}"
|
|
186
|
-
)
|
|
187
|
-
operations.batch_mediainsert(
|
|
188
|
-
value,
|
|
189
|
-
operations.write_media_table_batch,
|
|
190
|
-
model_id=model_id,
|
|
191
|
-
username=username,
|
|
192
|
-
downloaded=False,
|
|
193
|
-
)
|
|
194
|
-
operations.create_tables(model_id=model_id, username=username)
|
|
195
|
-
operations.create_backup(model_id, username)
|
|
196
|
-
operations.write_profile_table(model_id=model_id, username=username)
|
|
197
|
-
download.download_picker(
|
|
198
|
-
username,
|
|
199
|
-
model_id,
|
|
200
|
-
value,
|
|
201
|
-
)
|
|
202
|
-
except Exception as E:
|
|
203
|
-
if isinstance(e, KeyboardInterrupt):
|
|
204
|
-
raise E
|
|
205
|
-
log.traceback_(f"failed with exception: {E}")
|
|
206
|
-
log.traceback_(traceback.format_exc())
|
|
207
|
-
|
|
208
179
|
|
|
209
180
|
@exit.exit_wrapper
|
|
210
181
|
def process_like():
|
|
@@ -183,11 +183,6 @@ async def get_archived_media(model_id, username, forced_after=None, rescan=None)
|
|
|
183
183
|
min_posts = 50
|
|
184
184
|
responseArray = []
|
|
185
185
|
page_count = 0
|
|
186
|
-
setCache = (
|
|
187
|
-
True
|
|
188
|
-
if (read_args.retriveArgs().after == 0 or not read_args.retriveArgs().after)
|
|
189
|
-
else False
|
|
190
|
-
)
|
|
191
186
|
|
|
192
187
|
with Live(
|
|
193
188
|
progress_group, refresh_per_second=5, console=console.get_shared_console()
|
|
@@ -221,11 +216,9 @@ async def get_archived_media(model_id, username, forced_after=None, rescan=None)
|
|
|
221
216
|
)
|
|
222
217
|
after = 0
|
|
223
218
|
|
|
224
|
-
elif forced_after != None:
|
|
225
|
-
after = forced_after
|
|
226
219
|
else:
|
|
227
|
-
after = get_after(model_id, username)
|
|
228
|
-
|
|
220
|
+
after = get_after(model_id, username, forced_after)
|
|
221
|
+
# set check
|
|
229
222
|
log.info(
|
|
230
223
|
f"""
|
|
231
224
|
Setting initial archived scan date for {username} to {arrow.get(after).format('YYYY.MM.DD')}
|
|
@@ -334,23 +327,28 @@ Setting initial archived scan date for {username} to {arrow.get(after).format('Y
|
|
|
334
327
|
)
|
|
335
328
|
)
|
|
336
329
|
log.debug(f"[bold]Archived Count without Dupes[/bold] {len(unduped)} found")
|
|
337
|
-
|
|
338
|
-
newCheck = {}
|
|
339
|
-
for post in cache.get(f"archived_check_{model_id}", []) + list(
|
|
340
|
-
unduped.values()
|
|
341
|
-
):
|
|
342
|
-
newCheck[post["id"]] = post
|
|
343
|
-
cache.set(
|
|
344
|
-
f"archived_check_{model_id}",
|
|
345
|
-
list(newCheck.values()),
|
|
346
|
-
expire=constants.getattr("DAY_SECONDS"),
|
|
347
|
-
)
|
|
348
|
-
cache.close()
|
|
349
|
-
|
|
330
|
+
set_check(unduped, model_id, after)
|
|
350
331
|
return list(unduped.values())
|
|
351
332
|
|
|
352
333
|
|
|
353
|
-
def
|
|
334
|
+
def set_check(unduped, model_id, after):
|
|
335
|
+
if not after:
|
|
336
|
+
newCheck = {}
|
|
337
|
+
for post in cache.get(f"archived_check_{model_id}", []) + list(
|
|
338
|
+
unduped.values()
|
|
339
|
+
):
|
|
340
|
+
newCheck[post["id"]] = post
|
|
341
|
+
cache.set(
|
|
342
|
+
f"archived_check_{model_id}",
|
|
343
|
+
list(newCheck.values()),
|
|
344
|
+
expire=constants.getattr("DAY_SECONDS"),
|
|
345
|
+
)
|
|
346
|
+
cache.close()
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def get_after(model_id, username, forced_after=None):
|
|
350
|
+
if forced_after != None:
|
|
351
|
+
return forced_after
|
|
354
352
|
if read_args.retriveArgs().after:
|
|
355
353
|
return read_args.retriveArgs().after.float_timestamp
|
|
356
354
|
curr = operations.get_archived_media(model_id=model_id, username=username)
|
|
@@ -111,10 +111,9 @@ async def get_messages(model_id, username, forced_after=None, rescan=None):
|
|
|
111
111
|
)
|
|
112
112
|
after = 0
|
|
113
113
|
|
|
114
|
-
elif forced_after != None:
|
|
115
|
-
after = forced_after
|
|
116
114
|
else:
|
|
117
|
-
after = get_after(model_id, username)
|
|
115
|
+
after = get_after(model_id, username, forced_after)
|
|
116
|
+
|
|
118
117
|
log.debug(f"Messages after = {after}")
|
|
119
118
|
|
|
120
119
|
log.debug(f"Messages before = {before}")
|
|
@@ -305,9 +304,23 @@ Setting initial message scan date for {username} to {arrow.get(after).format('YY
|
|
|
305
304
|
)
|
|
306
305
|
)
|
|
307
306
|
)
|
|
307
|
+
set_check(unduped, model_id, after)
|
|
308
308
|
return list(unduped.values())
|
|
309
309
|
|
|
310
310
|
|
|
311
|
+
def set_check(unduped, model_id, after):
|
|
312
|
+
if not after:
|
|
313
|
+
newCheck = {}
|
|
314
|
+
for post in cache.get(f"message_check_{model_id}", []) + list(unduped.values()):
|
|
315
|
+
newCheck[post["id"]] = post
|
|
316
|
+
cache.set(
|
|
317
|
+
f"message_check_{model_id}",
|
|
318
|
+
list(newCheck.values()),
|
|
319
|
+
expire=constants.getattr("DAY_SECONDS"),
|
|
320
|
+
)
|
|
321
|
+
cache.close()
|
|
322
|
+
|
|
323
|
+
|
|
311
324
|
async def scrape_messages(
|
|
312
325
|
c, model_id, progress, message_id=None, required_ids=None
|
|
313
326
|
) -> list:
|
|
@@ -449,7 +462,9 @@ def get_individual_post(model_id, postid, c=None):
|
|
|
449
462
|
log.debug(f"[bold]Individual message headers:[/bold] {r.headers}")
|
|
450
463
|
|
|
451
464
|
|
|
452
|
-
def get_after(model_id, username):
|
|
465
|
+
def get_after(model_id, username, forced_after=None):
|
|
466
|
+
if forced_after != None:
|
|
467
|
+
return forced_after
|
|
453
468
|
if read_args.retriveArgs().after:
|
|
454
469
|
return read_args.retriveArgs().after.float_timestamp
|
|
455
470
|
curr = operations.get_messages_media(model_id=model_id, username=username)
|
|
@@ -30,6 +30,7 @@ from tenacity import (
|
|
|
30
30
|
)
|
|
31
31
|
|
|
32
32
|
import ofscraper.classes.sessionbuilder as sessionbuilder
|
|
33
|
+
import ofscraper.utils.args.read as read_args
|
|
33
34
|
import ofscraper.utils.cache as cache
|
|
34
35
|
import ofscraper.utils.console as console
|
|
35
36
|
import ofscraper.utils.constants as constants
|
|
@@ -104,15 +105,22 @@ async def get_paid_posts(username, model_id):
|
|
|
104
105
|
)
|
|
105
106
|
)
|
|
106
107
|
)
|
|
107
|
-
|
|
108
|
-
f"purchased_check_{model_id}",
|
|
109
|
-
list(outdict.values()),
|
|
110
|
-
expire=constants.getattr("DAY_SECONDS"),
|
|
111
|
-
)
|
|
112
|
-
cache.close()
|
|
108
|
+
set_check(outdict, model_id)
|
|
113
109
|
return list(outdict.values())
|
|
114
110
|
|
|
115
111
|
|
|
112
|
+
def set_check(unduped, model_id):
|
|
113
|
+
newCheck = {}
|
|
114
|
+
for post in cache.get(f"purchased_check_{model_id}", []) + list(unduped.values()):
|
|
115
|
+
newCheck[post["id"]] = post
|
|
116
|
+
cache.set(
|
|
117
|
+
f"purchased_check_{model_id}",
|
|
118
|
+
list(newCheck.values()),
|
|
119
|
+
expire=constants.getattr("DAY_SECONDS"),
|
|
120
|
+
)
|
|
121
|
+
cache.close()
|
|
122
|
+
|
|
123
|
+
|
|
116
124
|
async def scrape_paid(c, username, job_progress, offset=0):
|
|
117
125
|
"""Takes headers to access onlyfans as an argument and then checks the purchased content
|
|
118
126
|
url to look for any purchased content. If it finds some it will return it as a list.
|
|
@@ -208,11 +208,8 @@ async def get_timeline_media(model_id, username, forced_after=None, rescan=None)
|
|
|
208
208
|
"Used --after previously. Scraping all timeline posts required to make sure content is not missing"
|
|
209
209
|
)
|
|
210
210
|
after = 0
|
|
211
|
-
|
|
212
|
-
elif forced_after != None:
|
|
213
|
-
after = forced_after
|
|
214
211
|
else:
|
|
215
|
-
after = get_after(model_id, username)
|
|
212
|
+
after = get_after(model_id, username, forced_after)
|
|
216
213
|
|
|
217
214
|
log.info(
|
|
218
215
|
f"""
|
|
@@ -330,9 +327,25 @@ Setting initial timeline scan date for {username} to {arrow.get(after).format('Y
|
|
|
330
327
|
)
|
|
331
328
|
)
|
|
332
329
|
log.debug(f"[bold]Timeline Count without Dupes[/bold] {len(unduped)} found")
|
|
330
|
+
set_check(unduped, model_id, after)
|
|
333
331
|
return list(unduped.values())
|
|
334
332
|
|
|
335
333
|
|
|
334
|
+
def set_check(unduped, model_id, after):
|
|
335
|
+
if not after:
|
|
336
|
+
newCheck = {}
|
|
337
|
+
for post in cache.get(f"timeline_check_{model_id}", []) + list(
|
|
338
|
+
unduped.values()
|
|
339
|
+
):
|
|
340
|
+
newCheck[post["id"]] = post
|
|
341
|
+
cache.set(
|
|
342
|
+
f"timeline_check_{model_id}",
|
|
343
|
+
list(newCheck.values()),
|
|
344
|
+
expire=constants.getattr("DAY_SECONDS"),
|
|
345
|
+
)
|
|
346
|
+
cache.close()
|
|
347
|
+
|
|
348
|
+
|
|
336
349
|
def get_individual_post(id, c=None):
|
|
337
350
|
with c.requests(constants.getattr("INDVIDUAL_TIMELINE").format(id))() as r:
|
|
338
351
|
if r.ok:
|
|
@@ -344,7 +357,9 @@ def get_individual_post(id, c=None):
|
|
|
344
357
|
log.debug(f"[bold]individual post headers:[/bold] {r.headers}")
|
|
345
358
|
|
|
346
359
|
|
|
347
|
-
def get_after(model_id, username):
|
|
360
|
+
def get_after(model_id, username, forced_after=None):
|
|
361
|
+
if forced_after != None:
|
|
362
|
+
return forced_after
|
|
348
363
|
if read_args.retriveArgs().after:
|
|
349
364
|
return read_args.retriveArgs().after.float_timestamp
|
|
350
365
|
curr = operations.get_timeline_media(model_id=model_id, username=username)
|
|
@@ -18,6 +18,7 @@ from tenacity import (
|
|
|
18
18
|
)
|
|
19
19
|
|
|
20
20
|
import ofscraper.classes.sessionbuilder as sessionbuilder
|
|
21
|
+
import ofscraper.utils.args.quality as quality
|
|
21
22
|
import ofscraper.utils.config.data as data
|
|
22
23
|
import ofscraper.utils.constants as constants
|
|
23
24
|
|
|
@@ -32,11 +33,29 @@ class Media:
|
|
|
32
33
|
self._media = media
|
|
33
34
|
self._count = count
|
|
34
35
|
self._post = post
|
|
36
|
+
self._final_url = None
|
|
35
37
|
|
|
36
38
|
@property
|
|
37
39
|
def expires(self):
|
|
38
40
|
return self._post.expires
|
|
39
41
|
|
|
42
|
+
@property
|
|
43
|
+
def media_source(self):
|
|
44
|
+
return self._media.get("source", {})
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def files_source(self):
|
|
48
|
+
return self._media.get("files", {}).get("source", {})
|
|
49
|
+
allowed = quality.get_allowed_qualities()
|
|
50
|
+
for ele in ["240", "720", "source"]:
|
|
51
|
+
if ele in allowed and self._media.get("files", {}).get(ele):
|
|
52
|
+
return self._media.get("files", {}).get(ele)
|
|
53
|
+
return {}
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def quality(self):
|
|
57
|
+
return self._media.get("videoSources", {})
|
|
58
|
+
|
|
40
59
|
@property
|
|
41
60
|
def mediatype(self):
|
|
42
61
|
if self._media["type"] == "photo":
|
|
@@ -50,9 +69,7 @@ class Media:
|
|
|
50
69
|
|
|
51
70
|
@property
|
|
52
71
|
def length(self):
|
|
53
|
-
return self._media.get("duration") or self.
|
|
54
|
-
"duration"
|
|
55
|
-
)
|
|
72
|
+
return self._media.get("duration") or self.media_source.get("duration")
|
|
56
73
|
|
|
57
74
|
@property
|
|
58
75
|
def numeric_length(self):
|
|
@@ -62,12 +79,21 @@ class Media:
|
|
|
62
79
|
|
|
63
80
|
@property
|
|
64
81
|
def url(self):
|
|
82
|
+
if self._final_url:
|
|
83
|
+
return self._final_url
|
|
65
84
|
if self.responsetype == "stories" or self.responsetype == "highlights":
|
|
66
|
-
|
|
85
|
+
self._final_url = self.files_source.get("url")
|
|
67
86
|
elif self.responsetype == "profile":
|
|
68
|
-
|
|
87
|
+
self._final_url = self._media.get("url")
|
|
69
88
|
else:
|
|
89
|
+
self._final_url = self._url_source_helper()
|
|
90
|
+
return self._final_url
|
|
91
|
+
|
|
92
|
+
def _url_source_helper(self):
|
|
93
|
+
quality = self.selected_quality
|
|
94
|
+
if quality == "source":
|
|
70
95
|
return self._media.get("source", {}).get("source")
|
|
96
|
+
return self._media.get("videoSources", {}).get(quality)
|
|
71
97
|
|
|
72
98
|
@property
|
|
73
99
|
def post(self):
|
|
@@ -256,9 +282,17 @@ class Media:
|
|
|
256
282
|
def final_filename(self):
|
|
257
283
|
filename = self.filename or self.id
|
|
258
284
|
if self.mediatype == "videos":
|
|
259
|
-
filename = (
|
|
260
|
-
|
|
261
|
-
|
|
285
|
+
filename = re.sub("_[a-z0-9]+$", f"", filename)
|
|
286
|
+
filename = f"{filename}_{self.selected_quality}"
|
|
287
|
+
# cleanup
|
|
288
|
+
filename = self.cleanup(filename)
|
|
289
|
+
return filename
|
|
290
|
+
|
|
291
|
+
@property
|
|
292
|
+
def no_quality_final_filename(self):
|
|
293
|
+
filename = self.filename or self.id
|
|
294
|
+
if self.mediatype == "videos":
|
|
295
|
+
filename = re.sub("_[a-z]+", f"", filename)
|
|
262
296
|
# cleanup
|
|
263
297
|
filename = self.cleanup(filename)
|
|
264
298
|
return filename
|
|
@@ -322,10 +356,20 @@ class Media:
|
|
|
322
356
|
def mediatype(self, val):
|
|
323
357
|
self._media["type"] = val
|
|
324
358
|
|
|
359
|
+
@property
|
|
360
|
+
def selected_quality(self):
|
|
361
|
+
allowed = quality.get_allowed_qualities()
|
|
362
|
+
if self.mediatype != "videos":
|
|
363
|
+
return "source"
|
|
364
|
+
for ele in ["240", "720"]:
|
|
365
|
+
if ele in allowed and self._media.get("videoSources", {}).get(ele):
|
|
366
|
+
return ele
|
|
367
|
+
return "source"
|
|
368
|
+
|
|
325
369
|
# for use in dynamic names
|
|
326
370
|
@property
|
|
327
371
|
def needs_count(self):
|
|
328
|
-
if set(["
|
|
372
|
+
if set(["text", "postid", "post_id"]).isdisjoint(
|
|
329
373
|
[
|
|
330
374
|
name
|
|
331
375
|
for text, name, spec, conv in list(
|
|
@@ -128,6 +128,12 @@ class Placeholders:
|
|
|
128
128
|
self._variables.update({"media_id": media_id})
|
|
129
129
|
self._variables.update({"mediaid": media_id})
|
|
130
130
|
self._variables.update({"modelObj": selector.get_model_fromParsed(username)})
|
|
131
|
+
self._variables.update({"quality": ele.selected_quality})
|
|
132
|
+
self._variables.update({"filename": ele.final_filename})
|
|
133
|
+
self._variables.update({"file_name": ele.final_filename})
|
|
134
|
+
self._variables.update({"only_file_name": ele.no_quality_final_filename})
|
|
135
|
+
self._variables.update({"only_filename": ele.no_quality_final_filename})
|
|
136
|
+
self._variables.update({"text": ele.file_text})
|
|
131
137
|
self.add_price_variables(username)
|
|
132
138
|
|
|
133
139
|
@wrapper
|
|
@@ -235,11 +241,7 @@ class Placeholders:
|
|
|
235
241
|
@wrapper
|
|
236
242
|
def createfilename(self, ele, username, model_id, ext):
|
|
237
243
|
filename = ele.final_filename
|
|
238
|
-
self._variables.update({"filename": filename})
|
|
239
|
-
self._variables.update({"file_name": filename})
|
|
240
244
|
self._variables.update({"ext": ext})
|
|
241
|
-
text = ele.file_text
|
|
242
|
-
self._variables.update({"text": text})
|
|
243
245
|
self.add_common_variables(ele, username, model_id)
|
|
244
246
|
globals().update(self._variables)
|
|
245
247
|
log.trace(
|
|
@@ -119,7 +119,7 @@ def scrapper():
|
|
|
119
119
|
if len(args.action) == 0 and not args.scrape_paid:
|
|
120
120
|
prompts.action_prompt()
|
|
121
121
|
daemon_process()
|
|
122
|
-
elif len(args.action) > 0:
|
|
122
|
+
elif len(args.action) > 0 or args.scrape_paid:
|
|
123
123
|
process_selected_areas()
|
|
124
124
|
elif len(args.action) == 0:
|
|
125
125
|
process_prompts()
|
|
@@ -8,5 +8,5 @@ FFMPEG_WINDOWS = "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest
|
|
|
8
8
|
FFMPEG_MAC = "https://evermeet.cx/ffmpeg/#api-download"
|
|
9
9
|
CDRM2 = "http://172.106.17.134:8080/wv"
|
|
10
10
|
KEYDB = "https://keysdb.net/api"
|
|
11
|
-
CDRM = "https://cdrm-project.com"
|
|
11
|
+
CDRM = "https://cdrm-project.com/wv"
|
|
12
12
|
LARGEZIP = "https://proof.ovh.net/files/100Mb.dat"
|
|
@@ -524,11 +524,8 @@ def log_download_progress(media_type):
|
|
|
524
524
|
if media_type is None:
|
|
525
525
|
return
|
|
526
526
|
if (photo_count + audio_count + video_count + forced_skipped + skipped) % 20 == 0:
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
# downloads total [{video_count} videos, {audio_count} audios, {photo_count} photos], \
|
|
533
|
-
# {forced_skipped} skipped, {skipped} failed)"
|
|
534
|
-
# )
|
|
527
|
+
log.debug(
|
|
528
|
+
f"In progress -> {format_size(total_bytes )}) ({photo_count+audio_count+video_count} \
|
|
529
|
+
downloads total [{video_count} videos, {audio_count} audios, {photo_count} photos], \
|
|
530
|
+
{forced_skipped} skipped, {skipped} failed)"
|
|
531
|
+
)
|
|
@@ -149,3 +149,7 @@ def mass_msg_filter(media):
|
|
|
149
149
|
return list((filter(lambda x: x.mass is True, media)))
|
|
150
150
|
elif read_args.retriveArgs().mass_msg is False:
|
|
151
151
|
return list((filter(lambda x: x.mass is False, media)))
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def url_filter(media):
|
|
155
|
+
return list((filter(lambda x: x.url or x.mpd, media)))
|
|
@@ -214,6 +214,24 @@ def filterMedia(media):
|
|
|
214
214
|
)
|
|
215
215
|
)
|
|
216
216
|
)
|
|
217
|
-
|
|
217
|
+
|
|
218
|
+
media = helpers.url_filter(media)
|
|
219
|
+
count += 1
|
|
220
|
+
log.trace(
|
|
221
|
+
"\n\n\n".join(
|
|
222
|
+
list(
|
|
223
|
+
map(
|
|
224
|
+
lambda x: logformater.format(
|
|
225
|
+
f"filter {count}-> valid url filter: ",
|
|
226
|
+
x.media,
|
|
227
|
+
x.id,
|
|
228
|
+
x.postid,
|
|
229
|
+
),
|
|
230
|
+
media,
|
|
231
|
+
)
|
|
232
|
+
)
|
|
233
|
+
)
|
|
234
|
+
)
|
|
235
|
+
log.debug(f"filter{count}-> all media valid url filter count {len(media)}")
|
|
218
236
|
|
|
219
237
|
return media
|
|
@@ -10,7 +10,7 @@ import ofscraper.utils.context.stdout as stdout
|
|
|
10
10
|
import ofscraper.utils.me as me_util
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
def get_models(
|
|
13
|
+
def get_models(main=False) -> list:
|
|
14
14
|
"""
|
|
15
15
|
Get user's subscriptions in form of a list.
|
|
16
16
|
"""
|
|
@@ -48,6 +48,17 @@ def get_via_list(count):
|
|
|
48
48
|
return models_objects
|
|
49
49
|
|
|
50
50
|
|
|
51
|
+
def get_main_list(count):
|
|
52
|
+
out = []
|
|
53
|
+
active_subscriptions = subscriptions.get_subscriptions(count[0], forced=True)
|
|
54
|
+
expired_subscriptions = subscriptions.get_subscriptions(
|
|
55
|
+
count[1], account="expired", forced=True
|
|
56
|
+
)
|
|
57
|
+
out.extend(active_subscriptions)
|
|
58
|
+
out.extend(expired_subscriptions)
|
|
59
|
+
return out
|
|
60
|
+
|
|
61
|
+
|
|
51
62
|
def get_via_individual():
|
|
52
63
|
out = individual.get_subscription()
|
|
53
64
|
console.get_shared_console().print(
|
|
@@ -34,9 +34,8 @@ def set_ALL_SUBS_DICT(subsDict=None):
|
|
|
34
34
|
subList = subsDict or ALL_SUBS
|
|
35
35
|
if not subList:
|
|
36
36
|
all_subs_helper()
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
[ALL_SUBS_DICT.update({ele.name: ele}) for ele in subList]
|
|
37
|
+
ALL_SUBS_DICT = {}
|
|
38
|
+
[ALL_SUBS_DICT.update({ele.name: ele}) for ele in subList]
|
|
40
39
|
|
|
41
40
|
|
|
42
41
|
def get_ALL_SUBS_DICT():
|
|
@@ -45,17 +44,8 @@ def get_ALL_SUBS_DICT():
|
|
|
45
44
|
|
|
46
45
|
|
|
47
46
|
def set_ALL_SUBS_DICTVManger(subsDict=None):
|
|
48
|
-
global ALL_SUBS
|
|
49
47
|
global ALL_SUBS_DICT
|
|
50
|
-
|
|
51
|
-
ALL_SUBS_DICT = subsDict
|
|
52
|
-
else:
|
|
53
|
-
subList = subsDict or ALL_SUBS
|
|
54
|
-
if not subList:
|
|
55
|
-
all_subs_helper()
|
|
56
|
-
if not ALL_SUBS_DICT:
|
|
57
|
-
ALL_SUBS_DICT = {}
|
|
58
|
-
[ALL_SUBS_DICT.update({ele.name: ele}) for ele in subList]
|
|
48
|
+
set_ALL_SUBS_DICT(subsDict)
|
|
59
49
|
manager.update_dict({"subs": ALL_SUBS_DICT})
|
|
60
50
|
|
|
61
51
|
|
|
@@ -94,12 +84,12 @@ def getselected_usernames(rescan=False, reset=False):
|
|
|
94
84
|
return PARSED_SUBS
|
|
95
85
|
|
|
96
86
|
|
|
97
|
-
def all_subs_helper(refetch=True):
|
|
87
|
+
def all_subs_helper(refetch=True, main=False):
|
|
98
88
|
global ALL_SUBS
|
|
99
89
|
if bool(ALL_SUBS) and not refetch:
|
|
100
90
|
return
|
|
101
91
|
while True:
|
|
102
|
-
ALL_SUBS = retriver.get_models()
|
|
92
|
+
ALL_SUBS = retriver.get_models(main)
|
|
103
93
|
if len(ALL_SUBS) > 0:
|
|
104
94
|
set_ALL_SUBS_DICTVManger()
|
|
105
95
|
break
|
|
@@ -8,7 +8,7 @@ import ofscraper.utils.args.helpers as helpers
|
|
|
8
8
|
import ofscraper.utils.args.write as write_args
|
|
9
9
|
import ofscraper.utils.system.system as system
|
|
10
10
|
from ofscraper.__version__ import __version__
|
|
11
|
-
from ofscraper.const.constants import KEY_OPTIONS
|
|
11
|
+
from ofscraper.const.constants import KEY_OPTIONS
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
def create_parser(input=None):
|
|
@@ -119,6 +119,15 @@ def create_parser(input=None):
|
|
|
119
119
|
default=False,
|
|
120
120
|
help="Bypass the dupe check and redownload all files",
|
|
121
121
|
)
|
|
122
|
+
|
|
123
|
+
post.add_argument(
|
|
124
|
+
"-q",
|
|
125
|
+
"--quality",
|
|
126
|
+
default="source",
|
|
127
|
+
help="Set the minimum allowed quality for videos",
|
|
128
|
+
choices=["240", "720", "source"],
|
|
129
|
+
)
|
|
130
|
+
|
|
122
131
|
post.add_argument(
|
|
123
132
|
"-o",
|
|
124
133
|
"--posts",
|
|
@@ -722,8 +731,7 @@ def create_parser(input=None):
|
|
|
722
731
|
group11.add_argument(
|
|
723
732
|
"-fl",
|
|
724
733
|
"--force-list",
|
|
725
|
-
help="When --username arg is provided searches entire list
|
|
726
|
-
config settings for userlist or the --user-list arg",
|
|
734
|
+
help="When --username arg is provided searches entire enabled list(s) before filtering for usernames",
|
|
727
735
|
default=False,
|
|
728
736
|
action="store_true",
|
|
729
737
|
)
|
|
@@ -732,7 +740,7 @@ config settings for userlist or the --user-list arg",
|
|
|
732
740
|
group12.add_argument(
|
|
733
741
|
"-md",
|
|
734
742
|
"--metadata",
|
|
735
|
-
help="Skip all media downloads and gathers metadata only\nNo change to download status",
|
|
743
|
+
help="Skip all media downloads and gathers metadata only\nNo change to download status in media table",
|
|
736
744
|
default=None,
|
|
737
745
|
action="store_const",
|
|
738
746
|
dest="metadata",
|
|
@@ -741,7 +749,7 @@ config settings for userlist or the --user-list arg",
|
|
|
741
749
|
group12.add_argument(
|
|
742
750
|
"-mu",
|
|
743
751
|
"--metadata-update",
|
|
744
|
-
help="Skip all media downloads and gathers metadata only\nUpdates
|
|
752
|
+
help="Skip all media downloads and gathers metadata only\nUpdates downloaded status in media table based on file presence",
|
|
745
753
|
default=None,
|
|
746
754
|
action="store_const",
|
|
747
755
|
dest="metadata",
|
|
@@ -750,7 +758,7 @@ config settings for userlist or the --user-list arg",
|
|
|
750
758
|
group12.add_argument(
|
|
751
759
|
"-mc",
|
|
752
760
|
"--metadata-complete",
|
|
753
|
-
help="Skip all media downloads and gathers metadata only\nMarks each media item as downloaded",
|
|
761
|
+
help="Skip all media downloads and gathers metadata only\nMarks each media item as downloaded in media table",
|
|
754
762
|
default=None,
|
|
755
763
|
action="store_const",
|
|
756
764
|
dest="metadata",
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import ofscraper.utils.args.read as read_args
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def get_allowed_qualities():
|
|
5
|
+
qualities = {"source": 4000, "240": 240, "720": 720}
|
|
6
|
+
minQuality = read_args.retriveArgs().quality
|
|
7
|
+
validQualities = list(
|
|
8
|
+
filter(
|
|
9
|
+
lambda x: x[1] >= qualities.get(minQuality),
|
|
10
|
+
qualities.items(),
|
|
11
|
+
)
|
|
12
|
+
)
|
|
13
|
+
return list(map(lambda x: x[0], validQualities))
|
|
@@ -11,7 +11,8 @@ def retriveArgsVManager():
|
|
|
11
11
|
manager.get_manager_dict()["args"] = parse_args.parse_args()
|
|
12
12
|
return manager.get_manager_dict().get("args")
|
|
13
13
|
except SystemExit as E:
|
|
14
|
-
|
|
14
|
+
if not any(ele in sys.argv[1:] for ele in ["-h", "-v"]):
|
|
15
|
+
print(f"Passed Args {sys.argv[1:]}")
|
|
15
16
|
raise E
|
|
16
17
|
|
|
17
18
|
|
|
@@ -21,5 +22,6 @@ def retriveArgs():
|
|
|
21
22
|
global_args.args = parse_args.parse_args()
|
|
22
23
|
return global_args.args
|
|
23
24
|
except SystemExit as E:
|
|
24
|
-
|
|
25
|
+
if not any(ele in sys.argv[1:] for ele in ["-h", "-v"]):
|
|
26
|
+
print(f"Passed Args {sys.argv[1:]}")
|
|
25
27
|
raise E
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|