ofscraper 3.10.dev6__tar.gz → 3.10.dev7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/PKG-INFO +1 -1
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/labels.py +5 -38
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/sessionmanager.py +5 -5
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/check.py +1 -1
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/helpers/strings.py +2 -2
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/scraper/actions/like.py +2 -2
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/scraper/post.py +1 -2
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/download/alt_download.py +12 -9
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/download/alt_downloadbatch.py +11 -11
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/download/download.py +2 -2
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/download/downloadbatch.py +7 -7
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/download/downloadnormal.py +6 -6
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/download/main_download.py +11 -9
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/download/main_downloadbatch.py +11 -9
- {ofscraper-3.10.dev6/ofscraper/download/shared/common → ofscraper-3.10.dev7/ofscraper/download/shared}/alt_common.py +3 -3
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/download/shared/classes/session.py +1 -1
- {ofscraper-3.10.dev6/ofscraper/download/shared/common → ofscraper-3.10.dev7/ofscraper/download/shared}/general.py +5 -50
- {ofscraper-3.10.dev6/ofscraper/download/shared/common → ofscraper-3.10.dev7/ofscraper/download/shared}/handle_result.py +4 -4
- {ofscraper-3.10.dev6/ofscraper/download/shared/utils → ofscraper-3.10.dev7/ofscraper/download/shared}/keyhelpers.py +2 -2
- {ofscraper-3.10.dev6/ofscraper/download/shared/utils → ofscraper-3.10.dev7/ofscraper/download/shared}/log.py +11 -11
- {ofscraper-3.10.dev6/ofscraper/download/shared/utils → ofscraper-3.10.dev7/ofscraper/download/shared}/metadata.py +4 -4
- {ofscraper-3.10.dev6/ofscraper/download/shared/utils → ofscraper-3.10.dev7/ofscraper/download/shared/paths}/paths.py +2 -2
- ofscraper-3.10.dev7/ofscraper/download/shared/progress/chunk.py +66 -0
- {ofscraper-3.10.dev6/ofscraper/download/shared/utils → ofscraper-3.10.dev7/ofscraper/download/shared/progress}/progress.py +1 -1
- {ofscraper-3.10.dev6/ofscraper/download/shared/utils → ofscraper-3.10.dev7/ofscraper/download/shared/send}/message.py +1 -1
- ofscraper-3.10.dev7/ofscraper/download/shared/send/send_bar_msg.py +27 -0
- {ofscraper-3.10.dev6/ofscraper/download/shared/utils → ofscraper-3.10.dev7/ofscraper/download/shared}/text.py +1 -1
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/logs/classes.py +12 -10
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/pyproject.toml +1 -1
- ofscraper-3.10.dev6/ofscraper/download/shared/common/send_bar_msg.py +0 -50
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/LICENSE +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/README.md +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/__main__.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/__version__.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/__version__.pye +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/archive.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/common/logs.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/highlights.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/init.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/me.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/messages.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/paid.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/pinned.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/profile.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/subscriptions/helpers.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/subscriptions/individual.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/subscriptions/lists.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/subscriptions/subscriptions.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/api/timeline.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/base.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/labels.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/media.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/models.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/multiprocessprogress.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/placeholder.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/posts.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/button.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/fields/boolfield.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/fields/datefield.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/fields/mediafield.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/fields/numfield.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/fields/pricefield.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/fields/responsefield.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/fields/selectfield.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/fields/textsearch.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/fields/timefield.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/inputs/filterinput.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/inputs/intergerinput.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/inputs/strinput.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/row_names.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/status.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/table.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/classes/table/table_console.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/helpers/data.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/helpers/final_log.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/helpers/normal.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/helpers/picker.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/helpers/scrape_paid.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/helpers/shared.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/helpers/user_first.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/manual.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/metadata.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/scraper/actions/download.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/scraper/runner.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/scraper/scrape_context.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/commands/scraper/scraper.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/constants.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/binary.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/config.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/date.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/download.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/files.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/general.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/like.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/live.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/logger.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/metadata.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/other_url.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/path.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/prompts.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/req.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/table.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/test_constants.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/time.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/const/values/url.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/__init__.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/operations.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/operations_/helpers.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/operations_/labels.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/operations_/media.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/operations_/merge.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/operations_/messages.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/operations_/others.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/operations_/posts.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/operations_/profile.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/operations_/stories.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/db/operations_/wrapper.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/download/shared/classes/retries.py +0 -0
- {ofscraper-3.10.dev6/ofscraper/download/shared → ofscraper-3.10.dev7/ofscraper/download/shared/globals}/globals.py +0 -0
- {ofscraper-3.10.dev6/ofscraper/download/shared/utils → ofscraper-3.10.dev7/ofscraper/download/shared/paths}/media.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/filters/media/helpers.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/filters/media/main.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/filters/models/date.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/filters/models/flags.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/filters/models/helpers.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/filters/models/other.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/filters/models/price.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/filters/models/sort.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/filters/models/subtype.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/models/retriver.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/models/selector.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/helpers/model_helpers.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/helpers/prompt_helpers.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/keybindings.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/promptConvert.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompt_groups/actions.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompt_groups/area.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompt_groups/auth.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompt_groups/binary.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompt_groups/config.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompt_groups/menu.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompt_groups/merge.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompt_groups/model.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompt_groups/profile.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompt_strings.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompt_validators.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/prompts/prompts.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/runner/exit.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/runner/load.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/runner/run.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/__init__.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/actions.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/advanced_processing.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/advanced_program.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/advanced_user_filter.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/automatic.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/content.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/download.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/file.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/logging.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/media_type.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/program.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/user_list.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/user_select.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/arguments/user_sort.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/bundles/advanced_common.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/bundles/common.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/bundles/main.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/bundles/manual.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/bundles/message_check.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/bundles/metadata.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/bundles/paid_check.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/bundles/post_check.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/bundles/story_check.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/commands/main.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/commands/manual.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/commands/message.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/commands/metadata.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/commands/paid.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/commands/post.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/commands/story.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/globals.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/groups/common_args.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/helpers/areas.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/helpers/before.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/helpers/check.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/helpers/date.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/helpers/type.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/output.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/parse.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/quality.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/read.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/user.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/args/write.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/auth/context.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/auth/data.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/auth/file.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/auth/helpers.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/auth/make.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/auth/request.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/auth/schema.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/binaries.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/cache.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/checkers.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/config/config.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/config/context.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/config/custom.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/config/data.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/config/file.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/config/menu.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/config/schema.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/config/wrapper.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/console.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/constants.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/context/exit.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/context/run_async.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/context/stdout.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/dates.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/encoding.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/hash.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/live/empty.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/live/groups.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/live/live.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/live/panel.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/live/progress.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/live/screens.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/live/tasks.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/live/updater.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/logs/close.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/logs/globals.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/logs/handle.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/logs/helpers.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/logs/logger.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/logs/logs.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/logs/other.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/logs/stdout.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/manager.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/me.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/menu.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/merge.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/paths/check.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/paths/common.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/paths/manage.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/paths/paths.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/profiles/data.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/profiles/manage.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/profiles/tools.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/run.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/sems.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/separate.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/settings.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/system/free.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/system/network.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/system/system.py +0 -0
- {ofscraper-3.10.dev6 → ofscraper-3.10.dev7}/ofscraper/utils/text.py +0 -0
|
@@ -27,8 +27,8 @@ log = logging.getLogger("shared")
|
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
@run
|
|
30
|
-
async def
|
|
31
|
-
labels_ = await
|
|
30
|
+
async def get_labels(model_id, c=None):
|
|
31
|
+
labels_ = await get_labels_data(model_id, c=c)
|
|
32
32
|
labels_ = (
|
|
33
33
|
labels_
|
|
34
34
|
if not read_args.retriveArgs().label
|
|
@@ -39,19 +39,18 @@ async def get_labels_progress(model_id, c=None):
|
|
|
39
39
|
)
|
|
40
40
|
)
|
|
41
41
|
)
|
|
42
|
-
return await
|
|
42
|
+
return await get_posts_for_labels(labels_, model_id, c=c)
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
@run
|
|
46
|
-
async def
|
|
46
|
+
async def get_labels_data(model_id, c=None):
|
|
47
47
|
tasks = []
|
|
48
48
|
tasks.append(asyncio.create_task(scrape_labels(c, model_id)))
|
|
49
|
-
progress_utils.labelled_layout.visible = False
|
|
50
49
|
return await process_tasks_labels(tasks)
|
|
51
50
|
|
|
52
51
|
|
|
53
52
|
@run
|
|
54
|
-
async def
|
|
53
|
+
async def get_posts_for_labels(labels, model_id, c=None):
|
|
55
54
|
tasks = []
|
|
56
55
|
|
|
57
56
|
[
|
|
@@ -59,43 +58,11 @@ async def get_posts_for_labels_progress(labels, model_id, c=None):
|
|
|
59
58
|
for label in labels
|
|
60
59
|
]
|
|
61
60
|
labels_final = await process_tasks_get_posts_for_labels(tasks, labels, model_id)
|
|
62
|
-
progress_utils.labelled_layout.visible = False
|
|
63
61
|
return labels_final
|
|
64
62
|
|
|
65
63
|
|
|
66
|
-
@run
|
|
67
|
-
async def get_labels(model_id, c=None):
|
|
68
|
-
labels_ = await get_labels_data(model_id, c=c)
|
|
69
|
-
labels_ = (
|
|
70
|
-
labels_
|
|
71
|
-
if not read_args.retriveArgs().label
|
|
72
|
-
else list(
|
|
73
|
-
filter(
|
|
74
|
-
lambda x: x.get("name").lower() in read_args.retriveArgs().label,
|
|
75
|
-
labels_,
|
|
76
|
-
)
|
|
77
|
-
)
|
|
78
|
-
)
|
|
79
|
-
return await get_posts_for_labels(labels_, model_id, c=c)
|
|
80
64
|
|
|
81
65
|
|
|
82
|
-
@run
|
|
83
|
-
async def get_labels_data(model_id, c=None):
|
|
84
|
-
with progress_utils.set_up_api_labels():
|
|
85
|
-
tasks = []
|
|
86
|
-
tasks.append(asyncio.create_task(scrape_labels(c, model_id)))
|
|
87
|
-
return await process_tasks_labels(tasks)
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
@run
|
|
91
|
-
async def get_posts_for_labels(labels, model_id, c=None):
|
|
92
|
-
with progress_utils.set_up_api_posts_labels():
|
|
93
|
-
tasks = []
|
|
94
|
-
[
|
|
95
|
-
tasks.append(asyncio.create_task(scrape_posts_labels(c, label, model_id)))
|
|
96
|
-
for label in labels
|
|
97
|
-
]
|
|
98
|
-
return await process_tasks_get_posts_for_labels(tasks, labels, model_id)
|
|
99
66
|
|
|
100
67
|
|
|
101
68
|
async def process_tasks_labels(tasks):
|
|
@@ -60,29 +60,29 @@ class SessionSleep:
|
|
|
60
60
|
log = logging.getLogger("shared")
|
|
61
61
|
if not self._sleep:
|
|
62
62
|
self._sleep = self._init_sleep if self._init_sleep else constants.getattr("SESSION_SLEEP_INIT")
|
|
63
|
-
log.debug(f"too many req => setting sleep to init [{self._sleep} seconds]")
|
|
63
|
+
log.debug(f"too many req => setting sleep to init \\[{self._sleep} seconds]")
|
|
64
64
|
elif arrow.now().float_timestamp - self._last_date.float_timestamp < self._difmin:
|
|
65
65
|
log.debug(
|
|
66
|
-
f"too many req => not changing sleep [{self._sleep} seconds] because last call less than {self._difmin} seconds"
|
|
66
|
+
f"too many req => not changing sleep \\[{self._sleep} seconds] because last call less than {self._difmin} seconds"
|
|
67
67
|
)
|
|
68
68
|
return self._sleep
|
|
69
69
|
else:
|
|
70
70
|
self._sleep = self._sleep * 2
|
|
71
|
-
log.debug(f"too many req => setting sleep to [{self._sleep} seconds]")
|
|
71
|
+
log.debug(f"too many req => setting sleep to \\[{self._sleep} seconds]")
|
|
72
72
|
self._last_date = arrow.now()
|
|
73
73
|
return self._sleep
|
|
74
74
|
|
|
75
75
|
async def async_do_sleep(self):
|
|
76
76
|
if self._sleep:
|
|
77
77
|
logging.getLogger("shared").debug(
|
|
78
|
-
f"too many req => waiting [{self._sleep} seconds] before next req"
|
|
78
|
+
f"too many req => waiting \\[{self._sleep} seconds] before next req"
|
|
79
79
|
)
|
|
80
80
|
await asyncio.sleep(self._sleep)
|
|
81
81
|
|
|
82
82
|
def do_sleep(self):
|
|
83
83
|
if self._sleep:
|
|
84
84
|
logging.getLogger("shared").debug(
|
|
85
|
-
f"too many req => waiting [{self._sleep} seconds] before next req"
|
|
85
|
+
f"too many req => waiting \\[{self._sleep} seconds] before next req"
|
|
86
86
|
)
|
|
87
87
|
time.sleep(self._sleep)
|
|
88
88
|
|
|
@@ -35,7 +35,7 @@ import ofscraper.utils.system.network as network
|
|
|
35
35
|
from ofscraper.classes.table.row_names import row_names_all
|
|
36
36
|
from ofscraper.commands.helpers.strings import check_str
|
|
37
37
|
from ofscraper.db.operations_.media import batch_mediainsert, get_media_ids_downloaded
|
|
38
|
-
from ofscraper.download.shared.
|
|
38
|
+
from ofscraper.download.shared.text import textDownloader
|
|
39
39
|
from ofscraper.utils.context.run_async import run
|
|
40
40
|
|
|
41
41
|
log = logging.getLogger("shared")
|
|
@@ -14,8 +14,8 @@ post_str_manual = "Manual Mode: Getting media from {url}"
|
|
|
14
14
|
download_manual_str = "Downloading individual media for {username}"
|
|
15
15
|
check_str = "Checking {activity} for {username}"
|
|
16
16
|
|
|
17
|
-
all_paid_progress_download_str="[{model_id}_{username}] Downloading Progress :{count}/{length} models"
|
|
17
|
+
all_paid_progress_download_str="\\[{model_id}_{username}] Downloading Progress :{count}/{length} models"
|
|
18
18
|
|
|
19
|
-
all_paid_progress_metadata_str="[{model_id}_{username}] Metadata Update Progress :{count}/{length} models"
|
|
19
|
+
all_paid_progress_metadata_str="\\[{model_id}_{username}] Metadata Update Progress :{count}/{length} models"
|
|
20
20
|
|
|
21
21
|
mark_stray_str="Marking Stray Items for {username} as downloaded"
|
|
@@ -161,10 +161,10 @@ def get_final_like_log(like_action,username,failed,post,liked):
|
|
|
161
161
|
|
|
162
162
|
text_out=""
|
|
163
163
|
if post==0:
|
|
164
|
-
text_out=f"[bold][{username}][/bold] [bold][Action {title}][/bold] [{post} post checked), ({liked_log}, {alt_liked_log}), {failed} post failed]"
|
|
164
|
+
text_out=f"[bold]\\[{username}][/bold] [bold][Action {title}][/bold] \\[{post} post checked), ({liked_log}, {alt_liked_log}), {failed} post failed]"
|
|
165
165
|
log.warning(text_out)
|
|
166
166
|
else:
|
|
167
|
-
text_out=f"[blue][bold][{username}][/bold] [bold][Action {title}][/bold] [[yellow]{post} post checked[/yellow], ({liked_log}, {alt_liked_log}), {failed_log}][/blue]"
|
|
167
|
+
text_out=f"[blue][bold]\\[{username}][/bold] [bold][Action {title}][/bold] [[yellow]{post} post checked[/yellow], ({liked_log}, {alt_liked_log}), {failed_log}][/blue]"
|
|
168
168
|
log.warning(text_out)
|
|
169
169
|
return text_out
|
|
170
170
|
|
|
@@ -365,7 +365,7 @@ async def process_all_paid():
|
|
|
365
365
|
@free.space_checker
|
|
366
366
|
async def process_labels(model_id, username, c):
|
|
367
367
|
try:
|
|
368
|
-
labelled_posts_ = await labels_api.
|
|
368
|
+
labelled_posts_ = await labels_api.get_labels(model_id, c=c)
|
|
369
369
|
|
|
370
370
|
labelled_posts_labels = list(
|
|
371
371
|
map(lambda x: labels.Label(x, model_id, username), labelled_posts_)
|
|
@@ -397,7 +397,6 @@ async def process_labels(model_id, username, c):
|
|
|
397
397
|
log.debug(
|
|
398
398
|
f"[bold]Label media count without locked[/bold] {sum(map(lambda x:len(x),unlocked_output))}"
|
|
399
399
|
)
|
|
400
|
-
a
|
|
401
400
|
|
|
402
401
|
return (
|
|
403
402
|
[item for sublist in all_output for item in sublist],
|
|
@@ -25,36 +25,39 @@ try:
|
|
|
25
25
|
except ModuleNotFoundError:
|
|
26
26
|
pass
|
|
27
27
|
import ofscraper.classes.placeholder as placeholder
|
|
28
|
-
import ofscraper.download.shared.
|
|
29
|
-
import ofscraper.download.shared.globals as common_globals
|
|
28
|
+
import ofscraper.download.shared.general as common
|
|
29
|
+
import ofscraper.download.shared.globals.globals as common_globals
|
|
30
30
|
import ofscraper.utils.cache as cache
|
|
31
31
|
import ofscraper.utils.constants as constants
|
|
32
32
|
import ofscraper.utils.live.screens as progress_utils
|
|
33
33
|
from ofscraper.download.shared.classes.retries import download_retry
|
|
34
|
-
from ofscraper.download.shared.
|
|
34
|
+
from ofscraper.download.shared.alt_common import (
|
|
35
35
|
media_item_keys_alt,
|
|
36
36
|
media_item_post_process_alt,
|
|
37
37
|
)
|
|
38
38
|
|
|
39
|
-
from ofscraper.download.shared.
|
|
39
|
+
from ofscraper.download.shared.handle_result import (
|
|
40
40
|
handle_result_alt,
|
|
41
41
|
)
|
|
42
|
-
from ofscraper.download.shared.
|
|
42
|
+
from ofscraper.download.shared.general import (
|
|
43
43
|
check_forced_skip,
|
|
44
44
|
downloadspace,
|
|
45
|
-
get_ideal_chunk_size,
|
|
46
45
|
get_medialog,
|
|
47
46
|
get_resume_size,
|
|
48
|
-
get_update_count,
|
|
49
47
|
size_checker,
|
|
50
48
|
)
|
|
51
|
-
from ofscraper.download.shared.
|
|
49
|
+
from ofscraper.download.shared.log import (
|
|
52
50
|
get_url_log,
|
|
53
51
|
path_to_file_logger,
|
|
54
52
|
temp_file_logger,
|
|
55
53
|
)
|
|
56
54
|
|
|
57
|
-
from ofscraper.download.shared.
|
|
55
|
+
from ofscraper.download.shared.progress.chunk import (
|
|
56
|
+
get_ideal_chunk_size,
|
|
57
|
+
get_update_count,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
from ofscraper.download.shared.send.send_bar_msg import (
|
|
58
61
|
send_bar_msg
|
|
59
62
|
)
|
|
60
63
|
|
|
@@ -11,36 +11,38 @@ try:
|
|
|
11
11
|
except ModuleNotFoundError:
|
|
12
12
|
pass
|
|
13
13
|
import ofscraper.classes.placeholder as placeholder
|
|
14
|
-
import ofscraper.download.shared.
|
|
15
|
-
import ofscraper.download.shared.globals as common_globals
|
|
14
|
+
import ofscraper.download.shared.general as common
|
|
15
|
+
import ofscraper.download.shared.globals.globals as common_globals
|
|
16
16
|
import ofscraper.utils.cache as cache
|
|
17
17
|
import ofscraper.utils.constants as constants
|
|
18
18
|
import ofscraper.utils.live.screens as progress_utils
|
|
19
19
|
import ofscraper.utils.system.system as system
|
|
20
20
|
from ofscraper.download.shared.classes.retries import download_retry
|
|
21
|
-
from ofscraper.download.shared.
|
|
21
|
+
from ofscraper.download.shared.alt_common import (
|
|
22
22
|
media_item_keys_alt,
|
|
23
23
|
media_item_post_process_alt,
|
|
24
24
|
)
|
|
25
|
-
from ofscraper.download.shared.
|
|
25
|
+
from ofscraper.download.shared.handle_result import (
|
|
26
26
|
handle_result_alt,
|
|
27
27
|
)
|
|
28
|
-
from ofscraper.download.shared.
|
|
28
|
+
from ofscraper.download.shared.general import (
|
|
29
29
|
check_forced_skip,
|
|
30
30
|
downloadspace,
|
|
31
|
-
get_ideal_chunk_size,
|
|
32
31
|
get_medialog,
|
|
33
32
|
get_resume_size,
|
|
34
|
-
get_update_count,
|
|
35
33
|
size_checker,
|
|
36
34
|
)
|
|
37
|
-
from ofscraper.download.shared.
|
|
35
|
+
from ofscraper.download.shared.log import (
|
|
38
36
|
get_url_log,
|
|
39
37
|
path_to_file_logger,
|
|
40
38
|
temp_file_logger,
|
|
41
39
|
)
|
|
42
40
|
|
|
43
|
-
from ofscraper.download.shared.
|
|
41
|
+
from ofscraper.download.shared.progress.chunk import (
|
|
42
|
+
get_ideal_chunk_size,
|
|
43
|
+
get_update_count,
|
|
44
|
+
)
|
|
45
|
+
from ofscraper.download.shared.send.send_bar_msg import (
|
|
44
46
|
send_bar_msg_batch
|
|
45
47
|
)
|
|
46
48
|
|
|
@@ -251,7 +253,6 @@ async def download_fileobject_writer(total, l, ele, placeholderObj):
|
|
|
251
253
|
)
|
|
252
254
|
await fileobject.write(chunk)
|
|
253
255
|
await send_bar_msg_batch(
|
|
254
|
-
{
|
|
255
256
|
partial(
|
|
256
257
|
progress_utils.update_download_multi_job_task,
|
|
257
258
|
ele.id,
|
|
@@ -260,7 +261,6 @@ async def download_fileobject_writer(total, l, ele, placeholderObj):
|
|
|
260
261
|
.stat()
|
|
261
262
|
.st_size,
|
|
262
263
|
),count,update_count
|
|
263
|
-
}
|
|
264
264
|
)
|
|
265
265
|
count += 1
|
|
266
266
|
(await asyncio.sleep(download_sleep)) if download_sleep else None
|
|
@@ -11,9 +11,9 @@ import ofscraper.utils.constants as constants
|
|
|
11
11
|
import ofscraper.utils.hash as hash
|
|
12
12
|
import ofscraper.utils.settings as settings
|
|
13
13
|
import ofscraper.utils.system.system as system
|
|
14
|
-
from ofscraper.download.shared.
|
|
14
|
+
from ofscraper.download.shared.text import textDownloader
|
|
15
15
|
from ofscraper.utils.context.run_async import run
|
|
16
|
-
from ofscraper.download.shared.
|
|
16
|
+
from ofscraper.download.shared.log import empty_log
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
@run
|
|
@@ -14,8 +14,8 @@ import more_itertools
|
|
|
14
14
|
import psutil
|
|
15
15
|
from aioprocessing import AioPipe
|
|
16
16
|
|
|
17
|
-
import ofscraper.download.shared.
|
|
18
|
-
import ofscraper.download.shared.globals as common_globals
|
|
17
|
+
import ofscraper.download.shared.general as common
|
|
18
|
+
import ofscraper.download.shared.globals.globals as common_globals
|
|
19
19
|
import ofscraper.models.selector as selector
|
|
20
20
|
import ofscraper.utils.args.read as read_args
|
|
21
21
|
import ofscraper.utils.cache as cache
|
|
@@ -32,14 +32,14 @@ import ofscraper.utils.system.system as system
|
|
|
32
32
|
from ofscraper.download.alt_downloadbatch import alt_download
|
|
33
33
|
from ofscraper.download.main_downloadbatch import main_download
|
|
34
34
|
from ofscraper.download.shared.classes.session import download_session
|
|
35
|
-
from ofscraper.download.shared.
|
|
35
|
+
from ofscraper.download.shared.general import (
|
|
36
36
|
get_medialog,
|
|
37
37
|
subProcessVariableInit,
|
|
38
38
|
)
|
|
39
|
-
from ofscraper.download.shared.
|
|
40
|
-
from ofscraper.download.shared.
|
|
41
|
-
from ofscraper.download.shared.
|
|
42
|
-
from ofscraper.download.shared.
|
|
39
|
+
from ofscraper.download.shared.log import final_log, final_log_text ,log_download_progress
|
|
40
|
+
from ofscraper.download.shared.metadata import metadata
|
|
41
|
+
from ofscraper.download.shared.paths.paths import addGlobalDir, setDirectoriesDate
|
|
42
|
+
from ofscraper.download.shared.progress.progress import convert_num_bytes
|
|
43
43
|
from ofscraper.utils.context.run_async import run
|
|
44
44
|
|
|
45
45
|
platform_name = platform.system()
|
|
@@ -16,7 +16,7 @@ import logging
|
|
|
16
16
|
import traceback
|
|
17
17
|
from functools import partial
|
|
18
18
|
|
|
19
|
-
import ofscraper.download.shared.globals as common_globals
|
|
19
|
+
import ofscraper.download.shared.globals.globals as common_globals
|
|
20
20
|
import ofscraper.utils.args.read as read_args
|
|
21
21
|
import ofscraper.utils.cache as cache
|
|
22
22
|
import ofscraper.utils.context.exit as exit
|
|
@@ -28,11 +28,11 @@ import ofscraper.utils.manager as manager_
|
|
|
28
28
|
from ofscraper.download.alt_download import alt_download
|
|
29
29
|
from ofscraper.download.main_download import main_download
|
|
30
30
|
from ofscraper.download.shared.classes.session import download_session
|
|
31
|
-
from ofscraper.download.shared.
|
|
32
|
-
from ofscraper.download.shared.
|
|
33
|
-
from ofscraper.download.shared.
|
|
34
|
-
from ofscraper.download.shared.
|
|
35
|
-
from ofscraper.download.shared.
|
|
31
|
+
from ofscraper.download.shared.general import get_medialog
|
|
32
|
+
from ofscraper.download.shared.log import final_log, final_log_text,log_download_progress
|
|
33
|
+
from ofscraper.download.shared.metadata import metadata
|
|
34
|
+
from ofscraper.download.shared.paths.paths import setDirectoriesDate
|
|
35
|
+
from ofscraper.download.shared.progress.progress import convert_num_bytes
|
|
36
36
|
from ofscraper.utils.context.run_async import run
|
|
37
37
|
|
|
38
38
|
|
|
@@ -24,29 +24,31 @@ try:
|
|
|
24
24
|
except ModuleNotFoundError:
|
|
25
25
|
pass
|
|
26
26
|
import ofscraper.classes.placeholder as placeholder
|
|
27
|
-
import ofscraper.download.shared.
|
|
28
|
-
import ofscraper.download.shared.globals as common_globals
|
|
27
|
+
import ofscraper.download.shared.general as common
|
|
28
|
+
import ofscraper.download.shared.globals.globals as common_globals
|
|
29
29
|
import ofscraper.utils.cache as cache
|
|
30
30
|
import ofscraper.utils.constants as constants
|
|
31
31
|
import ofscraper.utils.live.screens as progress_utils
|
|
32
32
|
from ofscraper.download.shared.classes.retries import download_retry
|
|
33
|
-
from ofscraper.download.shared.
|
|
33
|
+
from ofscraper.download.shared.general import (
|
|
34
34
|
check_forced_skip,
|
|
35
35
|
downloadspace,
|
|
36
36
|
get_data,
|
|
37
|
-
get_ideal_chunk_size,
|
|
38
37
|
get_medialog,
|
|
39
38
|
get_resume_size,
|
|
40
39
|
get_unknown_content_type,
|
|
41
|
-
get_update_count,
|
|
42
40
|
size_checker,
|
|
43
41
|
)
|
|
44
|
-
from ofscraper.download.shared.
|
|
45
|
-
from ofscraper.download.shared.
|
|
46
|
-
from ofscraper.download.shared.
|
|
47
|
-
from ofscraper.download.shared.
|
|
42
|
+
from ofscraper.download.shared.handle_result import handle_result_main
|
|
43
|
+
from ofscraper.download.shared.log import get_url_log, path_to_file_logger
|
|
44
|
+
from ofscraper.download.shared.metadata import force_download
|
|
45
|
+
from ofscraper.download.shared.send.send_bar_msg import (
|
|
48
46
|
send_bar_msg
|
|
49
47
|
)
|
|
48
|
+
from ofscraper.download.shared.progress.chunk import (
|
|
49
|
+
get_ideal_chunk_size,
|
|
50
|
+
get_update_count,
|
|
51
|
+
)
|
|
50
52
|
|
|
51
53
|
|
|
52
54
|
async def main_download(c, ele, username, model_id):
|
|
@@ -23,30 +23,32 @@ try:
|
|
|
23
23
|
except ModuleNotFoundError:
|
|
24
24
|
pass
|
|
25
25
|
import ofscraper.classes.placeholder as placeholder
|
|
26
|
-
import ofscraper.download.shared.
|
|
27
|
-
import ofscraper.download.shared.globals as common_globals
|
|
26
|
+
import ofscraper.download.shared.general as common
|
|
27
|
+
import ofscraper.download.shared.globals.globals as common_globals
|
|
28
28
|
import ofscraper.utils.cache as cache
|
|
29
29
|
import ofscraper.utils.constants as constants
|
|
30
30
|
import ofscraper.utils.live.screens as progress_utils
|
|
31
31
|
import ofscraper.utils.settings as settings
|
|
32
32
|
import ofscraper.utils.system.system as system
|
|
33
33
|
from ofscraper.download.shared.classes.retries import download_retry
|
|
34
|
-
from ofscraper.download.shared.
|
|
34
|
+
from ofscraper.download.shared.general import (
|
|
35
35
|
check_forced_skip,
|
|
36
36
|
downloadspace,
|
|
37
37
|
get_data,
|
|
38
|
-
get_ideal_chunk_size,
|
|
39
38
|
get_medialog,
|
|
40
39
|
get_resume_size,
|
|
41
40
|
get_unknown_content_type,
|
|
42
|
-
get_update_count,
|
|
43
41
|
size_checker,
|
|
44
42
|
)
|
|
45
|
-
from ofscraper.download.shared.
|
|
46
|
-
|
|
47
|
-
|
|
43
|
+
from ofscraper.download.shared.progress.chunk import (
|
|
44
|
+
get_ideal_chunk_size,
|
|
45
|
+
get_update_count,
|
|
46
|
+
)
|
|
47
|
+
from ofscraper.download.shared.handle_result import handle_result_main
|
|
48
|
+
from ofscraper.download.shared.log import get_url_log, path_to_file_logger
|
|
49
|
+
from ofscraper.download.shared.metadata import force_download
|
|
48
50
|
|
|
49
|
-
from ofscraper.download.shared.
|
|
51
|
+
from ofscraper.download.shared.send.send_bar_msg import (
|
|
50
52
|
send_bar_msg_batch
|
|
51
53
|
)
|
|
52
54
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import ofscraper.download.shared.
|
|
2
|
-
import ofscraper.download.shared.
|
|
3
|
-
from ofscraper.download.shared.
|
|
1
|
+
import ofscraper.download.shared.general as common
|
|
2
|
+
import ofscraper.download.shared.keyhelpers as keyhelpers
|
|
3
|
+
from ofscraper.download.shared.metadata import force_download
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import ofscraper.classes.sessionmanager as sessionManager
|
|
2
|
-
import ofscraper.download.shared.globals as common_globals
|
|
2
|
+
import ofscraper.download.shared.globals.globals as common_globals
|
|
3
3
|
import ofscraper.utils.constants as constants
|
|
4
4
|
from ofscraper.download.shared.classes.retries import get_download_req_retries
|
|
5
5
|
|
|
@@ -16,10 +16,9 @@ import pathlib
|
|
|
16
16
|
import re
|
|
17
17
|
from functools import partial
|
|
18
18
|
|
|
19
|
-
import psutil
|
|
20
19
|
from humanfriendly import format_size
|
|
21
20
|
|
|
22
|
-
import ofscraper.download.shared.globals as common_globals
|
|
21
|
+
import ofscraper.download.shared.globals.globals as common_globals
|
|
23
22
|
import ofscraper.models.selector as selector
|
|
24
23
|
import ofscraper.utils.args.write as write_args
|
|
25
24
|
import ofscraper.utils.cache as cache
|
|
@@ -29,10 +28,10 @@ import ofscraper.utils.dates as dates
|
|
|
29
28
|
import ofscraper.utils.hash as hash
|
|
30
29
|
import ofscraper.utils.settings as settings
|
|
31
30
|
import ofscraper.utils.system.free as system
|
|
32
|
-
from ofscraper.download.shared.
|
|
33
|
-
from ofscraper.download.shared.
|
|
34
|
-
from ofscraper.download.shared.
|
|
35
|
-
from ofscraper.download.shared.
|
|
31
|
+
from ofscraper.download.shared.log import get_medialog
|
|
32
|
+
from ofscraper.download.shared.paths.media import add_path
|
|
33
|
+
from ofscraper.download.shared.send.message import send_msg, set_send_msg
|
|
34
|
+
from ofscraper.download.shared.progress.progress import update_total
|
|
36
35
|
|
|
37
36
|
|
|
38
37
|
def add_additional_data(placeholderObj, ele):
|
|
@@ -178,48 +177,4 @@ def is_bad_url(url):
|
|
|
178
177
|
return True
|
|
179
178
|
|
|
180
179
|
|
|
181
|
-
def get_ideal_chunk_size(total_size, curr_file):
|
|
182
|
-
"""
|
|
183
|
-
Suggests a chunk size based on file size and a calculated available memory buffer.
|
|
184
|
-
|
|
185
|
-
Args:
|
|
186
|
-
file_size (int): Size of the file being downloaded in bytes.
|
|
187
|
-
|
|
188
|
-
Returns:
|
|
189
|
-
int: Suggested chunk size in bytes.
|
|
190
|
-
"""
|
|
191
|
-
curr_file_size = (
|
|
192
|
-
pathlib.Path(curr_file).absolute().stat().st_size
|
|
193
|
-
if pathlib.Path(curr_file).exists()
|
|
194
|
-
else 0
|
|
195
|
-
)
|
|
196
|
-
file_size = total_size - curr_file_size
|
|
197
|
-
|
|
198
|
-
# Estimate available memory (considering a buffer for system operations)
|
|
199
|
-
available_memory = (
|
|
200
|
-
psutil.virtual_memory().available - 1024 * 1024 * 512
|
|
201
|
-
) # Reserve 512MB buffer
|
|
202
|
-
|
|
203
|
-
# Target a chunk size that utilizes a reasonable portion of available memory
|
|
204
|
-
max_chunk_size = min(
|
|
205
|
-
available_memory // 512, constants.getattr("MAX_CHUNK_SIZE")
|
|
206
|
-
) # Max 10MB
|
|
207
|
-
# Adjust chunk size based on file size (consider smaller sizes for larger files, with minimum)
|
|
208
|
-
ideal_chunk_size = min(max_chunk_size, file_size // 512)
|
|
209
|
-
ideal_chunk_size = max(
|
|
210
|
-
ideal_chunk_size - (ideal_chunk_size % 4096),
|
|
211
|
-
constants.getattr("MIN_CHUNK_SIZE"),
|
|
212
|
-
) # Minimum 4KB chunk
|
|
213
|
-
return ideal_chunk_size
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
def get_update_count(total_size, curr_file, chunk_size):
|
|
217
|
-
curr_file_size = (
|
|
218
|
-
pathlib.Path(curr_file).absolute().stat().st_size
|
|
219
|
-
if pathlib.Path(curr_file).exists()
|
|
220
|
-
else 0
|
|
221
|
-
)
|
|
222
|
-
file_size = total_size - curr_file_size
|
|
223
|
-
|
|
224
|
-
return max((file_size // chunk_size) // constants.getattr("CHUNK_UPDATE_COUNT"), 1)
|
|
225
180
|
|
|
@@ -3,10 +3,10 @@ import subprocess
|
|
|
3
3
|
|
|
4
4
|
import arrow
|
|
5
5
|
|
|
6
|
-
import ofscraper.download.shared.
|
|
7
|
-
import ofscraper.download.shared.globals as common_globals
|
|
8
|
-
import ofscraper.download.shared.
|
|
9
|
-
import ofscraper.download.shared.
|
|
6
|
+
import ofscraper.download.shared.general as common
|
|
7
|
+
import ofscraper.download.shared.globals.globals as common_globals
|
|
8
|
+
import ofscraper.download.shared.log as common_logs
|
|
9
|
+
import ofscraper.download.shared.paths.paths as common_paths
|
|
10
10
|
import ofscraper.utils.dates as dates
|
|
11
11
|
import ofscraper.utils.system.system as system
|
|
12
12
|
import ofscraper.classes.placeholder as placeholder
|
|
@@ -11,7 +11,7 @@ from pywidevine.cdm import Cdm
|
|
|
11
11
|
from pywidevine.device import Device
|
|
12
12
|
from pywidevine.pssh import PSSH
|
|
13
13
|
|
|
14
|
-
import ofscraper.download.shared.globals as common_globals
|
|
14
|
+
import ofscraper.download.shared.globals.globals as common_globals
|
|
15
15
|
import ofscraper.utils.auth.request as auth_requests
|
|
16
16
|
import ofscraper.utils.cache as cache
|
|
17
17
|
import ofscraper.utils.config.data as config_data
|
|
@@ -19,7 +19,7 @@ import ofscraper.utils.constants as constants
|
|
|
19
19
|
import ofscraper.utils.settings as settings
|
|
20
20
|
from ofscraper.download.shared.classes.retries import get_cmd_download_req_retries
|
|
21
21
|
from ofscraper.download.shared.classes.session import cdm_session
|
|
22
|
-
from ofscraper.download.shared.
|
|
22
|
+
from ofscraper.download.shared.general import get_medialog
|
|
23
23
|
|
|
24
24
|
log = None
|
|
25
25
|
|
|
@@ -2,7 +2,7 @@ import re
|
|
|
2
2
|
|
|
3
3
|
from humanfriendly import format_size
|
|
4
4
|
|
|
5
|
-
import ofscraper.download.shared.globals as common_globals
|
|
5
|
+
import ofscraper.download.shared.globals.globals as common_globals
|
|
6
6
|
import ofscraper.utils.args.read as read_args
|
|
7
7
|
import ofscraper.utils.constants as constants
|
|
8
8
|
|
|
@@ -14,20 +14,20 @@ def get_medialog(ele):
|
|
|
14
14
|
def path_to_file_logger(placeholderObj, ele, innerlog=None):
|
|
15
15
|
innerlog = innerlog or common_globals.log
|
|
16
16
|
innerlog.debug(
|
|
17
|
-
f"{get_medialog(ele)} [attempt {common_globals.attempt.get()}/{constants.getattr('API_NUM_TRIES')}] filename from config {placeholderObj.filename}"
|
|
17
|
+
f"{get_medialog(ele)} \\[attempt {common_globals.attempt.get()}/{constants.getattr('API_NUM_TRIES')}] filename from config {placeholderObj.filename}"
|
|
18
18
|
)
|
|
19
19
|
innerlog.debug(
|
|
20
|
-
f"{get_medialog(ele)} [attempt {common_globals.attempt.get()}/{constants.getattr('API_NUM_TRIES')}] full path from config {placeholderObj.filepath}"
|
|
20
|
+
f"{get_medialog(ele)} \\[attempt {common_globals.attempt.get()}/{constants.getattr('API_NUM_TRIES')}] full path from config {placeholderObj.filepath}"
|
|
21
21
|
)
|
|
22
22
|
innerlog.debug(
|
|
23
|
-
f"{get_medialog(ele)} [attempt {common_globals.attempt.get()}/{constants.getattr('API_NUM_TRIES')}] full path trunicated from config {placeholderObj.trunicated_filepath}"
|
|
23
|
+
f"{get_medialog(ele)} \\[attempt {common_globals.attempt.get()}/{constants.getattr('API_NUM_TRIES')}] full path trunicated from config {placeholderObj.trunicated_filepath}"
|
|
24
24
|
)
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
def temp_file_logger(placeholderObj, ele, innerlog=None):
|
|
28
28
|
innerlog = innerlog or common_globals.log
|
|
29
29
|
innerlog.debug(
|
|
30
|
-
f"{get_medialog(ele)} [attempt {common_globals.attempt.get()}/{constants.getattr('API_NUM_TRIES')}] filename from config {placeholderObj.tempfilepath}"
|
|
30
|
+
f"{get_medialog(ele)} \\[attempt {common_globals.attempt.get()}/{constants.getattr('API_NUM_TRIES')}] filename from config {placeholderObj.tempfilepath}"
|
|
31
31
|
)
|
|
32
32
|
|
|
33
33
|
|
|
@@ -51,7 +51,7 @@ def log_download_progress(media_type):
|
|
|
51
51
|
) % 20 == 0:
|
|
52
52
|
common_globals.log.debug(
|
|
53
53
|
f"In progress -> {format_size(common_globals.total_bytes )}) ({common_globals.photo_count+common_globals.audio_count+common_globals.video_count} \
|
|
54
|
-
downloads total [{common_globals.video_count} videos, {common_globals.audio_count} audios, {common_globals.photo_count} photos], \
|
|
54
|
+
downloads total \\[{common_globals.video_count} videos, {common_globals.audio_count} audios, {common_globals.photo_count} photos], \
|
|
55
55
|
{common_globals.forced_skipped} skipped, {common_globals.skipped} failed)"
|
|
56
56
|
)
|
|
57
57
|
|
|
@@ -62,7 +62,7 @@ def final_log(username, log=None):
|
|
|
62
62
|
)
|
|
63
63
|
|
|
64
64
|
def final_log_text(username):
|
|
65
|
-
total_count=common_globals.audio_count+common_globals.
|
|
65
|
+
total_count=common_globals.audio_count+common_globals.photo_count+common_globals.video_count
|
|
66
66
|
size_log=f"[green]{format_size(common_globals.total_bytes )}[/green]"
|
|
67
67
|
|
|
68
68
|
|
|
@@ -79,10 +79,10 @@ def final_log_text(username):
|
|
|
79
79
|
log_format=None
|
|
80
80
|
skipped_log=""
|
|
81
81
|
if read_args.retriveArgs().metadata:
|
|
82
|
-
log_format="[blue][bold][{username}][/bold] [bold][Action Metadata][/bold] ({size_log}) ([green]{total_count} changed media item total[/green][{video_log}, {audio_log}, {photo_log}], {skipped_log}, {failed_log}))[/blue]"
|
|
82
|
+
log_format="[blue][bold]\\[{username}][/bold] [bold]\\[Action Metadata][/bold] ({size_log}) ([green]{total_count} changed media item total[/green]\\[{video_log}, {audio_log}, {photo_log}], {skipped_log}, {failed_log}))[/blue]"
|
|
83
83
|
skipped_log=f"[yellow]{common_globals.skipped} metadata unchanged[/yellow]"if common_globals.skipped>0 else f"{common_globals.forced_skipped} items unchanged"
|
|
84
84
|
else:
|
|
85
|
-
log_format="[blue][bold]\\[{username}][/bold][bold][Action Download][/bold] ({size_log}) ([green]{total_count} downloads total[/green][{video_log}, {audio_log}, {photo_log}], {skipped_log}, {failed_log}))[/blue]"
|
|
85
|
+
log_format="[blue][bold]\\[{username}][/bold][bold]\\[Action Download][/bold] ({size_log}) ([green]{total_count} downloads total[/green]\\[{video_log}, {audio_log}, {photo_log}], {skipped_log}, {failed_log}))[/blue]"
|
|
86
86
|
skipped_log=f"[yellow]{common_globals.video_count} skipped[/yellow]"if common_globals.skipped>0 else f"{common_globals.forced_skipped} skipped"
|
|
87
87
|
|
|
88
88
|
return log_format.format(username=username,total_count=total_count,video_log=video_log,audio_log=audio_log,skipped_log=skipped_log,failed_log=failed_log,photo_log=photo_log,size_log=size_log)
|
|
@@ -92,9 +92,9 @@ def final_log_text(username):
|
|
|
92
92
|
def empty_log(username):
|
|
93
93
|
|
|
94
94
|
if read_args.retriveArgs().metadata:
|
|
95
|
-
return f"[white][bold][{username}][/bold] [bold][Action Metadata][/bold] ({0} MB) ({0} changed media items total [{0} videos, {0} audios, {0} photos], {0} items unchanged, {0} failed))[/white]"
|
|
95
|
+
return f"[white][bold]\\[{username}][/bold] [bold]\\[Action Metadata][/bold] ({0} MB) ({0} changed media items total \\[{0} videos, {0} audios, {0} photos], {0} items unchanged, {0} failed))[/white]"
|
|
96
96
|
else:
|
|
97
|
-
return f"[white][bold][{username}][/bold] [bold][Action Download][/bold] ({0} MB) ({0} downloads total [{0} videos, {0} audios, {0} photos], {0} skipped, {0} failed))[/white]"
|
|
97
|
+
return f"[white][bold]\\[{username}][/bold] [bold]\\[Action Download][/bold] ({0} MB) ({0} downloads total \\[{0} videos, {0} audios, {0} photos], {0} skipped, {0} failed))[/white]"
|
|
98
98
|
|
|
99
99
|
|
|
100
100
|
|