ofscraper 3.7.0.dev11__tar.gz → 3.8.0.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/PKG-INFO +1 -1
  2. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/actions/process.py +8 -37
  3. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/actions/scraper.py +1 -1
  4. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/archive.py +21 -23
  5. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/messages.py +19 -4
  6. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/paid.py +14 -6
  7. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/timeline.py +20 -5
  8. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/media.py +53 -9
  9. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/placeholder.py +6 -4
  10. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/commands/scraper.py +1 -1
  11. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/other_url.py +1 -1
  12. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/common.py +5 -8
  13. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/media/helpers.py +4 -0
  14. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/media/main.py +19 -1
  15. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/models/retriver.py +12 -1
  16. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/models/selector.py +5 -15
  17. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/parse.py +14 -6
  18. ofscraper-3.8.0.dev0/ofscraper/utils/args/quality.py +13 -0
  19. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/read.py +4 -2
  20. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/pyproject.toml +1 -1
  21. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/LICENSE +0 -0
  22. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/README.md +0 -0
  23. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/__main__.py +0 -0
  24. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/__version__.py +0 -0
  25. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/__version__.pye +0 -0
  26. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/actions/__init__.py +0 -0
  27. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/actions/like.py +0 -0
  28. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/highlights.py +0 -0
  29. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/init.py +0 -0
  30. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/labels.py +0 -0
  31. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/me.py +0 -0
  32. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/pinned.py +0 -0
  33. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/profile.py +0 -0
  34. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/subscriptions/helpers.py +0 -0
  35. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/subscriptions/individual.py +0 -0
  36. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/subscriptions/lists.py +0 -0
  37. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/api/subscriptions/subscriptions.py +0 -0
  38. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/labels.py +0 -0
  39. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/models.py +0 -0
  40. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/multiprocessprogress.py +0 -0
  41. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/posts.py +0 -0
  42. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/semaphoreDelayed.py +0 -0
  43. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/sessionbuilder.py +0 -0
  44. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/classes/table.py +0 -0
  45. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/commands/check.py +0 -0
  46. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/commands/manual.py +0 -0
  47. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/commands/picker.py +0 -0
  48. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/config/paths.py +0 -0
  49. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/config.py +0 -0
  50. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/constants.py +0 -0
  51. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/download.py +0 -0
  52. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/files.py +0 -0
  53. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/general.py +0 -0
  54. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/logger.py +0 -0
  55. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/prompts.py +0 -0
  56. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/req.py +0 -0
  57. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/test_constants.py +0 -0
  58. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/time.py +0 -0
  59. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/const/url.py +0 -0
  60. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/db/__init__.py +0 -0
  61. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/db/operations.py +0 -0
  62. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/db/queries.py +0 -0
  63. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/alt_download.py +0 -0
  64. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/alt_downloadbatch.py +0 -0
  65. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/download.py +0 -0
  66. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/downloadbatch.py +0 -0
  67. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/downloadnormal.py +0 -0
  68. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/keyhelpers.py +0 -0
  69. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/main_download.py +0 -0
  70. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/download/main_downloadbatch.py +0 -0
  71. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/date.py +0 -0
  72. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/flags.py +0 -0
  73. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/other.py +0 -0
  74. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/price.py +0 -0
  75. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/sort.py +0 -0
  76. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/filters/models/subtype.py +0 -0
  77. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/keybindings.py +0 -0
  78. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/model_helpers.py +0 -0
  79. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/promptConvert.py +0 -0
  80. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/actions.py +0 -0
  81. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/area.py +0 -0
  82. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/auth.py +0 -0
  83. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/binary.py +0 -0
  84. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/config.py +0 -0
  85. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/menu.py +0 -0
  86. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/model.py +0 -0
  87. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_groups/profile.py +0 -0
  88. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_strings.py +0 -0
  89. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompt_validators.py +0 -0
  90. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/prompts/prompts.py +0 -0
  91. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/runner/exit.py +0 -0
  92. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/runner/load.py +0 -0
  93. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/runner/run.py +0 -0
  94. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/__init__.py +0 -0
  95. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/actions.py +0 -0
  96. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/areas.py +0 -0
  97. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/globals.py +0 -0
  98. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/helpers.py +0 -0
  99. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/args/write.py +0 -0
  100. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/auth.py +0 -0
  101. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/binaries.py +0 -0
  102. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/cache.py +0 -0
  103. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/checkers.py +0 -0
  104. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/config.py +0 -0
  105. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/context.py +0 -0
  106. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/custom.py +0 -0
  107. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/data.py +0 -0
  108. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/file.py +0 -0
  109. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/menu.py +0 -0
  110. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/schema.py +0 -0
  111. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/config/wrapper.py +0 -0
  112. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/console.py +0 -0
  113. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/constants.py +0 -0
  114. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/context/exit.py +0 -0
  115. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/context/run_async.py +0 -0
  116. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/context/stdout.py +0 -0
  117. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/dates.py +0 -0
  118. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/encoding.py +0 -0
  119. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/classes.py +0 -0
  120. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/close.py +0 -0
  121. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/globals.py +0 -0
  122. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/helpers.py +0 -0
  123. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/logger.py +0 -0
  124. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/logs.py +0 -0
  125. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/other.py +0 -0
  126. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/logs/stdout.py +0 -0
  127. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/manager.py +0 -0
  128. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/me.py +0 -0
  129. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/menu.py +0 -0
  130. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/paths/check.py +0 -0
  131. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/paths/common.py +0 -0
  132. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/paths/manage.py +0 -0
  133. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/paths/paths.py +0 -0
  134. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/profiles/data.py +0 -0
  135. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/profiles/manage.py +0 -0
  136. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/profiles/tools.py +0 -0
  137. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/run.py +0 -0
  138. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/separate.py +0 -0
  139. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/system/free.py +0 -0
  140. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/system/network.py +0 -0
  141. {ofscraper-3.7.0.dev11 → ofscraper-3.8.0.dev0}/ofscraper/utils/system/system.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ofscraper
3
- Version: 3.7.0.dev11
3
+ Version: 3.8.0.dev0
4
4
  Summary: automatically scrape onlyfans
5
5
  Author: datawhores
6
6
  Author-email: datawhores@riseup.net
@@ -23,11 +23,13 @@ import ofscraper.actions.like as like
23
23
  import ofscraper.actions.scraper as OF
24
24
  import ofscraper.api.init as init
25
25
  import ofscraper.api.profile as profile
26
+ import ofscraper.classes.models as models
26
27
  import ofscraper.classes.placeholder as placeholder
27
28
  import ofscraper.db.operations as operations
28
29
  import ofscraper.download.download as download
29
30
  import ofscraper.filters.media.main as filters
30
31
  import ofscraper.models.selector as userselector
32
+ import ofscraper.models.selector as selector
31
33
  import ofscraper.utils.actions as actions
32
34
  import ofscraper.utils.args.areas as areas
33
35
  import ofscraper.utils.args.read as read_args
@@ -119,10 +121,13 @@ def scrape_paid(user_dict=None):
119
121
  )
120
122
  for ele in output
121
123
  ]
124
+ oldUsers = selector.get_ALL_SUBS_DICT()
122
125
  for value in user_dict.values():
123
126
  model_id = value[0].post.model_id
124
127
  username = value[0].post.username
125
-
128
+ selector.set_ALL_SUBS_DICTVManger(
129
+ {username: models.Model(profile.scrape_profile(model_id))}
130
+ )
126
131
  operations.create_tables(model_id=model_id, username=username)
127
132
  operations.create_backup(model_id, username)
128
133
  operations.write_profile_table(model_id=model_id, username=username)
@@ -131,6 +136,8 @@ def scrape_paid(user_dict=None):
131
136
  model_id,
132
137
  value,
133
138
  )
139
+ # restore og users
140
+ selector.set_ALL_SUBS_DICT(oldUsers)
134
141
 
135
142
 
136
143
  @exit.exit_wrapper
@@ -169,42 +176,6 @@ def normal_post_process():
169
176
  log.traceback_(f"failed with exception: {e}")
170
177
  log.traceback_(traceback.format_exc())
171
178
 
172
- if read_args.retriveArgs().scrape_paid:
173
- user_dict = {}
174
- [
175
- user_dict.update(
176
- {ele.post.model_id: user_dict.get(ele.post.model_id, []) + [ele]}
177
- )
178
- for ele in OF.process_all_paid()
179
- ]
180
- for value in user_dict.values():
181
- try:
182
- model_id = value[0].post.model_id
183
- username = value[0].post.username
184
- log.info(
185
- f"inserting {len(value)} items into into media table for {username}"
186
- )
187
- operations.batch_mediainsert(
188
- value,
189
- operations.write_media_table_batch,
190
- model_id=model_id,
191
- username=username,
192
- downloaded=False,
193
- )
194
- operations.create_tables(model_id=model_id, username=username)
195
- operations.create_backup(model_id, username)
196
- operations.write_profile_table(model_id=model_id, username=username)
197
- download.download_picker(
198
- username,
199
- model_id,
200
- value,
201
- )
202
- except Exception as E:
203
- if isinstance(e, KeyboardInterrupt):
204
- raise E
205
- log.traceback_(f"failed with exception: {E}")
206
- log.traceback_(traceback.format_exc())
207
-
208
179
 
209
180
  @exit.exit_wrapper
210
181
  def process_like():
@@ -406,7 +406,7 @@ def process_all_paid():
406
406
  f"[bold]Paid Media for all models[/bold] {sum(map(lambda x:len(x.media),post_array))}"
407
407
  )
408
408
 
409
- return output
409
+ return filters.filterMedia(output)
410
410
 
411
411
 
412
412
  @free.space_checker
@@ -183,11 +183,6 @@ async def get_archived_media(model_id, username, forced_after=None, rescan=None)
183
183
  min_posts = 50
184
184
  responseArray = []
185
185
  page_count = 0
186
- setCache = (
187
- True
188
- if (read_args.retriveArgs().after == 0 or not read_args.retriveArgs().after)
189
- else False
190
- )
191
186
 
192
187
  with Live(
193
188
  progress_group, refresh_per_second=5, console=console.get_shared_console()
@@ -221,11 +216,9 @@ async def get_archived_media(model_id, username, forced_after=None, rescan=None)
221
216
  )
222
217
  after = 0
223
218
 
224
- elif forced_after != None:
225
- after = forced_after
226
219
  else:
227
- after = get_after(model_id, username)
228
-
220
+ after = get_after(model_id, username, forced_after)
221
+ # set check
229
222
  log.info(
230
223
  f"""
231
224
  Setting initial archived scan date for {username} to {arrow.get(after).format('YYYY.MM.DD')}
@@ -334,23 +327,28 @@ Setting initial archived scan date for {username} to {arrow.get(after).format('Y
334
327
  )
335
328
  )
336
329
  log.debug(f"[bold]Archived Count without Dupes[/bold] {len(unduped)} found")
337
- if setCache and not read_args.retriveArgs().after:
338
- newCheck = {}
339
- for post in cache.get(f"archived_check_{model_id}", []) + list(
340
- unduped.values()
341
- ):
342
- newCheck[post["id"]] = post
343
- cache.set(
344
- f"archived_check_{model_id}",
345
- list(newCheck.values()),
346
- expire=constants.getattr("DAY_SECONDS"),
347
- )
348
- cache.close()
349
-
330
+ set_check(unduped, model_id, after)
350
331
  return list(unduped.values())
351
332
 
352
333
 
353
- def get_after(model_id, username):
334
+ def set_check(unduped, model_id, after):
335
+ if not after:
336
+ newCheck = {}
337
+ for post in cache.get(f"archived_check_{model_id}", []) + list(
338
+ unduped.values()
339
+ ):
340
+ newCheck[post["id"]] = post
341
+ cache.set(
342
+ f"archived_check_{model_id}",
343
+ list(newCheck.values()),
344
+ expire=constants.getattr("DAY_SECONDS"),
345
+ )
346
+ cache.close()
347
+
348
+
349
+ def get_after(model_id, username, forced_after=None):
350
+ if forced_after != None:
351
+ return forced_after
354
352
  if read_args.retriveArgs().after:
355
353
  return read_args.retriveArgs().after.float_timestamp
356
354
  curr = operations.get_archived_media(model_id=model_id, username=username)
@@ -111,10 +111,9 @@ async def get_messages(model_id, username, forced_after=None, rescan=None):
111
111
  )
112
112
  after = 0
113
113
 
114
- elif forced_after != None:
115
- after = forced_after
116
114
  else:
117
- after = get_after(model_id, username)
115
+ after = get_after(model_id, username, forced_after)
116
+
118
117
  log.debug(f"Messages after = {after}")
119
118
 
120
119
  log.debug(f"Messages before = {before}")
@@ -305,9 +304,23 @@ Setting initial message scan date for {username} to {arrow.get(after).format('YY
305
304
  )
306
305
  )
307
306
  )
307
+ set_check(unduped, model_id, after)
308
308
  return list(unduped.values())
309
309
 
310
310
 
311
+ def set_check(unduped, model_id, after):
312
+ if not after:
313
+ newCheck = {}
314
+ for post in cache.get(f"message_check_{model_id}", []) + list(unduped.values()):
315
+ newCheck[post["id"]] = post
316
+ cache.set(
317
+ f"message_check_{model_id}",
318
+ list(newCheck.values()),
319
+ expire=constants.getattr("DAY_SECONDS"),
320
+ )
321
+ cache.close()
322
+
323
+
311
324
  async def scrape_messages(
312
325
  c, model_id, progress, message_id=None, required_ids=None
313
326
  ) -> list:
@@ -449,7 +462,9 @@ def get_individual_post(model_id, postid, c=None):
449
462
  log.debug(f"[bold]Individual message headers:[/bold] {r.headers}")
450
463
 
451
464
 
452
- def get_after(model_id, username):
465
+ def get_after(model_id, username, forced_after=None):
466
+ if forced_after != None:
467
+ return forced_after
453
468
  if read_args.retriveArgs().after:
454
469
  return read_args.retriveArgs().after.float_timestamp
455
470
  curr = operations.get_messages_media(model_id=model_id, username=username)
@@ -30,6 +30,7 @@ from tenacity import (
30
30
  )
31
31
 
32
32
  import ofscraper.classes.sessionbuilder as sessionbuilder
33
+ import ofscraper.utils.args.read as read_args
33
34
  import ofscraper.utils.cache as cache
34
35
  import ofscraper.utils.console as console
35
36
  import ofscraper.utils.constants as constants
@@ -104,15 +105,22 @@ async def get_paid_posts(username, model_id):
104
105
  )
105
106
  )
106
107
  )
107
- cache.set(
108
- f"purchased_check_{model_id}",
109
- list(outdict.values()),
110
- expire=constants.getattr("DAY_SECONDS"),
111
- )
112
- cache.close()
108
+ set_check(outdict, model_id)
113
109
  return list(outdict.values())
114
110
 
115
111
 
112
+ def set_check(unduped, model_id):
113
+ newCheck = {}
114
+ for post in cache.get(f"purchased_check_{model_id}", []) + list(unduped.values()):
115
+ newCheck[post["id"]] = post
116
+ cache.set(
117
+ f"purchased_check_{model_id}",
118
+ list(newCheck.values()),
119
+ expire=constants.getattr("DAY_SECONDS"),
120
+ )
121
+ cache.close()
122
+
123
+
116
124
  async def scrape_paid(c, username, job_progress, offset=0):
117
125
  """Takes headers to access onlyfans as an argument and then checks the purchased content
118
126
  url to look for any purchased content. If it finds some it will return it as a list.
@@ -208,11 +208,8 @@ async def get_timeline_media(model_id, username, forced_after=None, rescan=None)
208
208
  "Used --after previously. Scraping all timeline posts required to make sure content is not missing"
209
209
  )
210
210
  after = 0
211
-
212
- elif forced_after != None:
213
- after = forced_after
214
211
  else:
215
- after = get_after(model_id, username)
212
+ after = get_after(model_id, username, forced_after)
216
213
 
217
214
  log.info(
218
215
  f"""
@@ -330,9 +327,25 @@ Setting initial timeline scan date for {username} to {arrow.get(after).format('Y
330
327
  )
331
328
  )
332
329
  log.debug(f"[bold]Timeline Count without Dupes[/bold] {len(unduped)} found")
330
+ set_check(unduped, model_id, after)
333
331
  return list(unduped.values())
334
332
 
335
333
 
334
+ def set_check(unduped, model_id, after):
335
+ if not after:
336
+ newCheck = {}
337
+ for post in cache.get(f"timeline_check_{model_id}", []) + list(
338
+ unduped.values()
339
+ ):
340
+ newCheck[post["id"]] = post
341
+ cache.set(
342
+ f"timeline_check_{model_id}",
343
+ list(newCheck.values()),
344
+ expire=constants.getattr("DAY_SECONDS"),
345
+ )
346
+ cache.close()
347
+
348
+
336
349
  def get_individual_post(id, c=None):
337
350
  with c.requests(constants.getattr("INDVIDUAL_TIMELINE").format(id))() as r:
338
351
  if r.ok:
@@ -344,7 +357,9 @@ def get_individual_post(id, c=None):
344
357
  log.debug(f"[bold]individual post headers:[/bold] {r.headers}")
345
358
 
346
359
 
347
- def get_after(model_id, username):
360
+ def get_after(model_id, username, forced_after=None):
361
+ if forced_after != None:
362
+ return forced_after
348
363
  if read_args.retriveArgs().after:
349
364
  return read_args.retriveArgs().after.float_timestamp
350
365
  curr = operations.get_timeline_media(model_id=model_id, username=username)
@@ -18,6 +18,7 @@ from tenacity import (
18
18
  )
19
19
 
20
20
  import ofscraper.classes.sessionbuilder as sessionbuilder
21
+ import ofscraper.utils.args.quality as quality
21
22
  import ofscraper.utils.config.data as data
22
23
  import ofscraper.utils.constants as constants
23
24
 
@@ -32,11 +33,29 @@ class Media:
32
33
  self._media = media
33
34
  self._count = count
34
35
  self._post = post
36
+ self._final_url = None
35
37
 
36
38
  @property
37
39
  def expires(self):
38
40
  return self._post.expires
39
41
 
42
+ @property
43
+ def media_source(self):
44
+ return self._media.get("source", {})
45
+
46
+ @property
47
+ def files_source(self):
48
+ return self._media.get("files", {}).get("source", {})
49
+ allowed = quality.get_allowed_qualities()
50
+ for ele in ["240", "720", "source"]:
51
+ if ele in allowed and self._media.get("files", {}).get(ele):
52
+ return self._media.get("files", {}).get(ele)
53
+ return {}
54
+
55
+ @property
56
+ def quality(self):
57
+ return self._media.get("videoSources", {})
58
+
40
59
  @property
41
60
  def mediatype(self):
42
61
  if self._media["type"] == "photo":
@@ -50,9 +69,7 @@ class Media:
50
69
 
51
70
  @property
52
71
  def length(self):
53
- return self._media.get("duration") or self._media.get("source", {}).get(
54
- "duration"
55
- )
72
+ return self._media.get("duration") or self.media_source.get("duration")
56
73
 
57
74
  @property
58
75
  def numeric_length(self):
@@ -62,12 +79,21 @@ class Media:
62
79
 
63
80
  @property
64
81
  def url(self):
82
+ if self._final_url:
83
+ return self._final_url
65
84
  if self.responsetype == "stories" or self.responsetype == "highlights":
66
- return self._media.get("files", {}).get("source", {}).get("url")
85
+ self._final_url = self.files_source.get("url")
67
86
  elif self.responsetype == "profile":
68
- return self._media.get("url")
87
+ self._final_url = self._media.get("url")
69
88
  else:
89
+ self._final_url = self._url_source_helper()
90
+ return self._final_url
91
+
92
+ def _url_source_helper(self):
93
+ quality = self.selected_quality
94
+ if quality == "source":
70
95
  return self._media.get("source", {}).get("source")
96
+ return self._media.get("videoSources", {}).get(quality)
71
97
 
72
98
  @property
73
99
  def post(self):
@@ -256,9 +282,17 @@ class Media:
256
282
  def final_filename(self):
257
283
  filename = self.filename or self.id
258
284
  if self.mediatype == "videos":
259
- filename = (
260
- filename if re.search("_source", filename) else f"{filename}_source"
261
- )
285
+ filename = re.sub("_[a-z0-9]+$", f"", filename)
286
+ filename = f"{filename}_{self.selected_quality}"
287
+ # cleanup
288
+ filename = self.cleanup(filename)
289
+ return filename
290
+
291
+ @property
292
+ def no_quality_final_filename(self):
293
+ filename = self.filename or self.id
294
+ if self.mediatype == "videos":
295
+ filename = re.sub("_[a-z]+", f"", filename)
262
296
  # cleanup
263
297
  filename = self.cleanup(filename)
264
298
  return filename
@@ -322,10 +356,20 @@ class Media:
322
356
  def mediatype(self, val):
323
357
  self._media["type"] = val
324
358
 
359
+ @property
360
+ def selected_quality(self):
361
+ allowed = quality.get_allowed_qualities()
362
+ if self.mediatype != "videos":
363
+ return "source"
364
+ for ele in ["240", "720"]:
365
+ if ele in allowed and self._media.get("videoSources", {}).get(ele):
366
+ return ele
367
+ return "source"
368
+
325
369
  # for use in dynamic names
326
370
  @property
327
371
  def needs_count(self):
328
- if set(["filename", "file_name", "text", "postid", "post_id"]).isdisjoint(
372
+ if set(["text", "postid", "post_id"]).isdisjoint(
329
373
  [
330
374
  name
331
375
  for text, name, spec, conv in list(
@@ -128,6 +128,12 @@ class Placeholders:
128
128
  self._variables.update({"media_id": media_id})
129
129
  self._variables.update({"mediaid": media_id})
130
130
  self._variables.update({"modelObj": selector.get_model_fromParsed(username)})
131
+ self._variables.update({"quality": ele.selected_quality})
132
+ self._variables.update({"filename": ele.final_filename})
133
+ self._variables.update({"file_name": ele.final_filename})
134
+ self._variables.update({"only_file_name": ele.no_quality_final_filename})
135
+ self._variables.update({"only_filename": ele.no_quality_final_filename})
136
+ self._variables.update({"text": ele.file_text})
131
137
  self.add_price_variables(username)
132
138
 
133
139
  @wrapper
@@ -235,11 +241,7 @@ class Placeholders:
235
241
  @wrapper
236
242
  def createfilename(self, ele, username, model_id, ext):
237
243
  filename = ele.final_filename
238
- self._variables.update({"filename": filename})
239
- self._variables.update({"file_name": filename})
240
244
  self._variables.update({"ext": ext})
241
- text = ele.file_text
242
- self._variables.update({"text": text})
243
245
  self.add_common_variables(ele, username, model_id)
244
246
  globals().update(self._variables)
245
247
  log.trace(
@@ -119,7 +119,7 @@ def scrapper():
119
119
  if len(args.action) == 0 and not args.scrape_paid:
120
120
  prompts.action_prompt()
121
121
  daemon_process()
122
- elif len(args.action) > 0:
122
+ elif len(args.action) > 0 or args.scrape_paid:
123
123
  process_selected_areas()
124
124
  elif len(args.action) == 0:
125
125
  process_prompts()
@@ -8,5 +8,5 @@ FFMPEG_WINDOWS = "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest
8
8
  FFMPEG_MAC = "https://evermeet.cx/ffmpeg/#api-download"
9
9
  CDRM2 = "http://172.106.17.134:8080/wv"
10
10
  KEYDB = "https://keysdb.net/api"
11
- CDRM = "https://cdrm-project.com"
11
+ CDRM = "https://cdrm-project.com/wv"
12
12
  LARGEZIP = "https://proof.ovh.net/files/100Mb.dat"
@@ -524,11 +524,8 @@ def log_download_progress(media_type):
524
524
  if media_type is None:
525
525
  return
526
526
  if (photo_count + audio_count + video_count + forced_skipped + skipped) % 20 == 0:
527
- None
528
-
529
-
530
- # log.debug(
531
- # f"In progress -> {format_size(total_bytes )}) ({photo_count+audio_count+video_count} \
532
- # downloads total [{video_count} videos, {audio_count} audios, {photo_count} photos], \
533
- # {forced_skipped} skipped, {skipped} failed)"
534
- # )
527
+ log.debug(
528
+ f"In progress -> {format_size(total_bytes )}) ({photo_count+audio_count+video_count} \
529
+ downloads total [{video_count} videos, {audio_count} audios, {photo_count} photos], \
530
+ {forced_skipped} skipped, {skipped} failed)"
531
+ )
@@ -149,3 +149,7 @@ def mass_msg_filter(media):
149
149
  return list((filter(lambda x: x.mass is True, media)))
150
150
  elif read_args.retriveArgs().mass_msg is False:
151
151
  return list((filter(lambda x: x.mass is False, media)))
152
+
153
+
154
+ def url_filter(media):
155
+ return list((filter(lambda x: x.url or x.mpd, media)))
@@ -214,6 +214,24 @@ def filterMedia(media):
214
214
  )
215
215
  )
216
216
  )
217
- log.debug(f"filter{count}-> all media mass message filter count: {len(media)}")
217
+
218
+ media = helpers.url_filter(media)
219
+ count += 1
220
+ log.trace(
221
+ "\n\n\n".join(
222
+ list(
223
+ map(
224
+ lambda x: logformater.format(
225
+ f"filter {count}-> valid url filter: ",
226
+ x.media,
227
+ x.id,
228
+ x.postid,
229
+ ),
230
+ media,
231
+ )
232
+ )
233
+ )
234
+ )
235
+ log.debug(f"filter{count}-> all media valid url filter count {len(media)}")
218
236
 
219
237
  return media
@@ -10,7 +10,7 @@ import ofscraper.utils.context.stdout as stdout
10
10
  import ofscraper.utils.me as me_util
11
11
 
12
12
 
13
- def get_models(accounts=None) -> list:
13
+ def get_models(main=False) -> list:
14
14
  """
15
15
  Get user's subscriptions in form of a list.
16
16
  """
@@ -48,6 +48,17 @@ def get_via_list(count):
48
48
  return models_objects
49
49
 
50
50
 
51
+ def get_main_list(count):
52
+ out = []
53
+ active_subscriptions = subscriptions.get_subscriptions(count[0], forced=True)
54
+ expired_subscriptions = subscriptions.get_subscriptions(
55
+ count[1], account="expired", forced=True
56
+ )
57
+ out.extend(active_subscriptions)
58
+ out.extend(expired_subscriptions)
59
+ return out
60
+
61
+
51
62
  def get_via_individual():
52
63
  out = individual.get_subscription()
53
64
  console.get_shared_console().print(
@@ -34,9 +34,8 @@ def set_ALL_SUBS_DICT(subsDict=None):
34
34
  subList = subsDict or ALL_SUBS
35
35
  if not subList:
36
36
  all_subs_helper()
37
- if not ALL_SUBS_DICT:
38
- ALL_SUBS_DICT = {}
39
- [ALL_SUBS_DICT.update({ele.name: ele}) for ele in subList]
37
+ ALL_SUBS_DICT = {}
38
+ [ALL_SUBS_DICT.update({ele.name: ele}) for ele in subList]
40
39
 
41
40
 
42
41
  def get_ALL_SUBS_DICT():
@@ -45,17 +44,8 @@ def get_ALL_SUBS_DICT():
45
44
 
46
45
 
47
46
  def set_ALL_SUBS_DICTVManger(subsDict=None):
48
- global ALL_SUBS
49
47
  global ALL_SUBS_DICT
50
- if subsDict and isinstance(subsDict, dict):
51
- ALL_SUBS_DICT = subsDict
52
- else:
53
- subList = subsDict or ALL_SUBS
54
- if not subList:
55
- all_subs_helper()
56
- if not ALL_SUBS_DICT:
57
- ALL_SUBS_DICT = {}
58
- [ALL_SUBS_DICT.update({ele.name: ele}) for ele in subList]
48
+ set_ALL_SUBS_DICT(subsDict)
59
49
  manager.update_dict({"subs": ALL_SUBS_DICT})
60
50
 
61
51
 
@@ -94,12 +84,12 @@ def getselected_usernames(rescan=False, reset=False):
94
84
  return PARSED_SUBS
95
85
 
96
86
 
97
- def all_subs_helper(refetch=True):
87
+ def all_subs_helper(refetch=True, main=False):
98
88
  global ALL_SUBS
99
89
  if bool(ALL_SUBS) and not refetch:
100
90
  return
101
91
  while True:
102
- ALL_SUBS = retriver.get_models()
92
+ ALL_SUBS = retriver.get_models(main)
103
93
  if len(ALL_SUBS) > 0:
104
94
  set_ALL_SUBS_DICTVManger()
105
95
  break
@@ -8,7 +8,7 @@ import ofscraper.utils.args.helpers as helpers
8
8
  import ofscraper.utils.args.write as write_args
9
9
  import ofscraper.utils.system.system as system
10
10
  from ofscraper.__version__ import __version__
11
- from ofscraper.const.constants import KEY_OPTIONS, OFSCRAPER_RESERVED_LIST
11
+ from ofscraper.const.constants import KEY_OPTIONS
12
12
 
13
13
 
14
14
  def create_parser(input=None):
@@ -119,6 +119,15 @@ def create_parser(input=None):
119
119
  default=False,
120
120
  help="Bypass the dupe check and redownload all files",
121
121
  )
122
+
123
+ post.add_argument(
124
+ "-q",
125
+ "--quality",
126
+ default="source",
127
+ help="Set the minimum allowed quality for videos",
128
+ choices=["240", "720", "source"],
129
+ )
130
+
122
131
  post.add_argument(
123
132
  "-o",
124
133
  "--posts",
@@ -722,8 +731,7 @@ def create_parser(input=None):
722
731
  group11.add_argument(
723
732
  "-fl",
724
733
  "--force-list",
725
- help="When --username arg is provided searches entire list based on the currently enabled\
726
- config settings for userlist or the --user-list arg",
734
+ help="When --username arg is provided searches entire enabled list(s) before filtering for usernames",
727
735
  default=False,
728
736
  action="store_true",
729
737
  )
@@ -732,7 +740,7 @@ config settings for userlist or the --user-list arg",
732
740
  group12.add_argument(
733
741
  "-md",
734
742
  "--metadata",
735
- help="Skip all media downloads and gathers metadata only\nNo change to download status",
743
+ help="Skip all media downloads and gathers metadata only\nNo change to download status in media table",
736
744
  default=None,
737
745
  action="store_const",
738
746
  dest="metadata",
@@ -741,7 +749,7 @@ config settings for userlist or the --user-list arg",
741
749
  group12.add_argument(
742
750
  "-mu",
743
751
  "--metadata-update",
744
- help="Skip all media downloads and gathers metadata only\nUpdates downloads based on file presence",
752
+ help="Skip all media downloads and gathers metadata only\nUpdates downloaded status in media table based on file presence",
745
753
  default=None,
746
754
  action="store_const",
747
755
  dest="metadata",
@@ -750,7 +758,7 @@ config settings for userlist or the --user-list arg",
750
758
  group12.add_argument(
751
759
  "-mc",
752
760
  "--metadata-complete",
753
- help="Skip all media downloads and gathers metadata only\nMarks each media item as downloaded",
761
+ help="Skip all media downloads and gathers metadata only\nMarks each media item as downloaded in media table",
754
762
  default=None,
755
763
  action="store_const",
756
764
  dest="metadata",
@@ -0,0 +1,13 @@
1
+ import ofscraper.utils.args.read as read_args
2
+
3
+
4
+ def get_allowed_qualities():
5
+ qualities = {"source": 4000, "240": 240, "720": 720}
6
+ minQuality = read_args.retriveArgs().quality
7
+ validQualities = list(
8
+ filter(
9
+ lambda x: x[1] >= qualities.get(minQuality),
10
+ qualities.items(),
11
+ )
12
+ )
13
+ return list(map(lambda x: x[0], validQualities))
@@ -11,7 +11,8 @@ def retriveArgsVManager():
11
11
  manager.get_manager_dict()["args"] = parse_args.parse_args()
12
12
  return manager.get_manager_dict().get("args")
13
13
  except SystemExit as E:
14
- print(f"Passed Args {sys.argv[1:]}")
14
+ if not any(ele in sys.argv[1:] for ele in ["-h", "-v"]):
15
+ print(f"Passed Args {sys.argv[1:]}")
15
16
  raise E
16
17
 
17
18
 
@@ -21,5 +22,6 @@ def retriveArgs():
21
22
  global_args.args = parse_args.parse_args()
22
23
  return global_args.args
23
24
  except SystemExit as E:
24
- print(f"Passed Args {sys.argv[1:]}")
25
+ if not any(ele in sys.argv[1:] for ele in ["-h", "-v"]):
26
+ print(f"Passed Args {sys.argv[1:]}")
25
27
  raise E
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "ofscraper"
3
- version = "3.7.0dev11"
3
+ version = "3.8.0dev"
4
4
  description = "automatically scrape onlyfans"
5
5
  authors = ["datawhores <datawhores@riseup.net>"]
6
6
  readme = "README.md"
File without changes