netspy 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. netspy-1.0.0/.gitignore +36 -0
  2. netspy-1.0.0/CHANGELOG.md +1236 -0
  3. netspy-1.0.0/LICENSE +21 -0
  4. netspy-1.0.0/PKG-INFO +186 -0
  5. netspy-1.0.0/README.md +96 -0
  6. netspy-1.0.0/docs/anti-bot.md +118 -0
  7. netspy-1.0.0/docs/api.md +53 -0
  8. netspy-1.0.0/docs/async-kernel.md +231 -0
  9. netspy-1.0.0/docs/batch-spider.md +124 -0
  10. netspy-1.0.0/docs/cli.md +46 -0
  11. netspy-1.0.0/docs/distributed.md +378 -0
  12. netspy-1.0.0/docs/index.md +59 -0
  13. netspy-1.0.0/docs/item-pipeline.md +244 -0
  14. netspy-1.0.0/docs/middleware-proxy.md +119 -0
  15. netspy-1.0.0/docs/observability.md +114 -0
  16. netspy-1.0.0/docs/quickstart.md +89 -0
  17. netspy-1.0.0/docs/render.md +58 -0
  18. netspy-1.0.0/docs/roadmap.md +2230 -0
  19. netspy-1.0.0/docs/settings.md +383 -0
  20. netspy-1.0.0/docs/spider.md +361 -0
  21. netspy-1.0.0/docs/user-pool.md +95 -0
  22. netspy-1.0.0/examples/README.md +31 -0
  23. netspy-1.0.0/examples/books_toscrape.py +80 -0
  24. netspy-1.0.0/mkdocs.yml +59 -0
  25. netspy-1.0.0/netspy/__about__.py +3 -0
  26. netspy-1.0.0/netspy/__init__.py +81 -0
  27. netspy-1.0.0/netspy/buffer/__init__.py +8 -0
  28. netspy-1.0.0/netspy/buffer/item_buffer.py +304 -0
  29. netspy-1.0.0/netspy/buffer/request_buffer.py +140 -0
  30. netspy-1.0.0/netspy/commands/__init__.py +21 -0
  31. netspy-1.0.0/netspy/commands/cmdline.py +127 -0
  32. netspy-1.0.0/netspy/commands/create/__init__.py +181 -0
  33. netspy-1.0.0/netspy/commands/retry.py +134 -0
  34. netspy-1.0.0/netspy/commands/shell.py +38 -0
  35. netspy-1.0.0/netspy/core/__init__.py +5 -0
  36. netspy-1.0.0/netspy/core/base_parser.py +79 -0
  37. netspy-1.0.0/netspy/core/base_scheduler.py +397 -0
  38. netspy-1.0.0/netspy/core/batch_monitor.py +196 -0
  39. netspy-1.0.0/netspy/core/batch_store.py +386 -0
  40. netspy-1.0.0/netspy/core/collector.py +86 -0
  41. netspy-1.0.0/netspy/core/context.py +32 -0
  42. netspy-1.0.0/netspy/core/parser_control.py +332 -0
  43. netspy-1.0.0/netspy/core/redis_scheduler.py +437 -0
  44. netspy-1.0.0/netspy/core/redis_task_scheduler.py +112 -0
  45. netspy-1.0.0/netspy/core/scheduler.py +45 -0
  46. netspy-1.0.0/netspy/core/spiders/__init__.py +10 -0
  47. netspy-1.0.0/netspy/core/spiders/air_spider.py +50 -0
  48. netspy-1.0.0/netspy/core/spiders/batch_spider.py +200 -0
  49. netspy-1.0.0/netspy/core/spiders/spider.py +55 -0
  50. netspy-1.0.0/netspy/core/spiders/task_spider.py +96 -0
  51. netspy-1.0.0/netspy/core/task_queue.py +280 -0
  52. netspy-1.0.0/netspy/core/task_source.py +45 -0
  53. netspy-1.0.0/netspy/db/__init__.py +23 -0
  54. netspy-1.0.0/netspy/db/mysqldb.py +136 -0
  55. netspy-1.0.0/netspy/db/postgresdb.py +76 -0
  56. netspy-1.0.0/netspy/db/redisdb.py +50 -0
  57. netspy-1.0.0/netspy/dedup/__init__.py +127 -0
  58. netspy-1.0.0/netspy/dedup/bloom_filter.py +211 -0
  59. netspy-1.0.0/netspy/dedup/lite_filter.py +30 -0
  60. netspy-1.0.0/netspy/dedup/redis_filter.py +189 -0
  61. netspy-1.0.0/netspy/exceptions.py +95 -0
  62. netspy-1.0.0/netspy/network/__init__.py +8 -0
  63. netspy-1.0.0/netspy/network/antibot.py +90 -0
  64. netspy-1.0.0/netspy/network/cache.py +103 -0
  65. netspy-1.0.0/netspy/network/circuit.py +123 -0
  66. netspy-1.0.0/netspy/network/downloader/__init__.py +112 -0
  67. netspy-1.0.0/netspy/network/downloader/_async_httpx.py +240 -0
  68. netspy-1.0.0/netspy/network/downloader/_common.py +473 -0
  69. netspy-1.0.0/netspy/network/downloader/_curl.py +202 -0
  70. netspy-1.0.0/netspy/network/downloader/_httpx.py +188 -0
  71. netspy-1.0.0/netspy/network/downloader/_playwright.py +279 -0
  72. netspy-1.0.0/netspy/network/downloader/base.py +33 -0
  73. netspy-1.0.0/netspy/network/global_throttle.py +98 -0
  74. netspy-1.0.0/netspy/network/item.py +118 -0
  75. netspy-1.0.0/netspy/network/middleware.py +70 -0
  76. netspy-1.0.0/netspy/network/proxy_pool/__init__.py +34 -0
  77. netspy-1.0.0/netspy/network/proxy_pool/api.py +226 -0
  78. netspy-1.0.0/netspy/network/proxy_pool/base.py +31 -0
  79. netspy-1.0.0/netspy/network/request.py +191 -0
  80. netspy-1.0.0/netspy/network/response.py +188 -0
  81. netspy-1.0.0/netspy/network/robots.py +206 -0
  82. netspy-1.0.0/netspy/network/status.py +97 -0
  83. netspy-1.0.0/netspy/network/throttle.py +158 -0
  84. netspy-1.0.0/netspy/network/user_agent.py +23 -0
  85. netspy-1.0.0/netspy/network/user_pool/__init__.py +24 -0
  86. netspy-1.0.0/netspy/network/user_pool/base.py +39 -0
  87. netspy-1.0.0/netspy/network/user_pool/local.py +92 -0
  88. netspy-1.0.0/netspy/network/user_pool/middleware.py +94 -0
  89. netspy-1.0.0/netspy/network/user_pool/redis.py +82 -0
  90. netspy-1.0.0/netspy/pipelines/__init__.py +13 -0
  91. netspy-1.0.0/netspy/pipelines/_sql.py +96 -0
  92. netspy-1.0.0/netspy/pipelines/base.py +21 -0
  93. netspy-1.0.0/netspy/pipelines/console.py +23 -0
  94. netspy-1.0.0/netspy/pipelines/csv.py +62 -0
  95. netspy-1.0.0/netspy/pipelines/elasticsearch.py +66 -0
  96. netspy-1.0.0/netspy/pipelines/kafka.py +55 -0
  97. netspy-1.0.0/netspy/pipelines/mongo.py +56 -0
  98. netspy-1.0.0/netspy/pipelines/mysql.py +35 -0
  99. netspy-1.0.0/netspy/pipelines/postgres.py +57 -0
  100. netspy-1.0.0/netspy/py.typed +0 -0
  101. netspy-1.0.0/netspy/setting.py +500 -0
  102. netspy-1.0.0/netspy/templates/__init__.py +1 -0
  103. netspy-1.0.0/netspy/templates/air_spider.py.jinja +23 -0
  104. netspy-1.0.0/netspy/templates/item.py.jinja +13 -0
  105. netspy-1.0.0/netspy/templates/item_fields.py.jinja +21 -0
  106. netspy-1.0.0/netspy/templates/project/README.md.jinja +30 -0
  107. netspy-1.0.0/netspy/templates/project/main.py.jinja +6 -0
  108. netspy-1.0.0/netspy/templates/project/spider.py.jinja +21 -0
  109. netspy-1.0.0/netspy/templates/setting.py.jinja +52 -0
  110. netspy-1.0.0/netspy/utils/__init__.py +1 -0
  111. netspy-1.0.0/netspy/utils/alert.py +239 -0
  112. netspy-1.0.0/netspy/utils/log.py +80 -0
  113. netspy-1.0.0/netspy/utils/metrics.py +97 -0
  114. netspy-1.0.0/netspy/utils/stats.py +100 -0
  115. netspy-1.0.0/netspy/utils/tools.py +93 -0
  116. netspy-1.0.0/pyproject.toml +149 -0
  117. netspy-1.0.0/tests/conftest.py +85 -0
  118. netspy-1.0.0/tests/conn_counting_server.py +97 -0
  119. netspy-1.0.0/tests/integration_site.py +130 -0
  120. netspy-1.0.0/tests/test_ack_after_persist.py +214 -0
  121. netspy-1.0.0/tests/test_air_spider.py +364 -0
  122. netspy-1.0.0/tests/test_alert.py +158 -0
  123. netspy-1.0.0/tests/test_alert_channels.py +183 -0
  124. netspy-1.0.0/tests/test_antibot.py +129 -0
  125. netspy-1.0.0/tests/test_async_downloader.py +131 -0
  126. netspy-1.0.0/tests/test_async_loop_sharding.py +115 -0
  127. netspy-1.0.0/tests/test_batch_ack_after_persist.py +202 -0
  128. netspy-1.0.0/tests/test_batch_full_stack.py +216 -0
  129. netspy-1.0.0/tests/test_batch_spider.py +444 -0
  130. netspy-1.0.0/tests/test_batch_store_integration.py +264 -0
  131. netspy-1.0.0/tests/test_benchmarks_smoke.py +80 -0
  132. netspy-1.0.0/tests/test_circuit.py +133 -0
  133. netspy-1.0.0/tests/test_cli.py +240 -0
  134. netspy-1.0.0/tests/test_client_cache_race.py +131 -0
  135. netspy-1.0.0/tests/test_connection_reuse_truth.py +187 -0
  136. netspy-1.0.0/tests/test_curl_downloader.py +233 -0
  137. netspy-1.0.0/tests/test_curl_session_sharding.py +151 -0
  138. netspy-1.0.0/tests/test_dedup.py +67 -0
  139. netspy-1.0.0/tests/test_dedup_integration.py +102 -0
  140. netspy-1.0.0/tests/test_dedup_scale.py +210 -0
  141. netspy-1.0.0/tests/test_distributed_full_stack.py +292 -0
  142. netspy-1.0.0/tests/test_distributed_integration.py +669 -0
  143. netspy-1.0.0/tests/test_downloader.py +169 -0
  144. netspy-1.0.0/tests/test_dump_replay.py +166 -0
  145. netspy-1.0.0/tests/test_effective_concurrency.py +76 -0
  146. netspy-1.0.0/tests/test_es_kafka.py +139 -0
  147. netspy-1.0.0/tests/test_examples.py +72 -0
  148. netspy-1.0.0/tests/test_exit_integrity.py +173 -0
  149. netspy-1.0.0/tests/test_flush_failure_paths.py +136 -0
  150. netspy-1.0.0/tests/test_full_stack_integration.py +272 -0
  151. netspy-1.0.0/tests/test_integration_httpserver.py +59 -0
  152. netspy-1.0.0/tests/test_item.py +110 -0
  153. netspy-1.0.0/tests/test_item_buffer.py +141 -0
  154. netspy-1.0.0/tests/test_item_dedup_degrade.py +137 -0
  155. netspy-1.0.0/tests/test_lease_renewal.py +136 -0
  156. netspy-1.0.0/tests/test_limits.py +204 -0
  157. netspy-1.0.0/tests/test_log.py +123 -0
  158. netspy-1.0.0/tests/test_max_runtime.py +156 -0
  159. netspy-1.0.0/tests/test_metrics.py +59 -0
  160. netspy-1.0.0/tests/test_middleware.py +110 -0
  161. netspy-1.0.0/tests/test_middleware_retry_budget.py +117 -0
  162. netspy-1.0.0/tests/test_mysql.py +172 -0
  163. netspy-1.0.0/tests/test_packaging.py +107 -0
  164. netspy-1.0.0/tests/test_pipelines.py +80 -0
  165. netspy-1.0.0/tests/test_pool_limits.py +86 -0
  166. netspy-1.0.0/tests/test_postgres.py +141 -0
  167. netspy-1.0.0/tests/test_proxy_ban_decay.py +133 -0
  168. netspy-1.0.0/tests/test_proxy_blame.py +228 -0
  169. netspy-1.0.0/tests/test_proxy_pool.py +121 -0
  170. netspy-1.0.0/tests/test_proxy_pool_bookkeeping.py +113 -0
  171. netspy-1.0.0/tests/test_proxy_required.py +127 -0
  172. netspy-1.0.0/tests/test_redis_infra.py +176 -0
  173. netspy-1.0.0/tests/test_render.py +164 -0
  174. netspy-1.0.0/tests/test_render_shutdown.py +124 -0
  175. netspy-1.0.0/tests/test_request.py +108 -0
  176. netspy-1.0.0/tests/test_request_buffer_failure.py +133 -0
  177. netspy-1.0.0/tests/test_request_replay.py +132 -0
  178. netspy-1.0.0/tests/test_response.py +75 -0
  179. netspy-1.0.0/tests/test_response_cache.py +209 -0
  180. netspy-1.0.0/tests/test_robots.py +212 -0
  181. netspy-1.0.0/tests/test_run_scope.py +321 -0
  182. netspy-1.0.0/tests/test_run_summary.py +216 -0
  183. netspy-1.0.0/tests/test_session_reuse.py +183 -0
  184. netspy-1.0.0/tests/test_session_sharding.py +246 -0
  185. netspy-1.0.0/tests/test_setting.py +115 -0
  186. netspy-1.0.0/tests/test_setting_typo_warning.py +126 -0
  187. netspy-1.0.0/tests/test_smoke.py +18 -0
  188. netspy-1.0.0/tests/test_spider.py +182 -0
  189. netspy-1.0.0/tests/test_spider_persistence.py +99 -0
  190. netspy-1.0.0/tests/test_sql_integration.py +317 -0
  191. netspy-1.0.0/tests/test_startup_race.py +82 -0
  192. netspy-1.0.0/tests/test_status.py +143 -0
  193. netspy-1.0.0/tests/test_status_e2e.py +247 -0
  194. netspy-1.0.0/tests/test_task_spider.py +150 -0
  195. netspy-1.0.0/tests/test_teardown_isolation.py +119 -0
  196. netspy-1.0.0/tests/test_throttle.py +283 -0
  197. netspy-1.0.0/tests/test_tools.py +88 -0
  198. netspy-1.0.0/tests/test_updateitem_fingerprint.py +113 -0
  199. netspy-1.0.0/tests/test_user_pool.py +195 -0
  200. netspy-1.0.0/tests/test_user_pool_strict.py +177 -0
netspy-1.0.0/.gitignore ADDED
@@ -0,0 +1,36 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ .eggs/
6
+ build/
7
+ dist/
8
+ *.so
9
+
10
+ # 虚拟环境
11
+ .venv/
12
+ venv/
13
+ env/
14
+
15
+ # 测试 / 类型检查 / lint 缓存
16
+ .pytest_cache/
17
+ .mypy_cache/
18
+ .ruff_cache/
19
+ .coverage
20
+ .coverage.*
21
+ htmlcov/
22
+ coverage.xml
23
+
24
+ # 运行产物
25
+ logs/
26
+ *.log
27
+ failed_items.jsonl
28
+ failed_requests.jsonl
29
+
30
+ # IDE / 系统
31
+ .idea/
32
+ .vscode/
33
+ .DS_Store
34
+
35
+ # 文档构建
36
+ site/