crawlo 1.1.8__tar.gz → 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of crawlo might be problematic. Click here for more details.

Files changed (202) hide show
  1. {crawlo-1.1.8 → crawlo-1.2.0}/LICENSE +22 -22
  2. {crawlo-1.1.8 → crawlo-1.2.0}/MANIFEST.in +16 -16
  3. crawlo-1.2.0/PKG-INFO +697 -0
  4. crawlo-1.2.0/README.md +647 -0
  5. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/__init__.py +61 -61
  6. crawlo-1.2.0/crawlo/__version__.py +1 -0
  7. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/cleaners/__init__.py +60 -60
  8. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/cleaners/data_formatter.py +225 -225
  9. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/cleaners/encoding_converter.py +125 -125
  10. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/cleaners/text_cleaner.py +232 -232
  11. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/cli.py +65 -65
  12. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/commands/__init__.py +14 -14
  13. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/commands/check.py +594 -594
  14. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/commands/genspider.py +151 -151
  15. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/commands/help.py +132 -132
  16. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/commands/list.py +155 -155
  17. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/commands/run.py +292 -292
  18. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/commands/startproject.py +418 -418
  19. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/commands/stats.py +188 -188
  20. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/commands/utils.py +186 -186
  21. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/config.py +312 -312
  22. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/config_validator.py +252 -252
  23. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/core/__init__.py +2 -2
  24. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/core/engine.py +354 -345
  25. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/core/processor.py +40 -40
  26. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/core/scheduler.py +143 -136
  27. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/crawler.py +1027 -1027
  28. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/downloader/__init__.py +266 -266
  29. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/downloader/aiohttp_downloader.py +220 -220
  30. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/downloader/cffi_downloader.py +256 -256
  31. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/downloader/httpx_downloader.py +259 -259
  32. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/downloader/hybrid_downloader.py +213 -213
  33. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/downloader/playwright_downloader.py +402 -402
  34. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/downloader/selenium_downloader.py +472 -472
  35. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/event.py +11 -11
  36. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/exceptions.py +81 -81
  37. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/extension/__init__.py +37 -37
  38. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/extension/health_check.py +141 -141
  39. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/extension/log_interval.py +57 -57
  40. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/extension/log_stats.py +81 -81
  41. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/extension/logging_extension.py +43 -43
  42. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/extension/memory_monitor.py +104 -104
  43. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/extension/performance_profiler.py +133 -133
  44. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/extension/request_recorder.py +107 -107
  45. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/filters/__init__.py +154 -154
  46. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/filters/aioredis_filter.py +280 -280
  47. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/filters/memory_filter.py +269 -269
  48. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/items/__init__.py +23 -23
  49. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/items/base.py +21 -21
  50. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/items/fields.py +53 -53
  51. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/items/items.py +104 -104
  52. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/middleware/__init__.py +21 -21
  53. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/middleware/default_header.py +32 -32
  54. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/middleware/download_delay.py +28 -28
  55. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/middleware/middleware_manager.py +135 -135
  56. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/middleware/proxy.py +272 -272
  57. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/middleware/request_ignore.py +30 -30
  58. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/middleware/response_code.py +18 -18
  59. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/middleware/response_filter.py +26 -26
  60. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/middleware/retry.py +124 -124
  61. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/mode_manager.py +211 -211
  62. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/network/__init__.py +21 -21
  63. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/network/request.py +338 -338
  64. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/network/response.py +359 -359
  65. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/pipelines/__init__.py +21 -21
  66. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/pipelines/bloom_dedup_pipeline.py +156 -156
  67. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/pipelines/console_pipeline.py +39 -39
  68. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/pipelines/csv_pipeline.py +316 -316
  69. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/pipelines/database_dedup_pipeline.py +224 -224
  70. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/pipelines/json_pipeline.py +218 -218
  71. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/pipelines/memory_dedup_pipeline.py +115 -115
  72. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/pipelines/mongo_pipeline.py +131 -131
  73. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/pipelines/mysql_pipeline.py +316 -316
  74. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/pipelines/pipeline_manager.py +61 -61
  75. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/pipelines/redis_dedup_pipeline.py +167 -167
  76. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/project.py +187 -187
  77. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/queue/pqueue.py +37 -37
  78. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/queue/queue_manager.py +337 -334
  79. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/queue/redis_priority_queue.py +298 -298
  80. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/settings/__init__.py +7 -7
  81. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/settings/default_settings.py +219 -219
  82. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/settings/setting_manager.py +122 -122
  83. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/spider/__init__.py +639 -639
  84. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/stats_collector.py +59 -59
  85. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/subscriber.py +130 -130
  86. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/task_manager.py +30 -30
  87. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/crawlo.cfg.tmpl +10 -10
  88. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/project/__init__.py.tmpl +3 -3
  89. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/project/items.py.tmpl +17 -17
  90. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/project/middlewares.py.tmpl +109 -109
  91. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/project/pipelines.py.tmpl +96 -96
  92. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/project/run.py.tmpl +45 -45
  93. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/project/settings.py.tmpl +326 -326
  94. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/project/settings_distributed.py.tmpl +119 -119
  95. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/project/settings_gentle.py.tmpl +94 -94
  96. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/project/settings_high_performance.py.tmpl +151 -151
  97. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/project/settings_simple.py.tmpl +68 -68
  98. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/project/spiders/__init__.py.tmpl +5 -5
  99. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/templates/spider/spider.py.tmpl +141 -141
  100. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/tools/__init__.py +182 -182
  101. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/tools/anti_crawler.py +268 -268
  102. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/tools/authenticated_proxy.py +240 -240
  103. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/tools/data_validator.py +180 -180
  104. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/tools/date_tools.py +35 -35
  105. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/tools/distributed_coordinator.py +386 -386
  106. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/tools/retry_mechanism.py +220 -220
  107. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/tools/scenario_adapter.py +262 -262
  108. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/__init__.py +35 -35
  109. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/batch_processor.py +260 -260
  110. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/controlled_spider_mixin.py +439 -439
  111. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/date_tools.py +290 -290
  112. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/db_helper.py +343 -343
  113. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/enhanced_error_handler.py +359 -359
  114. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/env_config.py +105 -105
  115. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/error_handler.py +125 -125
  116. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/func_tools.py +82 -82
  117. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/large_scale_config.py +286 -286
  118. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/large_scale_helper.py +343 -343
  119. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/log.py +128 -128
  120. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/performance_monitor.py +284 -284
  121. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/queue_helper.py +175 -175
  122. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/redis_connection_pool.py +334 -334
  123. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/redis_key_validator.py +199 -199
  124. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/request.py +267 -267
  125. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/request_serializer.py +219 -219
  126. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/spider_loader.py +62 -62
  127. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/system.py +11 -11
  128. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/tools.py +4 -4
  129. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/utils/url.py +39 -39
  130. crawlo-1.2.0/crawlo.egg-info/PKG-INFO +697 -0
  131. {crawlo-1.1.8 → crawlo-1.2.0}/examples/__init__.py +7 -7
  132. {crawlo-1.1.8 → crawlo-1.2.0}/pyproject.toml +2 -2
  133. {crawlo-1.1.8 → crawlo-1.2.0}/requirements.txt +28 -29
  134. {crawlo-1.1.8 → crawlo-1.2.0}/setup.cfg +71 -71
  135. {crawlo-1.1.8 → crawlo-1.2.0}/tests/DOUBLE_CRAWLO_PREFIX_FIX_REPORT.md +81 -81
  136. {crawlo-1.1.8 → crawlo-1.2.0}/tests/__init__.py +7 -7
  137. {crawlo-1.1.8 → crawlo-1.2.0}/tests/advanced_tools_example.py +275 -275
  138. {crawlo-1.1.8 → crawlo-1.2.0}/tests/authenticated_proxy_example.py +236 -236
  139. {crawlo-1.1.8 → crawlo-1.2.0}/tests/cleaners_example.py +160 -160
  140. {crawlo-1.1.8 → crawlo-1.2.0}/tests/config_validation_demo.py +102 -102
  141. {crawlo-1.1.8 → crawlo-1.2.0}/tests/controlled_spider_example.py +205 -205
  142. {crawlo-1.1.8 → crawlo-1.2.0}/tests/date_tools_example.py +180 -180
  143. {crawlo-1.1.8 → crawlo-1.2.0}/tests/dynamic_loading_example.py +523 -523
  144. {crawlo-1.1.8 → crawlo-1.2.0}/tests/dynamic_loading_test.py +104 -104
  145. {crawlo-1.1.8 → crawlo-1.2.0}/tests/env_config_example.py +133 -133
  146. {crawlo-1.1.8 → crawlo-1.2.0}/tests/error_handling_example.py +171 -171
  147. {crawlo-1.1.8 → crawlo-1.2.0}/tests/redis_key_validation_demo.py +130 -130
  148. {crawlo-1.1.8 → crawlo-1.2.0}/tests/response_improvements_example.py +144 -144
  149. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_advanced_tools.py +148 -148
  150. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_all_redis_key_configs.py +145 -145
  151. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_authenticated_proxy.py +141 -141
  152. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_cleaners.py +54 -54
  153. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_comprehensive.py +146 -146
  154. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_config_validator.py +193 -193
  155. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_date_tools.py +123 -123
  156. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_double_crawlo_fix.py +207 -207
  157. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_double_crawlo_fix_simple.py +124 -124
  158. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_dynamic_downloaders_proxy.py +124 -124
  159. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_dynamic_proxy.py +92 -92
  160. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_dynamic_proxy_config.py +146 -146
  161. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_dynamic_proxy_real.py +109 -109
  162. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_edge_cases.py +303 -303
  163. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_enhanced_error_handler.py +270 -270
  164. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_env_config.py +121 -121
  165. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_error_handler_compatibility.py +112 -112
  166. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_final_validation.py +153 -153
  167. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_framework_env_usage.py +103 -103
  168. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_integration.py +356 -356
  169. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_item_dedup_redis_key.py +122 -122
  170. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_parsel.py +29 -29
  171. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_performance.py +327 -327
  172. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_proxy_health_check.py +32 -32
  173. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_proxy_middleware_integration.py +136 -136
  174. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_proxy_providers.py +56 -56
  175. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_proxy_stats.py +19 -19
  176. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_proxy_strategies.py +59 -59
  177. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_queue_manager_double_crawlo.py +174 -231
  178. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_queue_manager_redis_key.py +176 -176
  179. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_redis_config.py +28 -28
  180. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_redis_connection_pool.py +294 -294
  181. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_redis_key_naming.py +181 -181
  182. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_redis_key_validator.py +123 -123
  183. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_redis_queue.py +224 -224
  184. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_request_serialization.py +70 -70
  185. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_response_improvements.py +152 -152
  186. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_scheduler.py +241 -241
  187. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_simple_response.py +61 -61
  188. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_telecom_spider_redis_key.py +205 -205
  189. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_template_content.py +87 -87
  190. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_template_redis_key.py +134 -134
  191. {crawlo-1.1.8 → crawlo-1.2.0}/tests/test_tools.py +153 -153
  192. {crawlo-1.1.8 → crawlo-1.2.0}/tests/tools_example.py +257 -257
  193. crawlo-1.1.8/PKG-INFO +0 -626
  194. crawlo-1.1.8/README.md +0 -576
  195. crawlo-1.1.8/crawlo/__version__.py +0 -1
  196. crawlo-1.1.8/crawlo.egg-info/PKG-INFO +0 -626
  197. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo/queue/__init__.py +0 -0
  198. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo.egg-info/SOURCES.txt +0 -0
  199. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo.egg-info/dependency_links.txt +0 -0
  200. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo.egg-info/entry_points.txt +0 -0
  201. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo.egg-info/requires.txt +0 -0
  202. {crawlo-1.1.8 → crawlo-1.2.0}/crawlo.egg-info/top_level.txt +0 -0
@@ -1,23 +1,23 @@
1
- MIT License
2
-
3
- Modifications:
4
-
5
- Copyright (c) 2020 crawl-coder <2251018029@qq.com>
6
-
7
- Permission is hereby granted, free of charge, to any person obtaining a copy
8
- of this software and associated documentation files (the "Software"), to deal
9
- in the Software without restriction, including without limitation the rights
10
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11
- copies of the Software, and to permit persons to whom the Software is
12
- furnished to do so, subject to the following conditions:
13
-
14
- The above copyright notice and this permission notice shall be included in all
15
- copies or substantial portions of the Software.
16
-
17
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
1
+ MIT License
2
+
3
+ Modifications:
4
+
5
+ Copyright (c) 2020 crawl-coder <2251018029@qq.com>
6
+
7
+ Permission is hereby granted, free of charge, to any person obtaining a copy
8
+ of this software and associated documentation files (the "Software"), to deal
9
+ in the Software without restriction, including without limitation the rights
10
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11
+ copies of the Software, and to permit persons to whom the Software is
12
+ furnished to do so, subject to the following conditions:
13
+
14
+ The above copyright notice and this permission notice shall be included in all
15
+ copies or substantial portions of the Software.
16
+
17
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23
23
  SOFTWARE.
@@ -1,17 +1,17 @@
1
- include README.md
2
- include LICENSE
3
- include requirements.txt # 如果根目录有全局requirements.txt
4
- include VERSION # 如果根目录有全局VERSION文件
5
-
6
- # 包内文件包含
7
- recursive-include crawlo/utils/js *
8
- recursive-include crawlo/templates *
9
-
10
- # 测试文件(如果需要在分发包中包含测试)
11
- recursive-include tests *
12
-
13
- # 排除项
14
- global-exclude __pycache__ *.py[cod] .DS_Store *.so
15
- global-exclude *.bak *.swp *.orig *.rej
16
- prune samples # 排除示例目录
1
+ include README.md
2
+ include LICENSE
3
+ include requirements.txt # 如果根目录有全局requirements.txt
4
+ include VERSION # 如果根目录有全局VERSION文件
5
+
6
+ # 包内文件包含
7
+ recursive-include crawlo/utils/js *
8
+ recursive-include crawlo/templates *
9
+
10
+ # 测试文件(如果需要在分发包中包含测试)
11
+ recursive-include tests *
12
+
13
+ # 排除项
14
+ global-exclude __pycache__ *.py[cod] .DS_Store *.so
15
+ global-exclude *.bak *.swp *.orig *.rej
16
+ prune samples # 排除示例目录
17
17
  prune docs # 排除文档目录