cobweb-launcher 1.1.14__py3-none-any.whl → 1.1.16__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (6) hide show
  1. cobweb/crawlers/base_crawler.py +20 -21
  2. {cobweb_launcher-1.1.14.dist-info → cobweb_launcher-1.1.16.dist-info}/METADATA +1 -1
  3. {cobweb_launcher-1.1.14.dist-info → cobweb_launcher-1.1.16.dist-info}/RECORD +6 -6
  4. {cobweb_launcher-1.1.14.dist-info → cobweb_launcher-1.1.16.dist-info}/LICENSE +0 -0
  5. {cobweb_launcher-1.1.14.dist-info → cobweb_launcher-1.1.16.dist-info}/WHEEL +0 -0
  6. {cobweb_launcher-1.1.14.dist-info → cobweb_launcher-1.1.16.dist-info}/top_level.txt +0 -0
cobweb/crawlers/base_crawler.py CHANGED
@@ -69,30 +69,29 @@ class Crawler(threading.Thread):
69
69
  continue
70
70
 
71
71
  elif seed.params.retry >= self.max_retries:
72
- seed.params.identifier = DealModel.fail
72
+ seed.identifier = DealModel.fail
73
73
  self.launcher_queue['done'].push(seed)
74
74
  continue
75
75
 
76
76
  seed_detail_log_info = download_log_info(seed.to_dict)
77
77
 
78
- request_iterators = self.request(seed)
78
+ try:
79
+ request_iterators = self.request(seed)
79
80
 
80
- if not isgenerator(request_iterators):
81
- raise TypeError("request function isn't a generator!")
81
+ if not isgenerator(request_iterators):
82
+ raise TypeError("request function isn't a generator!")
82
83
 
83
- for request_item in request_iterators:
84
+ for request_item in request_iterators:
84
85
 
85
- if isinstance(request_item, BaseItem):
86
- self.upload_queue.push(request_item)
86
+ if isinstance(request_item, BaseItem):
87
+ self.upload_queue.push(request_item)
87
88
 
88
- elif isinstance(request_item, Request):
89
- try:
89
+ elif isinstance(request_item, Request):
90
90
  download_iterators = self.download(request_item)
91
91
  if not isgenerator(download_iterators):
92
92
  raise TypeError("download function isn't a generator")
93
93
  for download_item in download_iterators:
94
94
  if isinstance(download_item, Response):
95
- response_detail_log_info = download_log_info(download_item.to_dict)
96
95
  logger.info(LogTemplate.download_info.format(
97
96
  detail=seed_detail_log_info,
98
97
  retry=seed.params.retry,
@@ -100,7 +99,7 @@ class Crawler(threading.Thread):
100
99
  seed_version=seed.params.seed_version,
101
100
  identifier=seed.identifier or "",
102
101
  status=download_item.response,
103
- response=response_detail_log_info
102
+ response=download_log_info(download_item.to_dict)
104
103
  ))
105
104
  parse_iterators = self.parse(download_item)
106
105
  if not isgenerator(parse_iterators):
@@ -111,16 +110,16 @@ class Crawler(threading.Thread):
111
110
  self.distribute(parse_item, seed)
112
111
  else:
113
112
  self.distribute(download_item, seed)
114
- except Exception as e:
115
- logger.info(LogTemplate.download_exception.format(
116
- detail=seed_detail_log_info,
117
- retry=seed.params.retry,
118
- priority=seed.params.priority,
119
- seed_version=seed.params.seed_version,
120
- identifier=seed.identifier or "", exception=e
121
- ))
122
- seed.params.retry += 1
123
- self.launcher_queue['todo'].push(seed)
113
+ except Exception as e:
114
+ logger.info(LogTemplate.download_exception.format(
115
+ detail=seed_detail_log_info,
116
+ retry=seed.params.retry,
117
+ priority=seed.params.priority,
118
+ seed_version=seed.params.seed_version,
119
+ identifier=seed.identifier or "", exception=e
120
+ ))
121
+ seed.params.retry += 1
122
+ self.launcher_queue['todo'].push(seed)
124
123
 
125
124
  def run(self):
126
125
  for index in range(self.spider_thread_num):
{cobweb_launcher-1.1.14.dist-info → cobweb_launcher-1.1.16.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.1.14
3
+ Version: 1.1.16
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb_launcher-1.1.14.dist-info → cobweb_launcher-1.1.16.dist-info}/RECORD RENAMED
@@ -10,7 +10,7 @@ cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
10
10
  cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
11
11
  cobweb/base/seed.py,sha256=QxlXztWjV8VvcHu-cTzyoDHaDvoX26iyoJKiWk64HnE,2759
12
12
  cobweb/crawlers/__init__.py,sha256=Rr3DTjD-abMA1_FYcQJZYNvQvcWMuEVcsIU6duqHrw4,75
13
- cobweb/crawlers/base_crawler.py,sha256=U8MwxPlj0ld3o9dvUegcX-9arw8KvtbGUNe_LZUcjMQ,5325
13
+ cobweb/crawlers/base_crawler.py,sha256=XZZIi0zOyGax1T9TkKGJ0Enw7927VJOcrya2DgR_fV8,5174
14
14
  cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
15
15
  cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
16
16
  cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
@@ -25,8 +25,8 @@ cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzX
25
25
  cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
26
26
  cobweb/utils/oss.py,sha256=qAl05ybL2Jp6KFjHDHVMfmeBHQmDKPtZleHjHAY7LZc,3277
27
27
  cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
28
- cobweb_launcher-1.1.14.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
29
- cobweb_launcher-1.1.14.dist-info/METADATA,sha256=FoDq1JfDihDH-hEYOfYNpVcJSGHM44yWHLkWj2aUmJQ,1246
30
- cobweb_launcher-1.1.14.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
31
- cobweb_launcher-1.1.14.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
32
- cobweb_launcher-1.1.14.dist-info/RECORD,,
28
+ cobweb_launcher-1.1.16.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
29
+ cobweb_launcher-1.1.16.dist-info/METADATA,sha256=jMP0dSD4lbeLtjDjQuZSlfIRacdVMDMLPDCjWpPBKI8,1246
30
+ cobweb_launcher-1.1.16.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
31
+ cobweb_launcher-1.1.16.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
32
+ cobweb_launcher-1.1.16.dist-info/RECORD,,