cobweb-launcher 1.2.47__py3-none-any.whl → 1.2.49__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (7) hide show
  1. cobweb/crawlers/crawler.py +3 -0
  2. cobweb/utils/dotting.py +3 -2
  3. {cobweb_launcher-1.2.47.dist-info → cobweb_launcher-1.2.49.dist-info}/METADATA +1 -1
  4. {cobweb_launcher-1.2.47.dist-info → cobweb_launcher-1.2.49.dist-info}/RECORD +7 -7
  5. {cobweb_launcher-1.2.47.dist-info → cobweb_launcher-1.2.49.dist-info}/LICENSE +0 -0
  6. {cobweb_launcher-1.2.47.dist-info → cobweb_launcher-1.2.49.dist-info}/WHEEL +0 -0
  7. {cobweb_launcher-1.2.47.dist-info → cobweb_launcher-1.2.49.dist-info}/top_level.txt +0 -0
cobweb/crawlers/crawler.py CHANGED
@@ -150,6 +150,7 @@ class Crawler(threading.Thread):
150
150
  data_size=int(download_item.response.headers.get("content-length", 0)),
151
151
  cost_time=end_time - start_time, status = 200,
152
152
  url=download_item.response.url, proxy_type=proxy_type,
153
+ project=self.project, task=self.task,
153
154
  )
154
155
  parse_iterators = self.parse(download_item)
155
156
  if not isgenerator(parse_iterators):
@@ -178,6 +179,8 @@ class Crawler(threading.Thread):
178
179
  data_size=-1, cost_time=-1,
179
180
  status=status, url=url,
180
181
  proxy_type=proxy_type,
182
+ project=self.project,
183
+ task=self.task,
181
184
  msg=exception_msg,
182
185
  )
183
186
  logger.info(LogTemplate.download_exception.format(
cobweb/utils/dotting.py CHANGED
@@ -46,9 +46,10 @@ class LoghubDot:
46
46
  try:
47
47
  log_item_info = {}
48
48
  for _ in range(1000):
49
- topic, item = self.queue.pop()
50
- if not item:
49
+ its = self.queue.pop()
50
+ if not its:
51
51
  break
52
+ topic, item = its
52
53
  log_item_info.setdefault(topic, []).append(item)
53
54
  for topic, log_items in log_item_info.items():
54
55
  request = PutLogsRequest(
{cobweb_launcher-1.2.47.dist-info → cobweb_launcher-1.2.49.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.47
3
+ Version: 1.2.49
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb_launcher-1.2.47.dist-info → cobweb_launcher-1.2.49.dist-info}/RECORD RENAMED
@@ -13,7 +13,7 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
13
13
  cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
14
14
  cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
15
15
  cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
16
- cobweb/crawlers/crawler.py,sha256=s5kImH3lzkyRm2AQoH1fwLaIO_CPZlPhqiBknPNaglM,8676
16
+ cobweb/crawlers/crawler.py,sha256=dKyQJOVh1cnuazSryb9vA-IGJJnhw-gxAAlIqsEn6-M,8832
17
17
  cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
18
18
  cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
19
19
  cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
@@ -34,11 +34,11 @@ cobweb/schedulers/scheduler_api.py,sha256=pFEdS1H4zuzxwMhCV-G7CoLz-rEOPv4EVo3xZU
34
34
  cobweb/schedulers/scheduler_redis.py,sha256=E5fjc3nNld8GbUhUGT7uY4smRejj2J2ZIzp2g6lhxFM,2205
35
35
  cobweb/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
36
36
  cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
37
- cobweb/utils/dotting.py,sha256=xACGk8cLc8d2nk41exC0PFfnThJC9__ujNc-VykFk7I,2053
37
+ cobweb/utils/dotting.py,sha256=_QsC-LCSh1QYk9XmYIWBRlRTs-E9RfqKSkkRkedV2Eo,2082
38
38
  cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
39
39
  cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
40
- cobweb_launcher-1.2.47.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
41
- cobweb_launcher-1.2.47.dist-info/METADATA,sha256=mQmD2Igg-yKySo_wNt56SV-5nMOa2ECUu2FFzAecH68,6510
42
- cobweb_launcher-1.2.47.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
43
- cobweb_launcher-1.2.47.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
44
- cobweb_launcher-1.2.47.dist-info/RECORD,,
40
+ cobweb_launcher-1.2.49.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
41
+ cobweb_launcher-1.2.49.dist-info/METADATA,sha256=VmuY5V3Hhjw9bZ5JqQfwJi8GfPWq5g6VDb6G39q36IM,6510
42
+ cobweb_launcher-1.2.49.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
43
+ cobweb_launcher-1.2.49.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
44
+ cobweb_launcher-1.2.49.dist-info/RECORD,,