cobweb-launcher 1.2.32__py3-none-any.whl → 1.2.34__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of cobweb-launcher might be problematic. Click here for more details.

Files changed (7) hide show
  1. cobweb/crawlers/crawler.py +3 -2
  2. cobweb/utils/dotting.py +5 -3
  3. {cobweb_launcher-1.2.32.dist-info → cobweb_launcher-1.2.34.dist-info}/METADATA +1 -1
  4. {cobweb_launcher-1.2.32.dist-info → cobweb_launcher-1.2.34.dist-info}/RECORD +7 -7
  5. {cobweb_launcher-1.2.32.dist-info → cobweb_launcher-1.2.34.dist-info}/LICENSE +0 -0
  6. {cobweb_launcher-1.2.32.dist-info → cobweb_launcher-1.2.34.dist-info}/WHEEL +0 -0
  7. {cobweb_launcher-1.2.32.dist-info → cobweb_launcher-1.2.34.dist-info}/top_level.txt +0 -0
cobweb/crawlers/crawler.py CHANGED
@@ -141,10 +141,11 @@ class Crawler(threading.Thread):
141
141
  ))
142
142
  if isinstance(download_item, Response):
143
143
  end_time = time.time()
144
+ upload_log = seed.to_dict
145
+ upload_log.update(**download_item.to_dict)
144
146
  self.loghub_dot.build(
145
147
  topic=f"{self.project}:{self.task}",
146
- cost_time=end_time - start_time,
147
- **download_item.to_dict
148
+ cost_time=end_time - start_time, **upload_log
148
149
  )
149
150
  parse_iterators = self.parse(download_item)
150
151
  if not isgenerator(parse_iterators):
cobweb/utils/dotting.py CHANGED
@@ -1,6 +1,6 @@
1
1
  import json
2
2
 
3
- from aliyun.log import LogClient, PutLogsRequest, LogItem
3
+ from aliyun.log import LogClient, LogItem, PutLogsRequest
4
4
  from cobweb import setting
5
5
 
6
6
 
@@ -10,19 +10,21 @@ class LoghubDot:
10
10
  self.client = LogClient(**setting.LOGHUB_CONFIG)
11
11
 
12
12
  def build(self, topic, **kwargs):
13
+
13
14
  temp = {}
15
+ log_items = []
14
16
  log_item = LogItem()
15
17
  for key, value in kwargs.items():
16
18
  if not isinstance(value, str):
17
19
  temp[key] = json.dumps(value, ensure_ascii=False)
18
20
  contents = sorted(temp.items())
19
21
  log_item.set_contents(contents)
20
-
22
+ log_items.append(log_item)
21
23
  request = PutLogsRequest(
22
24
  project="databee-download-log",
23
25
  logstore="cobweb_log",
24
26
  topic=topic,
25
- logitems=[contents],
27
+ logitems=log_items,
26
28
  compress=True
27
29
  )
28
30
  self.client.put_logs(request=request)
{cobweb_launcher-1.2.32.dist-info → cobweb_launcher-1.2.34.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 1.2.32
3
+ Version: 1.2.34
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
{cobweb_launcher-1.2.32.dist-info → cobweb_launcher-1.2.34.dist-info}/RECORD RENAMED
@@ -11,7 +11,7 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
11
11
  cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
12
12
  cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
13
13
  cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
14
- cobweb/crawlers/crawler.py,sha256=mPRc9GBfWi5AoSxB1jlARxvG_AzsPVRFil5O8RnOxCY,7018
14
+ cobweb/crawlers/crawler.py,sha256=AyXXVQteTcl6BSRTnliFotEUe7UDYappOnhd3-Simjk,7108
15
15
  cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
16
16
  cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
17
17
  cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
@@ -31,7 +31,7 @@ cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3ot
31
31
  cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
32
32
  cobweb/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
33
33
  cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
34
- cobweb/utils/dotting.py,sha256=dANYZk4G2Lhz0dpN0Invrbk8mEZZGEMu8AYe-wtQTrw,770
34
+ cobweb/utils/dotting.py,sha256=RnZulCKvhbFFpVznnCBAGCTRNbpoFD7xvWw2LGnGW54,827
35
35
  cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
36
36
  cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
37
37
  cobweb_new/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
@@ -67,8 +67,8 @@ cobweb_new/utils/__init__.py,sha256=c9macpjc15hrCUCdzO5RR_sgK_B9kvJKreSGprZ1ld4,
67
67
  cobweb_new/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
68
68
  cobweb_new/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
69
69
  cobweb_new/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
70
- cobweb_launcher-1.2.32.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
71
- cobweb_launcher-1.2.32.dist-info/METADATA,sha256=uw1KrJ0tt_n0kidw854qOfEf6Hjlnpmt089bWcspc8Y,6510
72
- cobweb_launcher-1.2.32.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
73
- cobweb_launcher-1.2.32.dist-info/top_level.txt,sha256=UKwd478nkspJ6_fos3jtuc7SKXfmU42bJa_Ek5n2zsE,18
74
- cobweb_launcher-1.2.32.dist-info/RECORD,,
70
+ cobweb_launcher-1.2.34.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
71
+ cobweb_launcher-1.2.34.dist-info/METADATA,sha256=Dp0qVhxZq182YqyzkJtc-ayiYWwMWaUqYgwb_558N5k,6510
72
+ cobweb_launcher-1.2.34.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
73
+ cobweb_launcher-1.2.34.dist-info/top_level.txt,sha256=UKwd478nkspJ6_fos3jtuc7SKXfmU42bJa_Ek5n2zsE,18
74
+ cobweb_launcher-1.2.34.dist-info/RECORD,,