reyfetch 1.0.35__py3-none-any.whl → 1.0.37__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reyfetch/rali.py +28 -32
- reyfetch/rbaidu.py +23 -29
- reyfetch/rbase.py +2 -7
- reyfetch/rdouban.py +21 -26
- {reyfetch-1.0.35.dist-info → reyfetch-1.0.37.dist-info}/METADATA +1 -1
- reyfetch-1.0.37.dist-info/RECORD +14 -0
- reyfetch-1.0.35.dist-info/RECORD +0 -14
- {reyfetch-1.0.35.dist-info → reyfetch-1.0.37.dist-info}/WHEEL +0 -0
- {reyfetch-1.0.35.dist-info → reyfetch-1.0.37.dist-info}/licenses/LICENSE +0 -0
reyfetch/rali.py
CHANGED
@@ -22,7 +22,7 @@ from .rbase import FetchRequest, FetchRequestWithDatabase, FetchRequestDatabaseR
|
|
22
22
|
|
23
23
|
|
24
24
|
__all__ = (
|
25
|
-
'
|
25
|
+
'DatabaseORMTableAliQwen',
|
26
26
|
'FetchRequestAli',
|
27
27
|
'FetchRequestAliQwen'
|
28
28
|
)
|
@@ -55,9 +55,9 @@ ChatReplyGenerator = Generator[str, Any, None]
|
|
55
55
|
ChatThinkGenerator = Generator[str, Any, None]
|
56
56
|
|
57
57
|
|
58
|
-
class
|
58
|
+
class DatabaseORMTableAliQwen(rorm.Model, table=True):
|
59
59
|
"""
|
60
|
-
Database `ali_qwen` table model.
|
60
|
+
Database `ali_qwen` table ORM model.
|
61
61
|
"""
|
62
62
|
|
63
63
|
__name__ = 'ali_qwen'
|
@@ -86,22 +86,14 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
86
86
|
"""
|
87
87
|
Request Ali QWen API fetch type.
|
88
88
|
Can create database used `self.build_db` method.
|
89
|
-
|
90
|
-
Attributes
|
91
|
-
----------
|
92
|
-
url_api : API request URL.
|
93
|
-
url_doc : API document URL.
|
94
|
-
model = API AI model type.
|
95
|
-
db_names : Database table name mapping dictionary.
|
96
89
|
"""
|
97
90
|
|
98
91
|
url_api = 'https://dashscope.aliyuncs.com/api/v1/services/aigc/text-generation/generation'
|
92
|
+
'API request URL.'
|
99
93
|
url_doc = 'https://help.aliyun.com/zh/model-studio/use-qwen-by-calling-api?spm=a2c4g.11186623.0.0.330e7d9dSBCaZQ'
|
94
|
+
'API document URL.'
|
100
95
|
model = 'qwen-turbo-latest'
|
101
|
-
|
102
|
-
'ali_qwen': 'ali_qwen',
|
103
|
-
'stats_ali_qwen': 'stats_ali_qwen'
|
104
|
-
}
|
96
|
+
'API AI model type.'
|
105
97
|
|
106
98
|
|
107
99
|
def __init__(
|
@@ -141,7 +133,11 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
141
133
|
self.data: ChatRecordsData = {}
|
142
134
|
|
143
135
|
# Database.
|
144
|
-
self.db_record = FetchRequestDatabaseRecord(self, '
|
136
|
+
self.db_record = FetchRequestDatabaseRecord(self, 'ali_qwen')
|
137
|
+
|
138
|
+
## Build Database.
|
139
|
+
if self.db is not None:
|
140
|
+
self.build_db()
|
145
141
|
|
146
142
|
|
147
143
|
@overload
|
@@ -832,7 +828,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
832
828
|
|
833
829
|
def build_db(self) -> None:
|
834
830
|
"""
|
835
|
-
Check and build database tables
|
831
|
+
Check and build database tables.
|
836
832
|
"""
|
837
833
|
|
838
834
|
# Check.
|
@@ -840,21 +836,21 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
840
836
|
throw(ValueError, self.db)
|
841
837
|
|
842
838
|
# Parameter.
|
839
|
+
database = self.db.database
|
843
840
|
|
844
841
|
## Table.
|
845
|
-
tables = [
|
846
|
-
DatabaseTableAliQwen._set_name(self.db_names['ali_qwen'])
|
842
|
+
tables = [DatabaseORMTableAliQwen]
|
847
843
|
|
848
844
|
## View stats.
|
849
845
|
views_stats = [
|
850
846
|
{
|
851
|
-
'path':
|
847
|
+
'path': 'stats_ali_qwen',
|
852
848
|
'items': [
|
853
849
|
{
|
854
850
|
'name': 'count',
|
855
851
|
'select': (
|
856
852
|
'SELECT COUNT(1)\n'
|
857
|
-
f'FROM `{
|
853
|
+
f'FROM `{database}`.`ali_qwen`'
|
858
854
|
),
|
859
855
|
'comment': 'Request count.'
|
860
856
|
},
|
@@ -862,7 +858,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
862
858
|
'name': 'past_day_count',
|
863
859
|
'select': (
|
864
860
|
'SELECT COUNT(1)\n'
|
865
|
-
f'FROM `{
|
861
|
+
f'FROM `{database}`.`ali_qwen`'
|
866
862
|
'WHERE TIMESTAMPDIFF(DAY, `request_time`, NOW()) = 0'
|
867
863
|
),
|
868
864
|
'comment': 'Request count in the past day.'
|
@@ -871,7 +867,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
871
867
|
'name': 'past_week_count',
|
872
868
|
'select': (
|
873
869
|
'SELECT COUNT(1)\n'
|
874
|
-
f'FROM `{
|
870
|
+
f'FROM `{database}`.`ali_qwen`'
|
875
871
|
'WHERE TIMESTAMPDIFF(DAY, `request_time`, NOW()) <= 6'
|
876
872
|
),
|
877
873
|
'comment': 'Request count in the past week.'
|
@@ -880,7 +876,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
880
876
|
'name': 'past_month_count',
|
881
877
|
'select': (
|
882
878
|
'SELECT COUNT(1)\n'
|
883
|
-
f'FROM `{
|
879
|
+
f'FROM `{database}`.`ali_qwen`'
|
884
880
|
'WHERE TIMESTAMPDIFF(DAY, `request_time`, NOW()) <= 29'
|
885
881
|
),
|
886
882
|
'comment': 'Request count in the past month.'
|
@@ -889,7 +885,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
889
885
|
'name': 'total_token',
|
890
886
|
'select': (
|
891
887
|
'SELECT FORMAT(SUM(`token_total`), 0)\n'
|
892
|
-
f'FROM `{
|
888
|
+
f'FROM `{database}`.`ali_qwen`'
|
893
889
|
),
|
894
890
|
'comment': 'Usage total Token.'
|
895
891
|
},
|
@@ -897,7 +893,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
897
893
|
'name': 'total_token_input',
|
898
894
|
'select': (
|
899
895
|
'SELECT FORMAT(SUM(`token_input`), 0)\n'
|
900
|
-
f'FROM `{
|
896
|
+
f'FROM `{database}`.`ali_qwen`'
|
901
897
|
),
|
902
898
|
'comment': 'Usage input total Token.'
|
903
899
|
},
|
@@ -905,7 +901,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
905
901
|
'name': 'total_token_output',
|
906
902
|
'select': (
|
907
903
|
'SELECT FORMAT(SUM(`token_output`), 0)\n'
|
908
|
-
f'FROM `{
|
904
|
+
f'FROM `{database}`.`ali_qwen`'
|
909
905
|
),
|
910
906
|
'comment': 'Usage output total Token.'
|
911
907
|
},
|
@@ -913,7 +909,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
913
909
|
'name': 'total_token_output_think',
|
914
910
|
'select': (
|
915
911
|
'SELECT FORMAT(SUM(`token_output_think`), 0)\n'
|
916
|
-
f'FROM `{
|
912
|
+
f'FROM `{database}`.`ali_qwen`'
|
917
913
|
),
|
918
914
|
'comment': 'Usage output think total Token.'
|
919
915
|
},
|
@@ -921,7 +917,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
921
917
|
'name': 'avg_token',
|
922
918
|
'select': (
|
923
919
|
'SELECT FORMAT(AVG(`token_total`), 0)\n'
|
924
|
-
f'FROM `{
|
920
|
+
f'FROM `{database}`.`ali_qwen`'
|
925
921
|
),
|
926
922
|
'comment': 'Usage average Token.'
|
927
923
|
},
|
@@ -929,7 +925,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
929
925
|
'name': 'avg_token_input',
|
930
926
|
'select': (
|
931
927
|
'SELECT FORMAT(AVG(`token_input`), 0)\n'
|
932
|
-
f'FROM `{
|
928
|
+
f'FROM `{database}`.`ali_qwen`'
|
933
929
|
),
|
934
930
|
'comment': 'Usage input average Token.'
|
935
931
|
},
|
@@ -937,7 +933,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
937
933
|
'name': 'avg_token_output',
|
938
934
|
'select': (
|
939
935
|
'SELECT FORMAT(AVG(`token_output`), 0)\n'
|
940
|
-
f'FROM `{
|
936
|
+
f'FROM `{database}`.`ali_qwen`'
|
941
937
|
),
|
942
938
|
'comment': 'Usage output average Token.'
|
943
939
|
},
|
@@ -945,7 +941,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
945
941
|
'name': 'avg_token_output_think',
|
946
942
|
'select': (
|
947
943
|
'SELECT FORMAT(AVG(`token_output_think`), 0)\n'
|
948
|
-
f'FROM `{
|
944
|
+
f'FROM `{database}`.`ali_qwen`'
|
949
945
|
),
|
950
946
|
'comment': 'Usage output think average Token.'
|
951
947
|
},
|
@@ -953,7 +949,7 @@ class FetchRequestAliQwen(FetchRequestAli, FetchRequestWithDatabase):
|
|
953
949
|
'name': 'last_time',
|
954
950
|
'select': (
|
955
951
|
'SELECT MAX(`request_time`)\n'
|
956
|
-
f'FROM `{
|
952
|
+
f'FROM `{database}`.`ali_qwen`'
|
957
953
|
),
|
958
954
|
'comment': 'Last record request time.'
|
959
955
|
}
|
reyfetch/rbaidu.py
CHANGED
@@ -24,7 +24,7 @@ from .rbase import FetchRequest, FetchRequestWithDatabase, FetchRequestDatabaseR
|
|
24
24
|
|
25
25
|
|
26
26
|
__all__ = (
|
27
|
-
'
|
27
|
+
'DatabaseORMTableBaiduTrans',
|
28
28
|
'FetchRequestBaidu',
|
29
29
|
'FetchRequestBaiduTranslateLangEnum',
|
30
30
|
'FetchRequestBaiduTranslateLangAutoEnum',
|
@@ -37,9 +37,9 @@ FanyiResponseResult = TypedDict('FanyiResponseResult', {'src': str, 'dst': str})
|
|
37
37
|
FanyiResponse = TypedDict('FanyiResponse', {'from': str, 'to': str, 'trans_result': list[FanyiResponseResult]})
|
38
38
|
|
39
39
|
|
40
|
-
class
|
40
|
+
class DatabaseORMTableBaiduTrans(rorm.Model, table=True):
|
41
41
|
"""
|
42
|
-
Database `baidu_trans` table model.
|
42
|
+
Database `baidu_trans` table ORM model.
|
43
43
|
"""
|
44
44
|
|
45
45
|
__name__ = 'baidu_trans'
|
@@ -106,24 +106,14 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
106
106
|
"""
|
107
107
|
Request Baidu translate API fetch type.
|
108
108
|
Can create database used `self.build_db` method.
|
109
|
-
|
110
|
-
Attributes
|
111
|
-
----------
|
112
|
-
url_api : API request URL.
|
113
|
-
url_doc : API document URL.
|
114
|
-
LangEnum : Baidu Fanyi APT language enumeration type.
|
115
|
-
LangEnum : Baidu Fanyi APT language auto type enumeration.
|
116
|
-
db_names : Database table name mapping dictionary.
|
117
109
|
"""
|
118
110
|
|
119
111
|
url_api = 'http://api.fanyi.baidu.com/api/trans/vip/translate'
|
112
|
+
'API request URL.'
|
120
113
|
url_doc = 'https://fanyi-api.baidu.com/product/113'
|
114
|
+
'API document URL.'
|
121
115
|
LangEnum = FetchRequestBaiduTranslateLangEnum
|
122
116
|
LangAutoEnum = FetchRequestBaiduTranslateLangAutoEnum
|
123
|
-
db_names = {
|
124
|
-
'baidu_trans': 'baidu_trans',
|
125
|
-
'stats_baidu_trans': 'stats_baidu_trans'
|
126
|
-
}
|
127
117
|
|
128
118
|
|
129
119
|
def __init__(
|
@@ -151,7 +141,11 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
151
141
|
self.max_len = max_len
|
152
142
|
|
153
143
|
# Database.
|
154
|
-
self.db_record = FetchRequestDatabaseRecord(self, '
|
144
|
+
self.db_record = FetchRequestDatabaseRecord(self, 'baidu_trans')
|
145
|
+
|
146
|
+
## Build Database.
|
147
|
+
if self.db is not None:
|
148
|
+
self.build_db()
|
155
149
|
|
156
150
|
|
157
151
|
def sign(self, text: str, num: int) -> str:
|
@@ -331,7 +325,7 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
331
325
|
|
332
326
|
def build_db(self) -> None:
|
333
327
|
"""
|
334
|
-
Check and build database tables
|
328
|
+
Check and build database tables.
|
335
329
|
"""
|
336
330
|
|
337
331
|
# Check.
|
@@ -339,21 +333,21 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
339
333
|
throw(ValueError, self.db)
|
340
334
|
|
341
335
|
# Parameter.
|
336
|
+
database = self.db.database
|
342
337
|
|
343
338
|
## Table.
|
344
|
-
tables = [
|
345
|
-
DatabaseTableBaiduTrans._set_name(self.db_names['baidu_trans'])
|
339
|
+
tables = [DatabaseORMTableBaiduTrans]
|
346
340
|
|
347
341
|
## View stats.
|
348
342
|
views_stats = [
|
349
343
|
{
|
350
|
-
'path':
|
344
|
+
'path': 'stats_baidu_trans',
|
351
345
|
'items': [
|
352
346
|
{
|
353
347
|
'name': 'count',
|
354
348
|
'select': (
|
355
349
|
'SELECT COUNT(1)\n'
|
356
|
-
f'FROM `{
|
350
|
+
f'FROM `{database}`.`baidu_trans`'
|
357
351
|
),
|
358
352
|
'comment': 'Request count.'
|
359
353
|
},
|
@@ -361,7 +355,7 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
361
355
|
'name': 'past_day_count',
|
362
356
|
'select': (
|
363
357
|
'SELECT COUNT(1)\n'
|
364
|
-
f'FROM `{
|
358
|
+
f'FROM `{database}`.`baidu_trans`'
|
365
359
|
'WHERE TIMESTAMPDIFF(DAY, `request_time`, NOW()) = 0'
|
366
360
|
),
|
367
361
|
'comment': 'Request count in the past day.'
|
@@ -370,7 +364,7 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
370
364
|
'name': 'past_week_count',
|
371
365
|
'select': (
|
372
366
|
'SELECT COUNT(1)\n'
|
373
|
-
f'FROM `{
|
367
|
+
f'FROM `{database}`.`baidu_trans`'
|
374
368
|
'WHERE TIMESTAMPDIFF(DAY, `request_time`, NOW()) <= 6'
|
375
369
|
),
|
376
370
|
'comment': 'Request count in the past week.'
|
@@ -379,7 +373,7 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
379
373
|
'name': 'past_month_count',
|
380
374
|
'select': (
|
381
375
|
'SELECT COUNT(1)\n'
|
382
|
-
f'FROM `{
|
376
|
+
f'FROM `{database}`.`baidu_trans`'
|
383
377
|
'WHERE TIMESTAMPDIFF(DAY, `request_time`, NOW()) <= 29'
|
384
378
|
),
|
385
379
|
'comment': 'Request count in the past month.'
|
@@ -388,7 +382,7 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
388
382
|
'name': 'total_input',
|
389
383
|
'select': (
|
390
384
|
'SELECT FORMAT(SUM(LENGTH(`input`)), 0)\n'
|
391
|
-
f'FROM `{
|
385
|
+
f'FROM `{database}`.`baidu_trans`'
|
392
386
|
),
|
393
387
|
'comment': 'Input original text total character.'
|
394
388
|
},
|
@@ -396,7 +390,7 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
396
390
|
'name': 'total_output',
|
397
391
|
'select': (
|
398
392
|
'SELECT FORMAT(SUM(LENGTH(`output`)), 0)\n'
|
399
|
-
f'FROM `{
|
393
|
+
f'FROM `{database}`.`baidu_trans`'
|
400
394
|
),
|
401
395
|
'comment': 'Output translation text total character.'
|
402
396
|
},
|
@@ -404,7 +398,7 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
404
398
|
'name': 'avg_input',
|
405
399
|
'select': (
|
406
400
|
'SELECT FORMAT(AVG(LENGTH(`input`)), 0)\n'
|
407
|
-
f'FROM `{
|
401
|
+
f'FROM `{database}`.`baidu_trans`'
|
408
402
|
),
|
409
403
|
'comment': 'Input original text average character.'
|
410
404
|
},
|
@@ -412,7 +406,7 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
412
406
|
'name': 'avg_output',
|
413
407
|
'select': (
|
414
408
|
'SELECT FORMAT(AVG(LENGTH(`output`)), 0)\n'
|
415
|
-
f'FROM `{
|
409
|
+
f'FROM `{database}`.`baidu_trans`'
|
416
410
|
),
|
417
411
|
'comment': 'Output translation text average character.'
|
418
412
|
},
|
@@ -420,7 +414,7 @@ class FetchRequestBaiduTranslate(FetchRequestBaidu, FetchRequestWithDatabase):
|
|
420
414
|
'name': 'last_time',
|
421
415
|
'select': (
|
422
416
|
'SELECT MAX(`request_time`)\n'
|
423
|
-
f'FROM `{
|
417
|
+
f'FROM `{database}`.`baidu_trans`'
|
424
418
|
),
|
425
419
|
'comment': 'Last record request time.'
|
426
420
|
}
|
reyfetch/rbase.py
CHANGED
@@ -166,7 +166,6 @@ class FetchRequestWithDatabase(FetchRequest):
|
|
166
166
|
"""
|
167
167
|
|
168
168
|
db: Database | None
|
169
|
-
db_names: dict[str, str]
|
170
169
|
build_db: MethodType
|
171
170
|
|
172
171
|
|
@@ -179,7 +178,6 @@ class FetchRequestDatabaseRecord(FetchRequest):
|
|
179
178
|
def __init__(
|
180
179
|
self,
|
181
180
|
api: FetchRequestWithDatabase | None = None,
|
182
|
-
database: str | None = None,
|
183
181
|
table: str | None = None
|
184
182
|
) -> None:
|
185
183
|
"""
|
@@ -189,13 +187,11 @@ class FetchRequestDatabaseRecord(FetchRequest):
|
|
189
187
|
----------
|
190
188
|
api : `API` instance.
|
191
189
|
- `None`: Not record.
|
192
|
-
|
193
|
-
table : Index `API.db_names` table name.
|
190
|
+
table : Table name.
|
194
191
|
"""
|
195
192
|
|
196
193
|
# Build.
|
197
194
|
self.api = api
|
198
|
-
self.database = database
|
199
195
|
self.table = table
|
200
196
|
self.data: dict[int, dict[str, Any]] = {}
|
201
197
|
|
@@ -234,10 +230,9 @@ class FetchRequestDatabaseRecord(FetchRequest):
|
|
234
230
|
# Parameter.
|
235
231
|
thread_id = threading_get_ident()
|
236
232
|
record = self.data.setdefault(thread_id, {})
|
237
|
-
table = self.api.db_names[self.table]
|
238
233
|
|
239
234
|
# Insert.
|
240
|
-
self.api.db.execute.insert(table, record)
|
235
|
+
self.api.db.execute.insert(self.table, record)
|
241
236
|
|
242
237
|
# Delete.
|
243
238
|
del self.data[thread_id]
|
reyfetch/rdouban.py
CHANGED
@@ -21,7 +21,7 @@ from .rbase import FetchCrawl
|
|
21
21
|
|
22
22
|
|
23
23
|
__all__ = (
|
24
|
-
'
|
24
|
+
'DatabaseORMTableDoubanMedia',
|
25
25
|
'FetchCrawlDouban'
|
26
26
|
)
|
27
27
|
|
@@ -72,9 +72,9 @@ MediaInfo = TypedDict(
|
|
72
72
|
)
|
73
73
|
|
74
74
|
|
75
|
-
class
|
75
|
+
class DatabaseORMTableDoubanMedia(rorm.Model, table=True):
|
76
76
|
"""
|
77
|
-
Database `douban_media` table model.
|
77
|
+
Database `douban_media` table ORM model.
|
78
78
|
"""
|
79
79
|
|
80
80
|
__name__ = 'douban_media'
|
@@ -109,17 +109,8 @@ class FetchCrawlDouban(FetchCrawl):
|
|
109
109
|
"""
|
110
110
|
Crawl Douban Web fetch type.
|
111
111
|
Can create database used `self.build_db` method.
|
112
|
-
|
113
|
-
Attributes
|
114
|
-
----------
|
115
|
-
db_names : Database table name mapping dictionary.
|
116
112
|
"""
|
117
113
|
|
118
|
-
db_names = {
|
119
|
-
'douban_media': 'douban_media',
|
120
|
-
'stats_douban': 'stats_douban'
|
121
|
-
}
|
122
|
-
|
123
114
|
|
124
115
|
def __init__(self, db: Database | None = None) -> None:
|
125
116
|
"""
|
@@ -135,6 +126,10 @@ class FetchCrawlDouban(FetchCrawl):
|
|
135
126
|
# Build.
|
136
127
|
self.db = db
|
137
128
|
|
129
|
+
# Build Database.
|
130
|
+
if self.db is not None:
|
131
|
+
self.build_db()
|
132
|
+
|
138
133
|
|
139
134
|
def crawl_table(self) -> MediaTable:
|
140
135
|
"""
|
@@ -262,7 +257,7 @@ class FetchCrawlDouban(FetchCrawl):
|
|
262
257
|
'year'
|
263
258
|
)
|
264
259
|
self.db.execute.insert(
|
265
|
-
|
260
|
+
'douban_media',
|
266
261
|
table,
|
267
262
|
update_fields
|
268
263
|
)
|
@@ -434,7 +429,7 @@ class FetchCrawlDouban(FetchCrawl):
|
|
434
429
|
data = {'id': id_}
|
435
430
|
data.update(infos)
|
436
431
|
self.db.execute.insert(
|
437
|
-
|
432
|
+
'douban_media',
|
438
433
|
data,
|
439
434
|
'update'
|
440
435
|
)
|
@@ -472,7 +467,7 @@ class FetchCrawlDouban(FetchCrawl):
|
|
472
467
|
|
473
468
|
def build_db(self) -> None:
|
474
469
|
"""
|
475
|
-
Check and build database tables
|
470
|
+
Check and build database tables.
|
476
471
|
"""
|
477
472
|
|
478
473
|
# Check.
|
@@ -480,21 +475,21 @@ class FetchCrawlDouban(FetchCrawl):
|
|
480
475
|
throw(ValueError, self.db)
|
481
476
|
|
482
477
|
# Parameter.
|
478
|
+
database = self.db.database
|
483
479
|
|
484
480
|
## Table.
|
485
|
-
tables = [
|
486
|
-
DatabaseTableDoubanMedia._set_name(self.db_names['douban_media'])
|
481
|
+
tables = [DatabaseORMTableDoubanMedia]
|
487
482
|
|
488
483
|
## View stats.
|
489
484
|
views_stats = [
|
490
485
|
{
|
491
|
-
'path':
|
486
|
+
'path': 'stats_douban',
|
492
487
|
'items': [
|
493
488
|
{
|
494
489
|
'name': 'count',
|
495
490
|
'select': (
|
496
491
|
'SELECT COUNT(1)\n'
|
497
|
-
f'FROM `{
|
492
|
+
f'FROM `{database}`.`douban_media`'
|
498
493
|
),
|
499
494
|
'comment': 'Media count.'
|
500
495
|
},
|
@@ -502,7 +497,7 @@ class FetchCrawlDouban(FetchCrawl):
|
|
502
497
|
'name': 'past_day_count',
|
503
498
|
'select': (
|
504
499
|
'SELECT COUNT(1)\n'
|
505
|
-
f'FROM `{
|
500
|
+
f'FROM `{database}`.`douban_media`\n'
|
506
501
|
'WHERE TIMESTAMPDIFF(DAY, `create_time`, NOW()) = 0'
|
507
502
|
),
|
508
503
|
'comment': 'Media count in the past day.'
|
@@ -511,7 +506,7 @@ class FetchCrawlDouban(FetchCrawl):
|
|
511
506
|
'name': 'past_week_count',
|
512
507
|
'select': (
|
513
508
|
'SELECT COUNT(1)\n'
|
514
|
-
f'FROM `{
|
509
|
+
f'FROM `{database}`.`douban_media`\n'
|
515
510
|
'WHERE TIMESTAMPDIFF(DAY, `create_time`, NOW()) <= 6'
|
516
511
|
),
|
517
512
|
'comment': 'Media count in the past week.'
|
@@ -520,7 +515,7 @@ class FetchCrawlDouban(FetchCrawl):
|
|
520
515
|
'name': 'past_month_count',
|
521
516
|
'select': (
|
522
517
|
'SELECT COUNT(1)\n'
|
523
|
-
f'FROM `{
|
518
|
+
f'FROM `{database}`.`douban_media`\n'
|
524
519
|
'WHERE TIMESTAMPDIFF(DAY, `create_time`, NOW()) <= 29'
|
525
520
|
),
|
526
521
|
'comment': 'Media count in the past month.'
|
@@ -529,7 +524,7 @@ class FetchCrawlDouban(FetchCrawl):
|
|
529
524
|
'name': 'avg_score',
|
530
525
|
'select': (
|
531
526
|
'SELECT ROUND(AVG(`score`), 1)\n'
|
532
|
-
f'FROM `{
|
527
|
+
f'FROM `{database}`.`douban_media`'
|
533
528
|
),
|
534
529
|
'comment': 'Media average score.'
|
535
530
|
},
|
@@ -537,7 +532,7 @@ class FetchCrawlDouban(FetchCrawl):
|
|
537
532
|
'name': 'score_count',
|
538
533
|
'select': (
|
539
534
|
'SELECT FORMAT(SUM(`score_count`), 0)\n'
|
540
|
-
f'FROM `{
|
535
|
+
f'FROM `{database}`.`douban_media`'
|
541
536
|
),
|
542
537
|
'comment': 'Media score count.'
|
543
538
|
},
|
@@ -545,7 +540,7 @@ class FetchCrawlDouban(FetchCrawl):
|
|
545
540
|
'name': 'last_create_time',
|
546
541
|
'select': (
|
547
542
|
'SELECT MAX(`create_time`)\n'
|
548
|
-
f'FROM `{
|
543
|
+
f'FROM `{database}`.`douban_media`'
|
549
544
|
),
|
550
545
|
'comment': 'Media last record create time.'
|
551
546
|
},
|
@@ -553,7 +548,7 @@ class FetchCrawlDouban(FetchCrawl):
|
|
553
548
|
'name': 'last_update_time',
|
554
549
|
'select': (
|
555
550
|
'SELECT IFNULL(MAX(`update_time`), MAX(`create_time`))\n'
|
556
|
-
f'FROM `{
|
551
|
+
f'FROM `{database}`.`douban_media`'
|
557
552
|
),
|
558
553
|
'comment': 'Media last record update time.'
|
559
554
|
}
|
@@ -0,0 +1,14 @@
|
|
1
|
+
reyfetch/__init__.py,sha256=gDkQmFLZGToSqb1HVfvGxvc9t3mt8slg7_MVc44llns,484
|
2
|
+
reyfetch/rali.py,sha256=mvcY67THU7dYVku13r7LEC5hmZauI1NcLV7pLC6NC5o,33135
|
3
|
+
reyfetch/rall.py,sha256=86TMZiNsv-UinZ2L3_m3ugTHekUWnfVlWMedte0MOwQ,350
|
4
|
+
reyfetch/rbaidu.py,sha256=MIaJwk3RgaoCKj7ig3InhvlUV6iqzwkxnBoI8wtL1RU,13464
|
5
|
+
reyfetch/rbase.py,sha256=7sXZyTsu-Tw2cEdusUf5xiyPFE1e7lEx0W0Oc6pGKHE,4800
|
6
|
+
reyfetch/rdouban.py,sha256=O-NfGkwfBQKsv9fO8MxCYjuEEYb8PKuL_8xLhluZ2NE,18959
|
7
|
+
reyfetch/rgeneral.py,sha256=s270rtlVv1owkGF57Tjs8Oc5bJRD_XtWISpKlEJ7mbE,3797
|
8
|
+
reyfetch/rsina.py,sha256=WONT6NN67seYjlvXZ39RbibticdDAuauGk6SAvtdEq8,6889
|
9
|
+
reyfetch/rtoutiao.py,sha256=zLPnf2_XSC0d2RZgtMVoVyWpA2UDGNXYpRjYi3DGabI,1671
|
10
|
+
reyfetch/rweibo.py,sha256=T58q6wOv0p7j6WBH4we5k9x8IgFhQN__wqsqTHYMbmU,2522
|
11
|
+
reyfetch-1.0.37.dist-info/METADATA,sha256=g8agdbNqvRAnpa4U2zA_r_RWC0Mn9BzHSApIx9PvL4Q,1589
|
12
|
+
reyfetch-1.0.37.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
13
|
+
reyfetch-1.0.37.dist-info/licenses/LICENSE,sha256=UYLPqp7BvPiH8yEZduJqmmyEl6hlM3lKrFIefiD4rvk,1059
|
14
|
+
reyfetch-1.0.37.dist-info/RECORD,,
|
reyfetch-1.0.35.dist-info/RECORD
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
reyfetch/__init__.py,sha256=gDkQmFLZGToSqb1HVfvGxvc9t3mt8slg7_MVc44llns,484
|
2
|
-
reyfetch/rali.py,sha256=fdwraDilYcSnsCdV3FaLAT_g-legTufm0H73x8OQkx0,33667
|
3
|
-
reyfetch/rall.py,sha256=86TMZiNsv-UinZ2L3_m3ugTHekUWnfVlWMedte0MOwQ,350
|
4
|
-
reyfetch/rbaidu.py,sha256=vwVHiWxeaKnklGcORH2b4njmnmtAdWclSqDdJFNRirY,14023
|
5
|
-
reyfetch/rbase.py,sha256=Q9FldtjDHy3GSgGBv1qzheFKVBnQB9UvwFswbv6dQDA,5021
|
6
|
-
reyfetch/rdouban.py,sha256=X2wGolwPiYXHBJj06_u2dfNSKDtB68qJQvmij1ysUn8,19372
|
7
|
-
reyfetch/rgeneral.py,sha256=s270rtlVv1owkGF57Tjs8Oc5bJRD_XtWISpKlEJ7mbE,3797
|
8
|
-
reyfetch/rsina.py,sha256=WONT6NN67seYjlvXZ39RbibticdDAuauGk6SAvtdEq8,6889
|
9
|
-
reyfetch/rtoutiao.py,sha256=zLPnf2_XSC0d2RZgtMVoVyWpA2UDGNXYpRjYi3DGabI,1671
|
10
|
-
reyfetch/rweibo.py,sha256=T58q6wOv0p7j6WBH4we5k9x8IgFhQN__wqsqTHYMbmU,2522
|
11
|
-
reyfetch-1.0.35.dist-info/METADATA,sha256=JF2TLmNHfJ1mv5LHy8wIn6CvyV91RmP2bGA0SRFI9Mw,1589
|
12
|
-
reyfetch-1.0.35.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
13
|
-
reyfetch-1.0.35.dist-info/licenses/LICENSE,sha256=UYLPqp7BvPiH8yEZduJqmmyEl6hlM3lKrFIefiD4rvk,1059
|
14
|
-
reyfetch-1.0.35.dist-info/RECORD,,
|
File without changes
|
File without changes
|