libgenrs 0.1.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- libgenrs-0.1.9/LICENSE +19 -0
- libgenrs-0.1.9/PKG-INFO +183 -0
- libgenrs-0.1.9/README.md +161 -0
- libgenrs-0.1.9/libgenrs/__init__.py +9 -0
- libgenrs-0.1.9/libgenrs/download.py +139 -0
- libgenrs-0.1.9/libgenrs/search.py +323 -0
- libgenrs-0.1.9/libgenrs/utils.py +35 -0
- libgenrs-0.1.9/libgenrs.egg-info/PKG-INFO +183 -0
- libgenrs-0.1.9/libgenrs.egg-info/SOURCES.txt +16 -0
- libgenrs-0.1.9/libgenrs.egg-info/dependency_links.txt +1 -0
- libgenrs-0.1.9/libgenrs.egg-info/not-zip-safe +1 -0
- libgenrs-0.1.9/libgenrs.egg-info/requires.txt +6 -0
- libgenrs-0.1.9/libgenrs.egg-info/top_level.txt +2 -0
- libgenrs-0.1.9/setup.cfg +10 -0
- libgenrs-0.1.9/setup.py +40 -0
- libgenrs-0.1.9/tests/__init__.py +0 -0
- libgenrs-0.1.9/tests/libgen_test.py +50 -0
libgenrs-0.1.9/LICENSE
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
4
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
5
|
+
in the Software without restriction, including without limitation the rights
|
|
6
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
7
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
8
|
+
furnished to do so, subject to the following conditions:
|
|
9
|
+
|
|
10
|
+
The above copyright notice and this permission notice shall be included in all
|
|
11
|
+
copies or substantial portions of the Software.
|
|
12
|
+
|
|
13
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
14
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
15
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
16
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
17
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
18
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
19
|
+
SOFTWARE.
|
libgenrs-0.1.9/PKG-INFO
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: libgenrs
|
|
3
|
+
Version: 0.1.9
|
|
4
|
+
Summary: Asynchronous python lib for Libgen.rs
|
|
5
|
+
Home-page: https://github.com/ramanveerji/libgenrs
|
|
6
|
+
Download-URL: https://github.com/ramanveerji/libgenrs/archive/v0.1.9.tar.gz
|
|
7
|
+
License: MIT License
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Programming Language :: Python
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: aiofiles>=0.5.0
|
|
17
|
+
Requires-Dist: aiohttp>=3.6.0
|
|
18
|
+
Requires-Dist: beautifulsoup4>=4.9.0
|
|
19
|
+
Requires-Dist: lxml>=4.6.5
|
|
20
|
+
Requires-Dist: requests>=2.25.0
|
|
21
|
+
Requires-Dist: tldextract>=3.1.0
|
|
22
|
+
|
|
23
|
+
# libgenrs
|
|
24
|
+
|
|
25
|
+
Asynchronous python library for Libgen.is to search and download books.
|
|
26
|
+
This is a forked version
|
|
27
|
+
|
|
28
|
+
[](https://pypi.org/project/libgenrs)
|
|
29
|
+
[](https://github.com/ramanveerji/libgenrs/actions/workflows/python-publish.yml)
|
|
30
|
+
|
|
31
|
+
## Installing libgenrs
|
|
32
|
+
|
|
33
|
+
using the command line
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
pip install libgenrs
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Importing libgenrs
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from libgenrs import Libgen
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Creating libgenrs object
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
lg = Libgen()
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
or
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
lg = Libgen(sort= 'year', sort_mode= 'ASC', result_limit= '50')
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
When creating an instance of LibGen, you can set 3 option.
|
|
58
|
+
|
|
59
|
+
- **sort**: You can use this to choose a sorting method from allowed fields ( 'id', 'author', 'title', 'publisher', 'year', 'pages', 'language', 'size', 'extension'). Defaults to 'def'.
|
|
60
|
+
- **sort_mode**: Pick the order of the sort ascending or decending ('ASC', 'DESC'). Defaults to 'DESC'.
|
|
61
|
+
- **result_limit**: Limit the number of results based on the allowed limit (25, 50, 100). Defaults to 25.
|
|
62
|
+
|
|
63
|
+
## Searching for a book
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
async def search():
|
|
67
|
+
q = 'japan history'
|
|
68
|
+
result = await lg.search(q)
|
|
69
|
+
for item in result:
|
|
70
|
+
print('id = ' + item)
|
|
71
|
+
print('title = ' + result[item]['title'])
|
|
72
|
+
print('md5 = ' + result[item]['md5'])
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
The returned data looks like this: (The dict key is the Libgen id of the book)
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
{
|
|
79
|
+
'881061':{
|
|
80
|
+
'title': 'The Japanese Experience: A Short History of Japan (History of Civilisation)',
|
|
81
|
+
'volumeinfo': '',
|
|
82
|
+
'series': '',
|
|
83
|
+
'periodical': '',
|
|
84
|
+
'author': 'W. G. Beasley',
|
|
85
|
+
'year': '1999',
|
|
86
|
+
'edition': '',
|
|
87
|
+
'publisher': 'University of California Press',
|
|
88
|
+
'city': '',
|
|
89
|
+
'pages': '331',
|
|
90
|
+
'language': 'English',
|
|
91
|
+
'topic': '',
|
|
92
|
+
'library': '',
|
|
93
|
+
'issue': '0',
|
|
94
|
+
'identifier': '0520220501, 9780520220508',
|
|
95
|
+
'issn': '',
|
|
96
|
+
'asin': '',
|
|
97
|
+
'udc': '',
|
|
98
|
+
'lbc': '',
|
|
99
|
+
'ddc': '',
|
|
100
|
+
'lcc': '',
|
|
101
|
+
'doi': '',
|
|
102
|
+
'googlebookid': '',
|
|
103
|
+
'openlibraryid': '',
|
|
104
|
+
'commentary': '',
|
|
105
|
+
'dpi': '0',
|
|
106
|
+
'color': '',
|
|
107
|
+
'cleaned': '',
|
|
108
|
+
'orientation': '',
|
|
109
|
+
'paginated': '',
|
|
110
|
+
'scanned': '',
|
|
111
|
+
'bookmarked': '',
|
|
112
|
+
'searchable': '1',
|
|
113
|
+
'filesize': '29750320',
|
|
114
|
+
'extension': 'pdf',
|
|
115
|
+
'md5': 'a382109f7fdde3be5b2cb4f82d97443b',
|
|
116
|
+
'generic': '',
|
|
117
|
+
'visible': '',
|
|
118
|
+
'local': '0',
|
|
119
|
+
'timeadded': '2013-02-15 18:40:21',
|
|
120
|
+
'timelastmodified': '2019-12-21 21:23:21',
|
|
121
|
+
'coverurl': 'http://libgen.is/covers/881000/a382109f7fdde3be5b2cb4f82d97443b-g.jpg',
|
|
122
|
+
'identifierwodash': '0520220501,9780520220508',
|
|
123
|
+
'tags': '',
|
|
124
|
+
'pagesinfile': '331',
|
|
125
|
+
'descr': '',
|
|
126
|
+
'toc': '',
|
|
127
|
+
'sha1': 'ZZYRRG56BOX3XAQ5D2IWAV2FUUC35ELG',
|
|
128
|
+
'sha256': '3B40D6524409A18573900B729AB1BFB872E26CF111A4845F375A84BD0CB12460',
|
|
129
|
+
'crc32': '3DB84D01',
|
|
130
|
+
'edonkey': 'A63EEBB71C46DAE130725C07F2CDC67C',
|
|
131
|
+
'aich': '733CLKTMCYOGD4W5PIG2GMA7CMLAFN2V',
|
|
132
|
+
'tth': 'O4O5Z7UL2YAOUG57PINCLOAN63HVZPCSDYACT6Q',
|
|
133
|
+
'btih': '84195af51fa738cb232297dd1376df6d8b8313be',
|
|
134
|
+
'mirrors':
|
|
135
|
+
{
|
|
136
|
+
'main': 'http://library.lol/main/a382109f7fdde3be5b2cb4f82d97443b',
|
|
137
|
+
'libgen.lc': 'http://libgen.lc/ads.php?md5=a382109f7fdde3be5b2cb4f82d97443b',
|
|
138
|
+
'z-library': 'http://b-ok.cc/md5/a382109f7fdde3be5b2cb4f82d97443b',
|
|
139
|
+
'libgen.pw': 'https://libgen.pw/item?id=881061',
|
|
140
|
+
'bookfi': 'http://bookfi.net/md5/a382109f7fdde3be5b2cb4f82d97443b',
|
|
141
|
+
'torrent': 'http://libgen.is/book/index.php?md5=a382109f7fdde3be5b2cb4f82d97443b&oftorrent=',
|
|
142
|
+
'torrent_1k': 'http://libgen.is/repository_torrent/r_881000.torrent',
|
|
143
|
+
'gnutella': 'magnet:?xt=urn:sha1:ZZYRRG56BOX3XAQ5D2IWAV2FUUC35ELG&xl=29750320&dn=a382109f7fdde3be5b2cb4f82d97443b.pdf',
|
|
144
|
+
'ed2k': 'ed2k://|file|A382109F7FDDE3BE5B2CB4F82D97443B.pdf|29750320|A63EEBB71C46DAE130725C07F2CDC67C|h=733CLKTMCYOGD4W5PIG2GMA7CMLAFN2V|/',
|
|
145
|
+
'dc++': 'magnet:?xt=urn:tree:tiger:O4O5Z7UL2YAOUG57PINCLOAN63HVZPCSDYACT6Q&xl=29750320&dn=a382109f7fdde3be5b2cb4f82d97443b.pdf'
|
|
146
|
+
}
|
|
147
|
+
},
|
|
148
|
+
'id_of_book':{
|
|
149
|
+
...
|
|
150
|
+
...
|
|
151
|
+
...
|
|
152
|
+
},
|
|
153
|
+
'id_of_book':{
|
|
154
|
+
...
|
|
155
|
+
...
|
|
156
|
+
...
|
|
157
|
+
},
|
|
158
|
+
}
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Note that the above information except for mirrors is fecthed as is from Libgen and it is up to the uploader of the books to include the metadata of the books.
|
|
162
|
+
|
|
163
|
+
## Downloading a book
|
|
164
|
+
|
|
165
|
+
Downloading is only supported for the top **3** mirrors in the returned result ('main', 'libgen.lc', 'z-library'). Pass one of the 3 links to the download method of the Libgen object. Optionaly a destination Path can be passed with the url using **dest_folder** argument. The output of the download method is a Path to the downloaded file.
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
async def download():
|
|
169
|
+
q = 'japan history'
|
|
170
|
+
result = await lg.search(q)
|
|
171
|
+
download_location = []
|
|
172
|
+
path = Path('Downloads')
|
|
173
|
+
for item in result:
|
|
174
|
+
file_path = await lg.download(result[item]['mirrors']['main'],
|
|
175
|
+
dest_folder=path,
|
|
176
|
+
progress=progress,
|
|
177
|
+
progress_args=[
|
|
178
|
+
result[item]['title']
|
|
179
|
+
])
|
|
180
|
+
download_location.append(file_path)
|
|
181
|
+
|
|
182
|
+
async def progress(current, total, title):
|
|
183
|
+
print('Downloading ', current, ' of ', total, ' ', title)
|
libgenrs-0.1.9/README.md
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# libgenrs
|
|
2
|
+
|
|
3
|
+
Asynchronous python library for Libgen.is to search and download books.
|
|
4
|
+
This is a forked version
|
|
5
|
+
|
|
6
|
+
[](https://pypi.org/project/libgenrs)
|
|
7
|
+
[](https://github.com/ramanveerji/libgenrs/actions/workflows/python-publish.yml)
|
|
8
|
+
|
|
9
|
+
## Installing libgenrs
|
|
10
|
+
|
|
11
|
+
using the command line
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
pip install libgenrs
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Importing libgenrs
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
from libgenrs import Libgen
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Creating libgenrs object
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
lg = Libgen()
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
or
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
lg = Libgen(sort= 'year', sort_mode= 'ASC', result_limit= '50')
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
When creating an instance of LibGen, you can set 3 option.
|
|
36
|
+
|
|
37
|
+
- **sort**: You can use this to choose a sorting method from allowed fields ( 'id', 'author', 'title', 'publisher', 'year', 'pages', 'language', 'size', 'extension'). Defaults to 'def'.
|
|
38
|
+
- **sort_mode**: Pick the order of the sort ascending or decending ('ASC', 'DESC'). Defaults to 'DESC'.
|
|
39
|
+
- **result_limit**: Limit the number of results based on the allowed limit (25, 50, 100). Defaults to 25.
|
|
40
|
+
|
|
41
|
+
## Searching for a book
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
async def search():
|
|
45
|
+
q = 'japan history'
|
|
46
|
+
result = await lg.search(q)
|
|
47
|
+
for item in result:
|
|
48
|
+
print('id = ' + item)
|
|
49
|
+
print('title = ' + result[item]['title'])
|
|
50
|
+
print('md5 = ' + result[item]['md5'])
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The returned data looks like this: (The dict key is the Libgen id of the book)
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
{
|
|
57
|
+
'881061':{
|
|
58
|
+
'title': 'The Japanese Experience: A Short History of Japan (History of Civilisation)',
|
|
59
|
+
'volumeinfo': '',
|
|
60
|
+
'series': '',
|
|
61
|
+
'periodical': '',
|
|
62
|
+
'author': 'W. G. Beasley',
|
|
63
|
+
'year': '1999',
|
|
64
|
+
'edition': '',
|
|
65
|
+
'publisher': 'University of California Press',
|
|
66
|
+
'city': '',
|
|
67
|
+
'pages': '331',
|
|
68
|
+
'language': 'English',
|
|
69
|
+
'topic': '',
|
|
70
|
+
'library': '',
|
|
71
|
+
'issue': '0',
|
|
72
|
+
'identifier': '0520220501, 9780520220508',
|
|
73
|
+
'issn': '',
|
|
74
|
+
'asin': '',
|
|
75
|
+
'udc': '',
|
|
76
|
+
'lbc': '',
|
|
77
|
+
'ddc': '',
|
|
78
|
+
'lcc': '',
|
|
79
|
+
'doi': '',
|
|
80
|
+
'googlebookid': '',
|
|
81
|
+
'openlibraryid': '',
|
|
82
|
+
'commentary': '',
|
|
83
|
+
'dpi': '0',
|
|
84
|
+
'color': '',
|
|
85
|
+
'cleaned': '',
|
|
86
|
+
'orientation': '',
|
|
87
|
+
'paginated': '',
|
|
88
|
+
'scanned': '',
|
|
89
|
+
'bookmarked': '',
|
|
90
|
+
'searchable': '1',
|
|
91
|
+
'filesize': '29750320',
|
|
92
|
+
'extension': 'pdf',
|
|
93
|
+
'md5': 'a382109f7fdde3be5b2cb4f82d97443b',
|
|
94
|
+
'generic': '',
|
|
95
|
+
'visible': '',
|
|
96
|
+
'local': '0',
|
|
97
|
+
'timeadded': '2013-02-15 18:40:21',
|
|
98
|
+
'timelastmodified': '2019-12-21 21:23:21',
|
|
99
|
+
'coverurl': 'http://libgen.is/covers/881000/a382109f7fdde3be5b2cb4f82d97443b-g.jpg',
|
|
100
|
+
'identifierwodash': '0520220501,9780520220508',
|
|
101
|
+
'tags': '',
|
|
102
|
+
'pagesinfile': '331',
|
|
103
|
+
'descr': '',
|
|
104
|
+
'toc': '',
|
|
105
|
+
'sha1': 'ZZYRRG56BOX3XAQ5D2IWAV2FUUC35ELG',
|
|
106
|
+
'sha256': '3B40D6524409A18573900B729AB1BFB872E26CF111A4845F375A84BD0CB12460',
|
|
107
|
+
'crc32': '3DB84D01',
|
|
108
|
+
'edonkey': 'A63EEBB71C46DAE130725C07F2CDC67C',
|
|
109
|
+
'aich': '733CLKTMCYOGD4W5PIG2GMA7CMLAFN2V',
|
|
110
|
+
'tth': 'O4O5Z7UL2YAOUG57PINCLOAN63HVZPCSDYACT6Q',
|
|
111
|
+
'btih': '84195af51fa738cb232297dd1376df6d8b8313be',
|
|
112
|
+
'mirrors':
|
|
113
|
+
{
|
|
114
|
+
'main': 'http://library.lol/main/a382109f7fdde3be5b2cb4f82d97443b',
|
|
115
|
+
'libgen.lc': 'http://libgen.lc/ads.php?md5=a382109f7fdde3be5b2cb4f82d97443b',
|
|
116
|
+
'z-library': 'http://b-ok.cc/md5/a382109f7fdde3be5b2cb4f82d97443b',
|
|
117
|
+
'libgen.pw': 'https://libgen.pw/item?id=881061',
|
|
118
|
+
'bookfi': 'http://bookfi.net/md5/a382109f7fdde3be5b2cb4f82d97443b',
|
|
119
|
+
'torrent': 'http://libgen.is/book/index.php?md5=a382109f7fdde3be5b2cb4f82d97443b&oftorrent=',
|
|
120
|
+
'torrent_1k': 'http://libgen.is/repository_torrent/r_881000.torrent',
|
|
121
|
+
'gnutella': 'magnet:?xt=urn:sha1:ZZYRRG56BOX3XAQ5D2IWAV2FUUC35ELG&xl=29750320&dn=a382109f7fdde3be5b2cb4f82d97443b.pdf',
|
|
122
|
+
'ed2k': 'ed2k://|file|A382109F7FDDE3BE5B2CB4F82D97443B.pdf|29750320|A63EEBB71C46DAE130725C07F2CDC67C|h=733CLKTMCYOGD4W5PIG2GMA7CMLAFN2V|/',
|
|
123
|
+
'dc++': 'magnet:?xt=urn:tree:tiger:O4O5Z7UL2YAOUG57PINCLOAN63HVZPCSDYACT6Q&xl=29750320&dn=a382109f7fdde3be5b2cb4f82d97443b.pdf'
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
'id_of_book':{
|
|
127
|
+
...
|
|
128
|
+
...
|
|
129
|
+
...
|
|
130
|
+
},
|
|
131
|
+
'id_of_book':{
|
|
132
|
+
...
|
|
133
|
+
...
|
|
134
|
+
...
|
|
135
|
+
},
|
|
136
|
+
}
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Note that the above information except for mirrors is fecthed as is from Libgen and it is up to the uploader of the books to include the metadata of the books.
|
|
140
|
+
|
|
141
|
+
## Downloading a book
|
|
142
|
+
|
|
143
|
+
Downloading is only supported for the top **3** mirrors in the returned result ('main', 'libgen.lc', 'z-library'). Pass one of the 3 links to the download method of the Libgen object. Optionaly a destination Path can be passed with the url using **dest_folder** argument. The output of the download method is a Path to the downloaded file.
|
|
144
|
+
|
|
145
|
+
```python
|
|
146
|
+
async def download():
|
|
147
|
+
q = 'japan history'
|
|
148
|
+
result = await lg.search(q)
|
|
149
|
+
download_location = []
|
|
150
|
+
path = Path('Downloads')
|
|
151
|
+
for item in result:
|
|
152
|
+
file_path = await lg.download(result[item]['mirrors']['main'],
|
|
153
|
+
dest_folder=path,
|
|
154
|
+
progress=progress,
|
|
155
|
+
progress_args=[
|
|
156
|
+
result[item]['title']
|
|
157
|
+
])
|
|
158
|
+
download_location.append(file_path)
|
|
159
|
+
|
|
160
|
+
async def progress(current, total, title):
|
|
161
|
+
print('Downloading ', current, ' of ', total, ' ', title)
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import time
|
|
3
|
+
import aiohttp
|
|
4
|
+
import aiofiles
|
|
5
|
+
import logging
|
|
6
|
+
import requests
|
|
7
|
+
from .utils import Util
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from tldextract import extract
|
|
10
|
+
from bs4 import BeautifulSoup as bsoup
|
|
11
|
+
from typing import Awaitable, Callable, Optional
|
|
12
|
+
|
|
13
|
+
logg = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
class LibgenDownload:
|
|
16
|
+
def __init__(self) -> None:
|
|
17
|
+
self.dest_folder = Path.cwd()
|
|
18
|
+
self.mirrors = ['library.lol', 'libgen.lc', 'libgen.gs', 'b-ok.cc']
|
|
19
|
+
self.regex = re.compile(
|
|
20
|
+
r'^(?:http|ftp)s?://' # http:// or https:// domain...
|
|
21
|
+
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'
|
|
22
|
+
r'localhost|' # localhost...
|
|
23
|
+
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
|
|
24
|
+
r'(?::\d+)?' # optional port
|
|
25
|
+
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
|
|
26
|
+
|
|
27
|
+
async def download(self,
|
|
28
|
+
url: str,
|
|
29
|
+
dest_folder: Path = None,
|
|
30
|
+
progress: Optional[Callable[..., Awaitable[None]]] = None,
|
|
31
|
+
progress_args: list = []) -> Path:
|
|
32
|
+
|
|
33
|
+
if not re.match(self.regex, url) or \
|
|
34
|
+
(str(extract(url).domain) + '.' + str(extract(url).suffix)) not in self.mirrors:
|
|
35
|
+
raise Exception(f'Supported links are {" - ".join(self.mirrors)}')
|
|
36
|
+
|
|
37
|
+
if not dest_folder:
|
|
38
|
+
dest_folder = self.dest_folder
|
|
39
|
+
if not Path.is_dir(dest_folder):
|
|
40
|
+
Path.mkdir(dest_folder)
|
|
41
|
+
|
|
42
|
+
direct_links = await self.get_directlink(url)
|
|
43
|
+
for link in direct_links:
|
|
44
|
+
file = await self.__download(link,
|
|
45
|
+
dest_folder,
|
|
46
|
+
progress,
|
|
47
|
+
progress_args)
|
|
48
|
+
if not file:
|
|
49
|
+
continue
|
|
50
|
+
return file
|
|
51
|
+
logg.error('Could not download the book from the given url.')
|
|
52
|
+
return None
|
|
53
|
+
|
|
54
|
+
@staticmethod
|
|
55
|
+
async def __download(url: str,
|
|
56
|
+
dest_folder: Path,
|
|
57
|
+
progress: Optional[Callable[..., Awaitable[None]]],
|
|
58
|
+
progress_args: list) -> Path:
|
|
59
|
+
|
|
60
|
+
try:
|
|
61
|
+
async with aiohttp.ClientSession() as dl_ses:
|
|
62
|
+
async with dl_ses.get(url) as resp:
|
|
63
|
+
total_size = int(resp.headers.get('Content-Length')) if resp.headers.get('Content-Length') \
|
|
64
|
+
else 'Unknown size'
|
|
65
|
+
file_name = await Util().get_filename(resp.headers.get('Content-Disposition'))
|
|
66
|
+
temp_file = Path.joinpath(dest_folder, file_name)
|
|
67
|
+
|
|
68
|
+
async with aiofiles.open(temp_file, mode="wb") as dl_file:
|
|
69
|
+
current = 0
|
|
70
|
+
logg.info(
|
|
71
|
+
f'Starting download: {file_name}')
|
|
72
|
+
st_time = time.time()
|
|
73
|
+
async for chunk, _ in resp.content.iter_chunks():
|
|
74
|
+
await dl_file.write(chunk)
|
|
75
|
+
current += len(chunk)
|
|
76
|
+
cr_time = time.time()
|
|
77
|
+
if cr_time - st_time > 1:
|
|
78
|
+
if progress:
|
|
79
|
+
await progress(current, total_size, *progress_args)
|
|
80
|
+
logg.debug(
|
|
81
|
+
f'Downloading: {current} of {total_size} Done.')
|
|
82
|
+
st_time = cr_time
|
|
83
|
+
except Exception as e:
|
|
84
|
+
logg.exception(e)
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
return temp_file
|
|
88
|
+
|
|
89
|
+
async def get_directlink(self,
|
|
90
|
+
url: str) -> list:
|
|
91
|
+
|
|
92
|
+
tld = extract(url)
|
|
93
|
+
r = requests.get(url,
|
|
94
|
+
allow_redirects=True)
|
|
95
|
+
logg.debug(
|
|
96
|
+
f'Requesting download page resulted in code: {r.status_code}')
|
|
97
|
+
if r.status_code != 200:
|
|
98
|
+
await Util().raise_error(r.status_code, r.reason)
|
|
99
|
+
|
|
100
|
+
soup = bsoup(r.content, 'lxml')
|
|
101
|
+
for s in soup.findAll('script'):
|
|
102
|
+
s.decompose()
|
|
103
|
+
|
|
104
|
+
domain = str(tld.domain) + '.' + str(tld.suffix)
|
|
105
|
+
direct_links = []
|
|
106
|
+
if domain == 'library.lol':
|
|
107
|
+
div = soup.find('div', attrs={'id': 'download'})
|
|
108
|
+
for li in div.ul.findAll('li'):
|
|
109
|
+
direct_links.append(li.a['href'])
|
|
110
|
+
direct_links.append(div.h2.a['href'])
|
|
111
|
+
elif domain == 'libgen.lc' or domain == 'libgen.gs':
|
|
112
|
+
table = soup.find('table', attrs={'id': 'main'})
|
|
113
|
+
direct_links.append(table.tr.findAll('td')[1].a['href'])
|
|
114
|
+
elif domain == 'b-ok.cc':
|
|
115
|
+
table = soup.find('table', attrs={'class': 'resItemTable'})
|
|
116
|
+
if table:
|
|
117
|
+
zl_page = 'http://b-ok.cc' + \
|
|
118
|
+
str(table.tr.findAll('td')[1].table.tr.td.h3.a['href'])
|
|
119
|
+
zl_r = requests.get(zl_page,
|
|
120
|
+
allow_redirects=True)
|
|
121
|
+
logg.debug(
|
|
122
|
+
f'Requesting b-ok page resulted in code: {zl_r.status_code}')
|
|
123
|
+
if zl_r.status_code != 200:
|
|
124
|
+
await Util().raise_error(zl_r.status_code, r.reason)
|
|
125
|
+
|
|
126
|
+
zl_soup = bsoup(zl_r.content, 'lxml')
|
|
127
|
+
for s in zl_soup.findAll('script'):
|
|
128
|
+
s.decompose()
|
|
129
|
+
|
|
130
|
+
direct_links.append(zl_soup.find(
|
|
131
|
+
'a', attrs={'class': 'btn btn-primary dlButton addDownloadedBook'}))
|
|
132
|
+
else:
|
|
133
|
+
await Util().raise_error(0, 'Book not found on Z-library')
|
|
134
|
+
else:
|
|
135
|
+
logg.error(
|
|
136
|
+
f'Unsuportted link. Only support download from {",".join(self.mirrors)}')
|
|
137
|
+
return direct_links
|
|
138
|
+
|
|
139
|
+
return direct_links
|
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import logging
|
|
3
|
+
import requests
|
|
4
|
+
from .utils import Util
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from .download import LibgenDownload
|
|
7
|
+
from bs4 import BeautifulSoup as bsoup
|
|
8
|
+
from typing import Awaitable, Callable, Optional
|
|
9
|
+
|
|
10
|
+
logg = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
class Libgen:
|
|
13
|
+
def __init__(self,
|
|
14
|
+
sort: str = 'def',
|
|
15
|
+
sort_mode: str = 'DESC',
|
|
16
|
+
result_limit: int = 25) -> None:
|
|
17
|
+
"""This class contains an async method to search Library Genesis and return a dictionary of search
|
|
18
|
+
results using the results id as the dictinary key.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
|
|
22
|
+
sort (str): It is used to sort the results based on the following allowed column values (based on libgen) - id, author, title, publisher, year, pages, language, size, extension.
|
|
23
|
+
Default value = 'def'
|
|
24
|
+
|
|
25
|
+
sort_mode (str): It is used to sort the results in ascending or descending order based on the the column chosen with sort parameter.
|
|
26
|
+
Default vaule = 'DESC'
|
|
27
|
+
|
|
28
|
+
result_limit (int): It is used to limit the returned values. Allowed values (based on libgen) are 25, 50, 100. If other than the allowed number in given it will return 25 results.
|
|
29
|
+
Default vaule = 25
|
|
30
|
+
|
|
31
|
+
Methods:
|
|
32
|
+
search -> dict
|
|
33
|
+
|
|
34
|
+
Retrun:
|
|
35
|
+
An object with a search and download method.
|
|
36
|
+
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
if sort.lower() in ['def', 'id', 'author', 'title', 'publisher', 'year', 'pages', 'language', 'size', 'extension']:
|
|
40
|
+
self.sort = sort.lower()
|
|
41
|
+
else:
|
|
42
|
+
raise ValueError(
|
|
43
|
+
f'sort parameter invalid. Allowed value is one of these (def, id, author, title, publisher, year, pages, language, size, extension)'
|
|
44
|
+
)
|
|
45
|
+
if sort_mode in ['ASC', 'DESC']:
|
|
46
|
+
self.sort_mode = sort_mode.upper()
|
|
47
|
+
else:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f'sort_mode parameter invalid. Allowed vaule is one of these (ASC, DESC)'
|
|
50
|
+
)
|
|
51
|
+
self.result_limit = result_limit
|
|
52
|
+
self.__fields = ['def', 'title', 'author', 'series', 'publisher', 'year',
|
|
53
|
+
'identifier', 'language', 'md5', 'tags', 'extension']
|
|
54
|
+
self.__libgen_url = 'http://libgen.is'
|
|
55
|
+
self.__json_url = f'{self.__libgen_url}/json.php?'
|
|
56
|
+
self.__search_url = f'{self.__libgen_url}/search.php?'
|
|
57
|
+
|
|
58
|
+
self.__ses = requests.Session()
|
|
59
|
+
self.__ses.cookies.set('lg_topic', 'libgen',
|
|
60
|
+
domain='libgen.is', expires=None)
|
|
61
|
+
self.__ses.headers.update({
|
|
62
|
+
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
|
63
|
+
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
|
64
|
+
'Chrome/86.0.4240.198 Safari/537.36'
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
async def search(self,
|
|
68
|
+
query: str,
|
|
69
|
+
search_field: str = 'def',
|
|
70
|
+
filters: dict = {},
|
|
71
|
+
return_fields: list = []) -> dict:
|
|
72
|
+
"""A method used to search libgen.rs with diffrent filter and search fields.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
|
|
76
|
+
query (str): The text to be searched
|
|
77
|
+
|
|
78
|
+
search_field (str, optional): The colomun to search the query from, allowed columns to search are,
|
|
79
|
+
|
|
80
|
+
'def' = default search according to relevance
|
|
81
|
+
'title' = search the query from the title column of the books
|
|
82
|
+
'author' = search the query from the author column of the books
|
|
83
|
+
'series' = search the query from the series column of the books
|
|
84
|
+
'publisher' = search the query from the publisher column of the books
|
|
85
|
+
'year' = search the query from the year column of the books
|
|
86
|
+
'identifier' = search the query from the identifier column of the books
|
|
87
|
+
'language' = search the query from the language column of the books
|
|
88
|
+
'md5' = search the query from the md5 column of the books
|
|
89
|
+
'tags' = search the query from the tags column of the books
|
|
90
|
+
'extension' = search the query from the extention column of the books
|
|
91
|
+
|
|
92
|
+
filters (dict, optional): filter the results based on a given criterial. Works on every returned fields. Becareful when using this, it might decrese the limit of the returned results
|
|
93
|
+
|
|
94
|
+
Example;
|
|
95
|
+
{
|
|
96
|
+
'year': '2009'
|
|
97
|
+
'extention': 'pdf'
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return_fields (list, optional): limit the reurned fields to a given fields in the list. Can pick from aviliable return fields.
|
|
101
|
+
|
|
102
|
+
Example;
|
|
103
|
+
[
|
|
104
|
+
'id'
|
|
105
|
+
'title'
|
|
106
|
+
'md5'
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
Raises:
|
|
110
|
+
|
|
111
|
+
ValueError: if query is empty.
|
|
112
|
+
ValueError: if the query is less than 2 characters.
|
|
113
|
+
ValueError: if search_field parameter is not from the allowed columns
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
|
|
117
|
+
dict: returns a dictionary of dictionaries where the 'id' of the book is the key to the detailed result dictionary
|
|
118
|
+
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
if query:
|
|
122
|
+
if len(query.strip()) < 2:
|
|
123
|
+
raise ValueError(
|
|
124
|
+
f'The query "{query}" is less than 2 characters.')
|
|
125
|
+
req = 'req=' + '+'.join(query.strip().split(' '))
|
|
126
|
+
else:
|
|
127
|
+
raise ValueError(f'Query not set.')
|
|
128
|
+
|
|
129
|
+
if search_field.lower() in self.__fields:
|
|
130
|
+
column = 'column=' + search_field.lower()
|
|
131
|
+
else:
|
|
132
|
+
raise ValueError(
|
|
133
|
+
f'search_field invalid. Allowed fields are {",".join(self.__fields)}')
|
|
134
|
+
|
|
135
|
+
sort = 'sort=' + self.sort
|
|
136
|
+
sort_mode = 'sortmode=' + self.sort_mode
|
|
137
|
+
res = 'res=' + str(self.result_limit)
|
|
138
|
+
|
|
139
|
+
url = '&'.join([self.__search_url, req, res, column, sort, sort_mode])
|
|
140
|
+
|
|
141
|
+
return await self.__search(url,
|
|
142
|
+
filters,
|
|
143
|
+
return_fields)
|
|
144
|
+
|
|
145
|
+
async def __search(self,
|
|
146
|
+
url: str,
|
|
147
|
+
filters: dict,
|
|
148
|
+
return_fields: list) -> dict:
|
|
149
|
+
|
|
150
|
+
ids_list = await self.__get_ids(url=url)
|
|
151
|
+
data = {}
|
|
152
|
+
if ids_list:
|
|
153
|
+
data = await self.__get_json(ids_list=ids_list,
|
|
154
|
+
return_fields=return_fields,
|
|
155
|
+
filters=filters)
|
|
156
|
+
return data
|
|
157
|
+
|
|
158
|
+
async def __get_ids(self,
|
|
159
|
+
url: str) -> list:
|
|
160
|
+
|
|
161
|
+
r = self.__ses.get(url=url,
|
|
162
|
+
allow_redirects=True)
|
|
163
|
+
logg.debug(
|
|
164
|
+
f'Requesting IDs page resulted in code: {r.status_code}')
|
|
165
|
+
if r.status_code != 200:
|
|
166
|
+
await Util().raise_error(r.status_code, str(r.reason) + ' - ' + str(bsoup(r.text, 'lxml').get_text()))
|
|
167
|
+
|
|
168
|
+
soup = bsoup(r.content, 'lxml')
|
|
169
|
+
for s in soup.findAll('script'):
|
|
170
|
+
s.decompose()
|
|
171
|
+
table = soup.find('table', attrs={'rules': 'rows'})
|
|
172
|
+
|
|
173
|
+
ids = []
|
|
174
|
+
for tr in table.findAll('tr')[1:]:
|
|
175
|
+
ids.append(tr.td.get_text(strip=True))
|
|
176
|
+
return ids
|
|
177
|
+
|
|
178
|
+
async def __get_json(self,
|
|
179
|
+
ids_list: list,
|
|
180
|
+
return_fields: list,
|
|
181
|
+
filters: dict) -> dict:
|
|
182
|
+
|
|
183
|
+
ids = 'ids=' + ','.join(ids_list)
|
|
184
|
+
fields = 'fields=' + \
|
|
185
|
+
('id,' if return_fields and 'id' not in return_fields else '')
|
|
186
|
+
if return_fields and 'mirrors' in return_fields:
|
|
187
|
+
fields += f'md5,sha1,filesize,edonkey,aich,tth,extension{"," if len(return_fields) > 1 else ""}'
|
|
188
|
+
|
|
189
|
+
fields += (','.join([fld for fld in return_fields if fld !=
|
|
190
|
+
'mirrors']) if return_fields else '*')
|
|
191
|
+
|
|
192
|
+
url = '&'.join([self.__json_url, ids, fields])
|
|
193
|
+
r = self.__ses.get(url,
|
|
194
|
+
allow_redirects=True)
|
|
195
|
+
logg.debug(
|
|
196
|
+
f'Requesting JSON data from resulted in code: {r.status_code}')
|
|
197
|
+
if r.status_code != 200:
|
|
198
|
+
await Util().raise_error(r.status_code, str(r.reason) + ' - ' + str(bsoup(r.text, 'lxml').get_text()))
|
|
199
|
+
|
|
200
|
+
raw_data = r.json()
|
|
201
|
+
return await self.__format_json(raw_data=raw_data,
|
|
202
|
+
ids_list=ids_list,
|
|
203
|
+
filters=filters,
|
|
204
|
+
return_fields=return_fields)
|
|
205
|
+
|
|
206
|
+
async def __format_json(self,
|
|
207
|
+
raw_data: list,
|
|
208
|
+
ids_list: list,
|
|
209
|
+
filters: dict,
|
|
210
|
+
return_fields: list) -> dict:
|
|
211
|
+
|
|
212
|
+
data = {}
|
|
213
|
+
if raw_data:
|
|
214
|
+
for res in ids_list:
|
|
215
|
+
data[str(res)] = next(
|
|
216
|
+
item for item in raw_data if item['id'] == str(res))
|
|
217
|
+
if data:
|
|
218
|
+
removed = []
|
|
219
|
+
for res_id in data:
|
|
220
|
+
if filters:
|
|
221
|
+
if not await Util().filter_result(data[str(res_id)], filters):
|
|
222
|
+
removed.append(res_id)
|
|
223
|
+
continue
|
|
224
|
+
|
|
225
|
+
cover_reg = re.compile(r'^\d+\\?\/[a-z-0-9]+\..{1,4}$', re.IGNORECASE)
|
|
226
|
+
if 'coverurl' in data[res_id].keys():
|
|
227
|
+
if re.match(cover_reg, data[res_id]["coverurl"]):
|
|
228
|
+
data[res_id][
|
|
229
|
+
'coverurl'] = f'{self.__libgen_url}/covers/{data[res_id]["coverurl"]}'
|
|
230
|
+
else:
|
|
231
|
+
data[res_id][
|
|
232
|
+
'coverurl'] = 'https://cdn2.iconfinder.com/data/icons/leto-blue-online-education/64/__book_mouth_education_online-512.png'
|
|
233
|
+
if not return_fields or 'mirrors' in return_fields:
|
|
234
|
+
md5 = data[res_id]['md5']
|
|
235
|
+
if return_fields and 'md5' not in return_fields:
|
|
236
|
+
data[res_id].pop('md5')
|
|
237
|
+
sha1 = data[res_id]['sha1']
|
|
238
|
+
if return_fields and 'sha1' not in return_fields:
|
|
239
|
+
data[res_id].pop('sha1')
|
|
240
|
+
size = data[res_id]['filesize']
|
|
241
|
+
if return_fields and 'filesize' not in return_fields:
|
|
242
|
+
data[res_id].pop('filesize')
|
|
243
|
+
edonkey = data[res_id]['edonkey']
|
|
244
|
+
if return_fields and 'edonkey' not in return_fields:
|
|
245
|
+
data[res_id].pop('edonkey')
|
|
246
|
+
aich = data[res_id]['aich']
|
|
247
|
+
if return_fields and 'aich' not in return_fields:
|
|
248
|
+
data[res_id].pop('aich')
|
|
249
|
+
tth = data[res_id]['tth']
|
|
250
|
+
if return_fields and 'tth' not in return_fields:
|
|
251
|
+
data[res_id].pop('tth')
|
|
252
|
+
extension = data[res_id]['extension']
|
|
253
|
+
if return_fields and 'extension' not in return_fields:
|
|
254
|
+
data[res_id].pop('extension')
|
|
255
|
+
tor_number = str(res_id)[
|
|
256
|
+
:-3] + '000' if int(res_id) >= 1000 else '000'
|
|
257
|
+
data[res_id]['mirrors'] = {}
|
|
258
|
+
|
|
259
|
+
data[res_id]['mirrors'][
|
|
260
|
+
'main'] = f'http://library.lol/main/{md5}'
|
|
261
|
+
|
|
262
|
+
data[res_id]['mirrors'][
|
|
263
|
+
'libgen.lc'] = f'http://libgen.lc/ads.php?md5={md5}'
|
|
264
|
+
|
|
265
|
+
data[res_id]['mirrors'][
|
|
266
|
+
'z-library'] = f'http://b-ok.cc/md5/{md5}'
|
|
267
|
+
|
|
268
|
+
data[res_id]['mirrors'][
|
|
269
|
+
'libgen.pw'] = f'https://libgen.pw/item?id={res_id}'
|
|
270
|
+
|
|
271
|
+
data[res_id]['mirrors'][
|
|
272
|
+
'bookfi'] = f'http://bookfi.net/md5/{md5}'
|
|
273
|
+
|
|
274
|
+
data[res_id]['mirrors'][
|
|
275
|
+
'torrent'] = f'{self.__libgen_url}/book/index.php?md5={md5}&oftorrent='
|
|
276
|
+
|
|
277
|
+
data[res_id]['mirrors'][
|
|
278
|
+
'torrent_1k'] = f'{self.__libgen_url}/repository_torrent/r_{tor_number}.torrent'
|
|
279
|
+
|
|
280
|
+
data[res_id]['mirrors'][
|
|
281
|
+
'gnutella'] = f'magnet:?xt=urn:sha1:{sha1}&xl={size}&dn={md5}.{extension}'
|
|
282
|
+
|
|
283
|
+
data[res_id]['mirrors'][
|
|
284
|
+
'ed2k'] = f'ed2k://|file|{md5.upper()}.{extension}|{size}|{edonkey}|h={aich}|/'
|
|
285
|
+
|
|
286
|
+
data[res_id]['mirrors'][
|
|
287
|
+
'dc++'] = f'magnet:?xt=urn:tree:tiger:{tth}&xl={size}&dn={md5}.{extension}'
|
|
288
|
+
|
|
289
|
+
if 'torrent' in data[res_id].keys():
|
|
290
|
+
data[res_id].pop('torrent')
|
|
291
|
+
if 'locator' in data[res_id].keys():
|
|
292
|
+
data[res_id].pop('locator')
|
|
293
|
+
data[res_id].pop('id')
|
|
294
|
+
if removed:
|
|
295
|
+
for ids in removed:
|
|
296
|
+
data.pop(ids)
|
|
297
|
+
logg.info(f'Finished processing {len(data)} results.')
|
|
298
|
+
return data
|
|
299
|
+
|
|
300
|
+
@staticmethod
|
|
301
|
+
async def download(url: str,
|
|
302
|
+
dest_folder: Path = None,
|
|
303
|
+
progress: Optional[Callable[..., Awaitable[None]]] = None,
|
|
304
|
+
progress_args: list = []) -> Path:
|
|
305
|
+
"""Download the book from support mirro: 'library.lol', 'libgen.lc', 'libgen.gs', 'b-ok.cc'
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
Args:
|
|
309
|
+
url (str): The download like retrived from search method
|
|
310
|
+
dest_folder (Path, optional): A path where the book should be download. Defaults to the current working directory.
|
|
311
|
+
progress (Optional[Callable[..., Awaitable[None]]]): An awaitable function for progress updates that takes at least 2 arguments -
|
|
312
|
+
current: int and total int or str and other arguments passed through progress args.
|
|
313
|
+
progress_args (list): Arguments to be passed to the progress update function along with progress update
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
Returns:
|
|
317
|
+
Path: The full path of the download book.
|
|
318
|
+
|
|
319
|
+
"""
|
|
320
|
+
return await LibgenDownload().download(url,
|
|
321
|
+
dest_folder,
|
|
322
|
+
progress,
|
|
323
|
+
progress_args)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from urllib.parse import unquote_plus
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Util:
|
|
6
|
+
@staticmethod
|
|
7
|
+
async def get_filename(con_disp: str) -> str:
|
|
8
|
+
|
|
9
|
+
fname = re.findall("filename\*=([^;]+)", con_disp, flags=re.IGNORECASE)
|
|
10
|
+
if not fname:
|
|
11
|
+
fname = re.findall(
|
|
12
|
+
"filename=([^;]+)", con_disp, flags=re.IGNORECASE)
|
|
13
|
+
if "utf-8''" in fname[0].lower():
|
|
14
|
+
fname = re.sub("utf-8''", '', fname[0], flags=re.IGNORECASE)
|
|
15
|
+
fname = unquote_plus(fname)
|
|
16
|
+
else:
|
|
17
|
+
fname = fname[0]
|
|
18
|
+
return fname.strip().strip('"')
|
|
19
|
+
|
|
20
|
+
@staticmethod
|
|
21
|
+
async def filter_result(result: dict,
|
|
22
|
+
filters: dict) -> bool:
|
|
23
|
+
|
|
24
|
+
outcome = True
|
|
25
|
+
for key in filters:
|
|
26
|
+
if not filters[key] in result[key]:
|
|
27
|
+
outcome = False
|
|
28
|
+
return outcome
|
|
29
|
+
|
|
30
|
+
@staticmethod
|
|
31
|
+
async def raise_error(status_code: int,
|
|
32
|
+
resp: str) -> None:
|
|
33
|
+
|
|
34
|
+
raise ConnectionError(
|
|
35
|
+
f'{status_code}: {resp}')
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: libgenrs
|
|
3
|
+
Version: 0.1.9
|
|
4
|
+
Summary: Asynchronous python lib for Libgen.rs
|
|
5
|
+
Home-page: https://github.com/ramanveerji/libgenrs
|
|
6
|
+
Download-URL: https://github.com/ramanveerji/libgenrs/archive/v0.1.9.tar.gz
|
|
7
|
+
License: MIT License
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Programming Language :: Python
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: aiofiles>=0.5.0
|
|
17
|
+
Requires-Dist: aiohttp>=3.6.0
|
|
18
|
+
Requires-Dist: beautifulsoup4>=4.9.0
|
|
19
|
+
Requires-Dist: lxml>=4.6.5
|
|
20
|
+
Requires-Dist: requests>=2.25.0
|
|
21
|
+
Requires-Dist: tldextract>=3.1.0
|
|
22
|
+
|
|
23
|
+
# libgenrs
|
|
24
|
+
|
|
25
|
+
Asynchronous python library for Libgen.is to search and download books.
|
|
26
|
+
This is a forked version
|
|
27
|
+
|
|
28
|
+
[](https://pypi.org/project/libgenrs)
|
|
29
|
+
[](https://github.com/ramanveerji/libgenrs/actions/workflows/python-publish.yml)
|
|
30
|
+
|
|
31
|
+
## Installing libgenrs
|
|
32
|
+
|
|
33
|
+
using the command line
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
pip install libgenrs
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Importing libgenrs
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from libgenrs import Libgen
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Creating libgenrs object
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
lg = Libgen()
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
or
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
lg = Libgen(sort= 'year', sort_mode= 'ASC', result_limit= '50')
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
When creating an instance of LibGen, you can set 3 option.
|
|
58
|
+
|
|
59
|
+
- **sort**: You can use this to choose a sorting method from allowed fields ( 'id', 'author', 'title', 'publisher', 'year', 'pages', 'language', 'size', 'extension'). Defaults to 'def'.
|
|
60
|
+
- **sort_mode**: Pick the order of the sort ascending or decending ('ASC', 'DESC'). Defaults to 'DESC'.
|
|
61
|
+
- **result_limit**: Limit the number of results based on the allowed limit (25, 50, 100). Defaults to 25.
|
|
62
|
+
|
|
63
|
+
## Searching for a book
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
async def search():
|
|
67
|
+
q = 'japan history'
|
|
68
|
+
result = await lg.search(q)
|
|
69
|
+
for item in result:
|
|
70
|
+
print('id = ' + item)
|
|
71
|
+
print('title = ' + result[item]['title'])
|
|
72
|
+
print('md5 = ' + result[item]['md5'])
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
The returned data looks like this: (The dict key is the Libgen id of the book)
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
{
|
|
79
|
+
'881061':{
|
|
80
|
+
'title': 'The Japanese Experience: A Short History of Japan (History of Civilisation)',
|
|
81
|
+
'volumeinfo': '',
|
|
82
|
+
'series': '',
|
|
83
|
+
'periodical': '',
|
|
84
|
+
'author': 'W. G. Beasley',
|
|
85
|
+
'year': '1999',
|
|
86
|
+
'edition': '',
|
|
87
|
+
'publisher': 'University of California Press',
|
|
88
|
+
'city': '',
|
|
89
|
+
'pages': '331',
|
|
90
|
+
'language': 'English',
|
|
91
|
+
'topic': '',
|
|
92
|
+
'library': '',
|
|
93
|
+
'issue': '0',
|
|
94
|
+
'identifier': '0520220501, 9780520220508',
|
|
95
|
+
'issn': '',
|
|
96
|
+
'asin': '',
|
|
97
|
+
'udc': '',
|
|
98
|
+
'lbc': '',
|
|
99
|
+
'ddc': '',
|
|
100
|
+
'lcc': '',
|
|
101
|
+
'doi': '',
|
|
102
|
+
'googlebookid': '',
|
|
103
|
+
'openlibraryid': '',
|
|
104
|
+
'commentary': '',
|
|
105
|
+
'dpi': '0',
|
|
106
|
+
'color': '',
|
|
107
|
+
'cleaned': '',
|
|
108
|
+
'orientation': '',
|
|
109
|
+
'paginated': '',
|
|
110
|
+
'scanned': '',
|
|
111
|
+
'bookmarked': '',
|
|
112
|
+
'searchable': '1',
|
|
113
|
+
'filesize': '29750320',
|
|
114
|
+
'extension': 'pdf',
|
|
115
|
+
'md5': 'a382109f7fdde3be5b2cb4f82d97443b',
|
|
116
|
+
'generic': '',
|
|
117
|
+
'visible': '',
|
|
118
|
+
'local': '0',
|
|
119
|
+
'timeadded': '2013-02-15 18:40:21',
|
|
120
|
+
'timelastmodified': '2019-12-21 21:23:21',
|
|
121
|
+
'coverurl': 'http://libgen.is/covers/881000/a382109f7fdde3be5b2cb4f82d97443b-g.jpg',
|
|
122
|
+
'identifierwodash': '0520220501,9780520220508',
|
|
123
|
+
'tags': '',
|
|
124
|
+
'pagesinfile': '331',
|
|
125
|
+
'descr': '',
|
|
126
|
+
'toc': '',
|
|
127
|
+
'sha1': 'ZZYRRG56BOX3XAQ5D2IWAV2FUUC35ELG',
|
|
128
|
+
'sha256': '3B40D6524409A18573900B729AB1BFB872E26CF111A4845F375A84BD0CB12460',
|
|
129
|
+
'crc32': '3DB84D01',
|
|
130
|
+
'edonkey': 'A63EEBB71C46DAE130725C07F2CDC67C',
|
|
131
|
+
'aich': '733CLKTMCYOGD4W5PIG2GMA7CMLAFN2V',
|
|
132
|
+
'tth': 'O4O5Z7UL2YAOUG57PINCLOAN63HVZPCSDYACT6Q',
|
|
133
|
+
'btih': '84195af51fa738cb232297dd1376df6d8b8313be',
|
|
134
|
+
'mirrors':
|
|
135
|
+
{
|
|
136
|
+
'main': 'http://library.lol/main/a382109f7fdde3be5b2cb4f82d97443b',
|
|
137
|
+
'libgen.lc': 'http://libgen.lc/ads.php?md5=a382109f7fdde3be5b2cb4f82d97443b',
|
|
138
|
+
'z-library': 'http://b-ok.cc/md5/a382109f7fdde3be5b2cb4f82d97443b',
|
|
139
|
+
'libgen.pw': 'https://libgen.pw/item?id=881061',
|
|
140
|
+
'bookfi': 'http://bookfi.net/md5/a382109f7fdde3be5b2cb4f82d97443b',
|
|
141
|
+
'torrent': 'http://libgen.is/book/index.php?md5=a382109f7fdde3be5b2cb4f82d97443b&oftorrent=',
|
|
142
|
+
'torrent_1k': 'http://libgen.is/repository_torrent/r_881000.torrent',
|
|
143
|
+
'gnutella': 'magnet:?xt=urn:sha1:ZZYRRG56BOX3XAQ5D2IWAV2FUUC35ELG&xl=29750320&dn=a382109f7fdde3be5b2cb4f82d97443b.pdf',
|
|
144
|
+
'ed2k': 'ed2k://|file|A382109F7FDDE3BE5B2CB4F82D97443B.pdf|29750320|A63EEBB71C46DAE130725C07F2CDC67C|h=733CLKTMCYOGD4W5PIG2GMA7CMLAFN2V|/',
|
|
145
|
+
'dc++': 'magnet:?xt=urn:tree:tiger:O4O5Z7UL2YAOUG57PINCLOAN63HVZPCSDYACT6Q&xl=29750320&dn=a382109f7fdde3be5b2cb4f82d97443b.pdf'
|
|
146
|
+
}
|
|
147
|
+
},
|
|
148
|
+
'id_of_book':{
|
|
149
|
+
...
|
|
150
|
+
...
|
|
151
|
+
...
|
|
152
|
+
},
|
|
153
|
+
'id_of_book':{
|
|
154
|
+
...
|
|
155
|
+
...
|
|
156
|
+
...
|
|
157
|
+
},
|
|
158
|
+
}
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Note that the above information except for mirrors is fecthed as is from Libgen and it is up to the uploader of the books to include the metadata of the books.
|
|
162
|
+
|
|
163
|
+
## Downloading a book
|
|
164
|
+
|
|
165
|
+
Downloading is only supported for the top **3** mirrors in the returned result ('main', 'libgen.lc', 'z-library'). Pass one of the 3 links to the download method of the Libgen object. Optionaly a destination Path can be passed with the url using **dest_folder** argument. The output of the download method is a Path to the downloaded file.
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
async def download():
|
|
169
|
+
q = 'japan history'
|
|
170
|
+
result = await lg.search(q)
|
|
171
|
+
download_location = []
|
|
172
|
+
path = Path('Downloads')
|
|
173
|
+
for item in result:
|
|
174
|
+
file_path = await lg.download(result[item]['mirrors']['main'],
|
|
175
|
+
dest_folder=path,
|
|
176
|
+
progress=progress,
|
|
177
|
+
progress_args=[
|
|
178
|
+
result[item]['title']
|
|
179
|
+
])
|
|
180
|
+
download_location.append(file_path)
|
|
181
|
+
|
|
182
|
+
async def progress(current, total, title):
|
|
183
|
+
print('Downloading ', current, ' of ', total, ' ', title)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
setup.cfg
|
|
4
|
+
setup.py
|
|
5
|
+
libgenrs/__init__.py
|
|
6
|
+
libgenrs/download.py
|
|
7
|
+
libgenrs/search.py
|
|
8
|
+
libgenrs/utils.py
|
|
9
|
+
libgenrs.egg-info/PKG-INFO
|
|
10
|
+
libgenrs.egg-info/SOURCES.txt
|
|
11
|
+
libgenrs.egg-info/dependency_links.txt
|
|
12
|
+
libgenrs.egg-info/not-zip-safe
|
|
13
|
+
libgenrs.egg-info/requires.txt
|
|
14
|
+
libgenrs.egg-info/top_level.txt
|
|
15
|
+
tests/__init__.py
|
|
16
|
+
tests/libgen_test.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
libgenrs-0.1.9/setup.cfg
ADDED
libgenrs-0.1.9/setup.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*
|
|
3
|
+
|
|
4
|
+
from __future__ import absolute_import
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
from codecs import open
|
|
8
|
+
|
|
9
|
+
from setuptools import find_packages, setup
|
|
10
|
+
|
|
11
|
+
from libgenrs import __version__
|
|
12
|
+
# allow setup.py to be run from any path
|
|
13
|
+
os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))
|
|
14
|
+
|
|
15
|
+
with open('requirements.txt') as f:
|
|
16
|
+
install_requires = f.read().splitlines()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
with open('README.md', 'r', encoding='utf-8') as rm_file:
|
|
20
|
+
readme = rm_file.read()
|
|
21
|
+
|
|
22
|
+
setup(name='libgenrs',
|
|
23
|
+
version=__version__,
|
|
24
|
+
packages=find_packages(exclude=('tests')),
|
|
25
|
+
zip_safe=False,
|
|
26
|
+
url='https://github.com/ramanveerji/libgenrs',
|
|
27
|
+
long_description_content_type='text/markdown',
|
|
28
|
+
description='Asynchronous python lib for Libgen.rs',
|
|
29
|
+
download_url=f'https://github.com/ramanveerji/libgenrs/archive/v{__version__}.tar.gz',
|
|
30
|
+
long_description=readme,
|
|
31
|
+
license='MIT License',
|
|
32
|
+
install_requires=install_requires,
|
|
33
|
+
classifiers=[
|
|
34
|
+
'Intended Audience :: Developers',
|
|
35
|
+
'Operating System :: OS Independent',
|
|
36
|
+
'Programming Language :: Python',
|
|
37
|
+
'Programming Language :: Python :: 3.8',
|
|
38
|
+
'Programming Language :: Python :: 3.9',
|
|
39
|
+
'Topic :: Internet :: WWW/HTTP',
|
|
40
|
+
])
|
|
File without changes
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
from libgenrs import Libgen
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Testclass:
|
|
7
|
+
"""Testing of Libgen object, searching, result types, and download of the books."""
|
|
8
|
+
@staticmethod
|
|
9
|
+
def test_Libgen():
|
|
10
|
+
# create a Libgen object with return result limit set to 50
|
|
11
|
+
lg = Libgen(sort="title", sort_mode="ASC", result_limit=50)
|
|
12
|
+
assert isinstance(lg, Libgen)
|
|
13
|
+
|
|
14
|
+
@staticmethod
|
|
15
|
+
@pytest.mark.asyncio
|
|
16
|
+
async def test_search():
|
|
17
|
+
lg = Libgen()
|
|
18
|
+
# search Libgen
|
|
19
|
+
result = await lg.search('japan history')
|
|
20
|
+
assert isinstance(result, dict)
|
|
21
|
+
|
|
22
|
+
@staticmethod
|
|
23
|
+
@pytest.mark.asyncio
|
|
24
|
+
async def test_result():
|
|
25
|
+
lg = Libgen()
|
|
26
|
+
result = await lg.search('japan history')
|
|
27
|
+
# check first result
|
|
28
|
+
ids = [*result]
|
|
29
|
+
assert isinstance(result[ids[0]], dict)
|
|
30
|
+
|
|
31
|
+
@staticmethod
|
|
32
|
+
@pytest.mark.asyncio
|
|
33
|
+
async def test_download():
|
|
34
|
+
lg = Libgen()
|
|
35
|
+
result = await lg.search('japan history')
|
|
36
|
+
ids = [*result]
|
|
37
|
+
async def progress(current, total, test_arg, test2_arg):
|
|
38
|
+
assert isinstance(current, int)
|
|
39
|
+
assert isinstance(total, (int, str))
|
|
40
|
+
assert isinstance(test_arg, Libgen) and test_arg.test == 'Test string'
|
|
41
|
+
assert isinstance(test2_arg, int) and test2_arg == 123456
|
|
42
|
+
# download the first result and print the path
|
|
43
|
+
lg.test = 'Test string'
|
|
44
|
+
file = await lg.download(result[ids[0]]['mirrors']['main'],
|
|
45
|
+
dest_folder=Path("download_test"),
|
|
46
|
+
progress=progress,
|
|
47
|
+
progress_args=[
|
|
48
|
+
lg, 123456
|
|
49
|
+
])
|
|
50
|
+
assert Path.is_file(file)
|