nbno 1.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nbno-1.2.3/LICENSE +22 -0
- nbno-1.2.3/PKG-INFO +147 -0
- nbno-1.2.3/README.md +132 -0
- nbno-1.2.3/nbno.egg-info/PKG-INFO +147 -0
- nbno-1.2.3/nbno.egg-info/SOURCES.txt +10 -0
- nbno-1.2.3/nbno.egg-info/dependency_links.txt +1 -0
- nbno-1.2.3/nbno.egg-info/entry_points.txt +2 -0
- nbno-1.2.3/nbno.egg-info/requires.txt +7 -0
- nbno-1.2.3/nbno.egg-info/top_level.txt +1 -0
- nbno-1.2.3/nbno.py +712 -0
- nbno-1.2.3/pyproject.toml +27 -0
- nbno-1.2.3/setup.cfg +4 -0
nbno-1.2.3/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Lanjelin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
nbno-1.2.3/PKG-INFO
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nbno
|
|
3
|
+
Version: 1.2.3
|
|
4
|
+
Summary: NB.no nedlaster
|
|
5
|
+
Requires-Python: >=3.9
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Dist: Pillow<13,>=12
|
|
9
|
+
Requires-Dist: requests<3,>=2.32
|
|
10
|
+
Provides-Extra: web
|
|
11
|
+
Requires-Dist: Flask<4,>=2; extra == "web"
|
|
12
|
+
Requires-Dist: ocrmypdf>=13.5; extra == "web"
|
|
13
|
+
Requires-Dist: img2pdf>=0.4; extra == "web"
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# NBNO.py
|
|
17
|
+
|
|
18
|
+
<p align="center">
|
|
19
|
+
<img src="https://raw.githubusercontent.com/Lanjelin/NBNO.py/refs/heads/master/web/static/img/logo.png" width="300" alt="Logo">
|
|
20
|
+
</p>
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
Dette er et Python script som laster ned bøker og annet media fra Nasjonalbiblioteket (NB.no).
|
|
25
|
+
|
|
26
|
+
### Kjøring i Docker - Web
|
|
27
|
+
|
|
28
|
+
<details>
|
|
29
|
+
<summary><strong>Screenshot</strong></summary>
|
|
30
|
+
<p align="center">
|
|
31
|
+
<a href=https://github.com/Lanjelin/NBNO.py/blob/master/.github/screenshots/fullpage.png>
|
|
32
|
+
<img src="https://raw.githubusercontent.com/Lanjelin/NBNO.py/refs/heads/master/.github/screenshots/fullpage.png"
|
|
33
|
+
alt="Full-page screenshot preview" width="600"/>
|
|
34
|
+
</a>
|
|
35
|
+
</p>
|
|
36
|
+
</details>
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
Bind en lokal mappe til `/data` for å få tilgang til filer som lastes ned via utforsker, filene er ellers tilgjengelige via webgrensesnittet.
|
|
40
|
+
Om en ønsker flere språk enn Norsk og Engelsk for OCR, må en binde en lokal mappe til `/opt/tessdata` og plassere [*.traineddata](https://github.com/tesseract-ocr/tessdata) der.
|
|
41
|
+
|
|
42
|
+
Ellers er det bare å starte containeren, og peke nettlesen til [port 5000](http://127.0.0.1:5000).
|
|
43
|
+
|
|
44
|
+
For å finne medie-ID, ta en kikk [her](https://github.com/Lanjelin/NBNO.py/blob/master/.github/screenshots/medie_id.png)
|
|
45
|
+
|
|
46
|
+
#### Docker Run
|
|
47
|
+
`docker run --name nbno -p 5000:5000 -v ./nbno/data:/data -v ./nbno/tessdata:/opt/tessdata -d ghcr.io/lanjelin/nbno:latest`
|
|
48
|
+
|
|
49
|
+
#### Docker compose
|
|
50
|
+
```yaml
|
|
51
|
+
services:
|
|
52
|
+
nbno:
|
|
53
|
+
container_name: nbno
|
|
54
|
+
ports:
|
|
55
|
+
- 5000:5000
|
|
56
|
+
volumes:
|
|
57
|
+
- ./nbno/data:/data
|
|
58
|
+
- ./nbno/tessdata:/opt/tessdata
|
|
59
|
+
image: ghcr.io/lanjelin/nbno:latest
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
### Kjøring uten Docker - CLI
|
|
64
|
+
For å kjøre denne koden trengs Python 3.9 eller nyere.
|
|
65
|
+
|
|
66
|
+
Linux og Mac kommer normalt med python installert.
|
|
67
|
+
For Windows, last ned Python fra [python.org](https://www.python.org/downloads/), få med 'Add Python 3.xx to PATH'
|
|
68
|
+
|
|
69
|
+
For å sjekke versjon av python, kjør `python --version`(Windows), `python3 --version`(Mac/Linux), fra kommandolinjen.
|
|
70
|
+
|
|
71
|
+
#### Installer koden fra kilde
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
git clone https://github.com/Lanjelin/NBNO.py.git
|
|
75
|
+
cd NBNO.py
|
|
76
|
+
python3 -m pip install .
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
#### Installere koden fra PyPi
|
|
80
|
+
```bash
|
|
81
|
+
python -m pip install nbno
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
#### Kjøring
|
|
85
|
+
Etter installering kan programmet kjøres slik:
|
|
86
|
+
```bash
|
|
87
|
+
nbno --id <ID>
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
#### Avinstallering
|
|
91
|
+
```bash
|
|
92
|
+
python3 -m pip uninstall nbno
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Argumenter
|
|
96
|
+
Eneste påkrevde argumentet er ID, som finnes ved å trykke Referere/Sitere for så å kopiere alt av tekst og tall etter no-nb_ eks. digitidsskrift_202101..etc --> `nbno --id digitidsskrift_202101..etc`
|
|
97
|
+
|
|
98
|
+
Følgende er støttet:
|
|
99
|
+
- Bøker (digibok)
|
|
100
|
+
- Aviser (digavis)
|
|
101
|
+
- Bilder (digifoto)
|
|
102
|
+
- Tidsskrift (digitidsskrift)
|
|
103
|
+
- Kart (digikart)
|
|
104
|
+
- Brev og Manuskripter (digimanus)
|
|
105
|
+
- Noter (digibok)
|
|
106
|
+
- Musikkmanuskripter (digimanus)
|
|
107
|
+
- Plakater (digifoto)
|
|
108
|
+
- Programrapport (digiprogramrapport)
|
|
109
|
+
|
|
110
|
+
<details>
|
|
111
|
+
<summary>Innlogget innhold</summary>
|
|
112
|
+
Som innlogget, åpne boka der en kan lese den, åpne Utviklerverktøy og finn Nettverkfanen.
|
|
113
|
+
Refresh siden, og nettverksfanen populeres med innhold.
|
|
114
|
+
Finn og velg manifest?fields=etcetc i listen, og bla igjen ned og finn Request Headers.
|
|
115
|
+
|
|
116
|
+
Her kopieres innholdet fra authorization og cookie og lagres i en textfil ved scriptet.
|
|
117
|
+
Formaten på tekstfilen er
|
|
118
|
+
```
|
|
119
|
+
authorization=4JjcVi6faGF-GhD6wMoXZ80rUkg.*AAJTSQACMDIAAlNLABxxRandomRandomxxSNVpvUTlQxxRandomRandomxxDVFMAAlMxAAIwMQ..
|
|
120
|
+
cookie=_ga=GA1.1.1234543217.123454321; _hjSessionUser_123454321=eyJpZCI6IjUzOTZmxxRandomRandomxxy1hNDEwLTc0ZjA4NTJhxxRandomRandomxxOjE3MjYwNTEyNzcxxRandomRandomxxW5nIjp0cnVlfQ==;osvosvetc
|
|
121
|
+
```
|
|
122
|
+
Scriptet kjøres deretter med `--cookie` flagget som peker til filen, feks `python3 nbno.py --id blabla --cookie nbno-cookie.txt`
|
|
123
|
+
|
|
124
|
+
</details>
|
|
125
|
+
|
|
126
|
+
```
|
|
127
|
+
bruk: nbno [-h] [--id <ID>] [--cover] [--pdf] [--f2pdf] [--url] [--error]
|
|
128
|
+
[--v] [--resize <int>] [--start <int>] [--stop <int>]
|
|
129
|
+
|
|
130
|
+
påkrevd argument:
|
|
131
|
+
--id <ID> IDen på innholdet som skal lastes ned
|
|
132
|
+
|
|
133
|
+
valgfrie argumenter:
|
|
134
|
+
-h, --help show this help message and exit
|
|
135
|
+
--cover Settes for å laste covers
|
|
136
|
+
--title Settes for å hente tittel på bok automatisk
|
|
137
|
+
--pdf Settes for å lage pdf av bildene som lastes
|
|
138
|
+
--f2pdf Settes for å lage pdf av bilder i eksisterende mappe
|
|
139
|
+
--url Settes for å printe URL på hver del
|
|
140
|
+
--error Settes for å printe HTTP feilkoder
|
|
141
|
+
--v Settes for å printe mer info
|
|
142
|
+
--resize <int> Prosent av originalstørrelse på bilder
|
|
143
|
+
--start <int> Sidetall å starte på
|
|
144
|
+
--stop <int> Sidetall å stoppe på
|
|
145
|
+
--cookie <string> Sti til fil for autentisering
|
|
146
|
+
```
|
|
147
|
+
|
nbno-1.2.3/README.md
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# NBNO.py
|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
<img src="https://raw.githubusercontent.com/Lanjelin/NBNO.py/refs/heads/master/web/static/img/logo.png" width="300" alt="Logo">
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
Dette er et Python script som laster ned bøker og annet media fra Nasjonalbiblioteket (NB.no).
|
|
10
|
+
|
|
11
|
+
### Kjøring i Docker - Web
|
|
12
|
+
|
|
13
|
+
<details>
|
|
14
|
+
<summary><strong>Screenshot</strong></summary>
|
|
15
|
+
<p align="center">
|
|
16
|
+
<a href=https://github.com/Lanjelin/NBNO.py/blob/master/.github/screenshots/fullpage.png>
|
|
17
|
+
<img src="https://raw.githubusercontent.com/Lanjelin/NBNO.py/refs/heads/master/.github/screenshots/fullpage.png"
|
|
18
|
+
alt="Full-page screenshot preview" width="600"/>
|
|
19
|
+
</a>
|
|
20
|
+
</p>
|
|
21
|
+
</details>
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
Bind en lokal mappe til `/data` for å få tilgang til filer som lastes ned via utforsker, filene er ellers tilgjengelige via webgrensesnittet.
|
|
25
|
+
Om en ønsker flere språk enn Norsk og Engelsk for OCR, må en binde en lokal mappe til `/opt/tessdata` og plassere [*.traineddata](https://github.com/tesseract-ocr/tessdata) der.
|
|
26
|
+
|
|
27
|
+
Ellers er det bare å starte containeren, og peke nettlesen til [port 5000](http://127.0.0.1:5000).
|
|
28
|
+
|
|
29
|
+
For å finne medie-ID, ta en kikk [her](https://github.com/Lanjelin/NBNO.py/blob/master/.github/screenshots/medie_id.png)
|
|
30
|
+
|
|
31
|
+
#### Docker Run
|
|
32
|
+
`docker run --name nbno -p 5000:5000 -v ./nbno/data:/data -v ./nbno/tessdata:/opt/tessdata -d ghcr.io/lanjelin/nbno:latest`
|
|
33
|
+
|
|
34
|
+
#### Docker compose
|
|
35
|
+
```yaml
|
|
36
|
+
services:
|
|
37
|
+
nbno:
|
|
38
|
+
container_name: nbno
|
|
39
|
+
ports:
|
|
40
|
+
- 5000:5000
|
|
41
|
+
volumes:
|
|
42
|
+
- ./nbno/data:/data
|
|
43
|
+
- ./nbno/tessdata:/opt/tessdata
|
|
44
|
+
image: ghcr.io/lanjelin/nbno:latest
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
### Kjøring uten Docker - CLI
|
|
49
|
+
For å kjøre denne koden trengs Python 3.9 eller nyere.
|
|
50
|
+
|
|
51
|
+
Linux og Mac kommer normalt med python installert.
|
|
52
|
+
For Windows, last ned Python fra [python.org](https://www.python.org/downloads/), få med 'Add Python 3.xx to PATH'
|
|
53
|
+
|
|
54
|
+
For å sjekke versjon av python, kjør `python --version`(Windows), `python3 --version`(Mac/Linux), fra kommandolinjen.
|
|
55
|
+
|
|
56
|
+
#### Installer koden fra kilde
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
git clone https://github.com/Lanjelin/NBNO.py.git
|
|
60
|
+
cd NBNO.py
|
|
61
|
+
python3 -m pip install .
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
#### Installere koden fra PyPi
|
|
65
|
+
```bash
|
|
66
|
+
python -m pip install nbno
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
#### Kjøring
|
|
70
|
+
Etter installering kan programmet kjøres slik:
|
|
71
|
+
```bash
|
|
72
|
+
nbno --id <ID>
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
#### Avinstallering
|
|
76
|
+
```bash
|
|
77
|
+
python3 -m pip uninstall nbno
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
### Argumenter
|
|
81
|
+
Eneste påkrevde argumentet er ID, som finnes ved å trykke Referere/Sitere for så å kopiere alt av tekst og tall etter no-nb_ eks. digitidsskrift_202101..etc --> `nbno --id digitidsskrift_202101..etc`
|
|
82
|
+
|
|
83
|
+
Følgende er støttet:
|
|
84
|
+
- Bøker (digibok)
|
|
85
|
+
- Aviser (digavis)
|
|
86
|
+
- Bilder (digifoto)
|
|
87
|
+
- Tidsskrift (digitidsskrift)
|
|
88
|
+
- Kart (digikart)
|
|
89
|
+
- Brev og Manuskripter (digimanus)
|
|
90
|
+
- Noter (digibok)
|
|
91
|
+
- Musikkmanuskripter (digimanus)
|
|
92
|
+
- Plakater (digifoto)
|
|
93
|
+
- Programrapport (digiprogramrapport)
|
|
94
|
+
|
|
95
|
+
<details>
|
|
96
|
+
<summary>Innlogget innhold</summary>
|
|
97
|
+
Som innlogget, åpne boka der en kan lese den, åpne Utviklerverktøy og finn Nettverkfanen.
|
|
98
|
+
Refresh siden, og nettverksfanen populeres med innhold.
|
|
99
|
+
Finn og velg manifest?fields=etcetc i listen, og bla igjen ned og finn Request Headers.
|
|
100
|
+
|
|
101
|
+
Her kopieres innholdet fra authorization og cookie og lagres i en textfil ved scriptet.
|
|
102
|
+
Formaten på tekstfilen er
|
|
103
|
+
```
|
|
104
|
+
authorization=4JjcVi6faGF-GhD6wMoXZ80rUkg.*AAJTSQACMDIAAlNLABxxRandomRandomxxSNVpvUTlQxxRandomRandomxxDVFMAAlMxAAIwMQ..
|
|
105
|
+
cookie=_ga=GA1.1.1234543217.123454321; _hjSessionUser_123454321=eyJpZCI6IjUzOTZmxxRandomRandomxxy1hNDEwLTc0ZjA4NTJhxxRandomRandomxxOjE3MjYwNTEyNzcxxRandomRandomxxW5nIjp0cnVlfQ==;osvosvetc
|
|
106
|
+
```
|
|
107
|
+
Scriptet kjøres deretter med `--cookie` flagget som peker til filen, feks `python3 nbno.py --id blabla --cookie nbno-cookie.txt`
|
|
108
|
+
|
|
109
|
+
</details>
|
|
110
|
+
|
|
111
|
+
```
|
|
112
|
+
bruk: nbno [-h] [--id <ID>] [--cover] [--pdf] [--f2pdf] [--url] [--error]
|
|
113
|
+
[--v] [--resize <int>] [--start <int>] [--stop <int>]
|
|
114
|
+
|
|
115
|
+
påkrevd argument:
|
|
116
|
+
--id <ID> IDen på innholdet som skal lastes ned
|
|
117
|
+
|
|
118
|
+
valgfrie argumenter:
|
|
119
|
+
-h, --help show this help message and exit
|
|
120
|
+
--cover Settes for å laste covers
|
|
121
|
+
--title Settes for å hente tittel på bok automatisk
|
|
122
|
+
--pdf Settes for å lage pdf av bildene som lastes
|
|
123
|
+
--f2pdf Settes for å lage pdf av bilder i eksisterende mappe
|
|
124
|
+
--url Settes for å printe URL på hver del
|
|
125
|
+
--error Settes for å printe HTTP feilkoder
|
|
126
|
+
--v Settes for å printe mer info
|
|
127
|
+
--resize <int> Prosent av originalstørrelse på bilder
|
|
128
|
+
--start <int> Sidetall å starte på
|
|
129
|
+
--stop <int> Sidetall å stoppe på
|
|
130
|
+
--cookie <string> Sti til fil for autentisering
|
|
131
|
+
```
|
|
132
|
+
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nbno
|
|
3
|
+
Version: 1.2.3
|
|
4
|
+
Summary: NB.no nedlaster
|
|
5
|
+
Requires-Python: >=3.9
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Dist: Pillow<13,>=12
|
|
9
|
+
Requires-Dist: requests<3,>=2.32
|
|
10
|
+
Provides-Extra: web
|
|
11
|
+
Requires-Dist: Flask<4,>=2; extra == "web"
|
|
12
|
+
Requires-Dist: ocrmypdf>=13.5; extra == "web"
|
|
13
|
+
Requires-Dist: img2pdf>=0.4; extra == "web"
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# NBNO.py
|
|
17
|
+
|
|
18
|
+
<p align="center">
|
|
19
|
+
<img src="https://raw.githubusercontent.com/Lanjelin/NBNO.py/refs/heads/master/web/static/img/logo.png" width="300" alt="Logo">
|
|
20
|
+
</p>
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
Dette er et Python script som laster ned bøker og annet media fra Nasjonalbiblioteket (NB.no).
|
|
25
|
+
|
|
26
|
+
### Kjøring i Docker - Web
|
|
27
|
+
|
|
28
|
+
<details>
|
|
29
|
+
<summary><strong>Screenshot</strong></summary>
|
|
30
|
+
<p align="center">
|
|
31
|
+
<a href=https://github.com/Lanjelin/NBNO.py/blob/master/.github/screenshots/fullpage.png>
|
|
32
|
+
<img src="https://raw.githubusercontent.com/Lanjelin/NBNO.py/refs/heads/master/.github/screenshots/fullpage.png"
|
|
33
|
+
alt="Full-page screenshot preview" width="600"/>
|
|
34
|
+
</a>
|
|
35
|
+
</p>
|
|
36
|
+
</details>
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
Bind en lokal mappe til `/data` for å få tilgang til filer som lastes ned via utforsker, filene er ellers tilgjengelige via webgrensesnittet.
|
|
40
|
+
Om en ønsker flere språk enn Norsk og Engelsk for OCR, må en binde en lokal mappe til `/opt/tessdata` og plassere [*.traineddata](https://github.com/tesseract-ocr/tessdata) der.
|
|
41
|
+
|
|
42
|
+
Ellers er det bare å starte containeren, og peke nettlesen til [port 5000](http://127.0.0.1:5000).
|
|
43
|
+
|
|
44
|
+
For å finne medie-ID, ta en kikk [her](https://github.com/Lanjelin/NBNO.py/blob/master/.github/screenshots/medie_id.png)
|
|
45
|
+
|
|
46
|
+
#### Docker Run
|
|
47
|
+
`docker run --name nbno -p 5000:5000 -v ./nbno/data:/data -v ./nbno/tessdata:/opt/tessdata -d ghcr.io/lanjelin/nbno:latest`
|
|
48
|
+
|
|
49
|
+
#### Docker compose
|
|
50
|
+
```yaml
|
|
51
|
+
services:
|
|
52
|
+
nbno:
|
|
53
|
+
container_name: nbno
|
|
54
|
+
ports:
|
|
55
|
+
- 5000:5000
|
|
56
|
+
volumes:
|
|
57
|
+
- ./nbno/data:/data
|
|
58
|
+
- ./nbno/tessdata:/opt/tessdata
|
|
59
|
+
image: ghcr.io/lanjelin/nbno:latest
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
### Kjøring uten Docker - CLI
|
|
64
|
+
For å kjøre denne koden trengs Python 3.9 eller nyere.
|
|
65
|
+
|
|
66
|
+
Linux og Mac kommer normalt med python installert.
|
|
67
|
+
For Windows, last ned Python fra [python.org](https://www.python.org/downloads/), få med 'Add Python 3.xx to PATH'
|
|
68
|
+
|
|
69
|
+
For å sjekke versjon av python, kjør `python --version`(Windows), `python3 --version`(Mac/Linux), fra kommandolinjen.
|
|
70
|
+
|
|
71
|
+
#### Installer koden fra kilde
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
git clone https://github.com/Lanjelin/NBNO.py.git
|
|
75
|
+
cd NBNO.py
|
|
76
|
+
python3 -m pip install .
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
#### Installere koden fra PyPi
|
|
80
|
+
```bash
|
|
81
|
+
python -m pip install nbno
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
#### Kjøring
|
|
85
|
+
Etter installering kan programmet kjøres slik:
|
|
86
|
+
```bash
|
|
87
|
+
nbno --id <ID>
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
#### Avinstallering
|
|
91
|
+
```bash
|
|
92
|
+
python3 -m pip uninstall nbno
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Argumenter
|
|
96
|
+
Eneste påkrevde argumentet er ID, som finnes ved å trykke Referere/Sitere for så å kopiere alt av tekst og tall etter no-nb_ eks. digitidsskrift_202101..etc --> `nbno --id digitidsskrift_202101..etc`
|
|
97
|
+
|
|
98
|
+
Følgende er støttet:
|
|
99
|
+
- Bøker (digibok)
|
|
100
|
+
- Aviser (digavis)
|
|
101
|
+
- Bilder (digifoto)
|
|
102
|
+
- Tidsskrift (digitidsskrift)
|
|
103
|
+
- Kart (digikart)
|
|
104
|
+
- Brev og Manuskripter (digimanus)
|
|
105
|
+
- Noter (digibok)
|
|
106
|
+
- Musikkmanuskripter (digimanus)
|
|
107
|
+
- Plakater (digifoto)
|
|
108
|
+
- Programrapport (digiprogramrapport)
|
|
109
|
+
|
|
110
|
+
<details>
|
|
111
|
+
<summary>Innlogget innhold</summary>
|
|
112
|
+
Som innlogget, åpne boka der en kan lese den, åpne Utviklerverktøy og finn Nettverkfanen.
|
|
113
|
+
Refresh siden, og nettverksfanen populeres med innhold.
|
|
114
|
+
Finn og velg manifest?fields=etcetc i listen, og bla igjen ned og finn Request Headers.
|
|
115
|
+
|
|
116
|
+
Her kopieres innholdet fra authorization og cookie og lagres i en textfil ved scriptet.
|
|
117
|
+
Formaten på tekstfilen er
|
|
118
|
+
```
|
|
119
|
+
authorization=4JjcVi6faGF-GhD6wMoXZ80rUkg.*AAJTSQACMDIAAlNLABxxRandomRandomxxSNVpvUTlQxxRandomRandomxxDVFMAAlMxAAIwMQ..
|
|
120
|
+
cookie=_ga=GA1.1.1234543217.123454321; _hjSessionUser_123454321=eyJpZCI6IjUzOTZmxxRandomRandomxxy1hNDEwLTc0ZjA4NTJhxxRandomRandomxxOjE3MjYwNTEyNzcxxRandomRandomxxW5nIjp0cnVlfQ==;osvosvetc
|
|
121
|
+
```
|
|
122
|
+
Scriptet kjøres deretter med `--cookie` flagget som peker til filen, feks `python3 nbno.py --id blabla --cookie nbno-cookie.txt`
|
|
123
|
+
|
|
124
|
+
</details>
|
|
125
|
+
|
|
126
|
+
```
|
|
127
|
+
bruk: nbno [-h] [--id <ID>] [--cover] [--pdf] [--f2pdf] [--url] [--error]
|
|
128
|
+
[--v] [--resize <int>] [--start <int>] [--stop <int>]
|
|
129
|
+
|
|
130
|
+
påkrevd argument:
|
|
131
|
+
--id <ID> IDen på innholdet som skal lastes ned
|
|
132
|
+
|
|
133
|
+
valgfrie argumenter:
|
|
134
|
+
-h, --help show this help message and exit
|
|
135
|
+
--cover Settes for å laste covers
|
|
136
|
+
--title Settes for å hente tittel på bok automatisk
|
|
137
|
+
--pdf Settes for å lage pdf av bildene som lastes
|
|
138
|
+
--f2pdf Settes for å lage pdf av bilder i eksisterende mappe
|
|
139
|
+
--url Settes for å printe URL på hver del
|
|
140
|
+
--error Settes for å printe HTTP feilkoder
|
|
141
|
+
--v Settes for å printe mer info
|
|
142
|
+
--resize <int> Prosent av originalstørrelse på bilder
|
|
143
|
+
--start <int> Sidetall å starte på
|
|
144
|
+
--stop <int> Sidetall å stoppe på
|
|
145
|
+
--cookie <string> Sti til fil for autentisering
|
|
146
|
+
```
|
|
147
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
nbno
|
nbno-1.2.3/nbno.py
ADDED
|
@@ -0,0 +1,712 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
import io
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import warnings
|
|
7
|
+
import argparse
|
|
8
|
+
import threading
|
|
9
|
+
from PIL import Image, UnidentifiedImageError
|
|
10
|
+
from glob import glob
|
|
11
|
+
from math import ceil
|
|
12
|
+
import multiprocessing
|
|
13
|
+
from requests import session
|
|
14
|
+
import concurrent.futures as cf
|
|
15
|
+
from urllib3.util.retry import Retry
|
|
16
|
+
from requests.adapters import HTTPAdapter
|
|
17
|
+
from requests.exceptions import RequestException
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
BASE_DIR = os.environ.get('DOWNLOAD_DIR', '.')
|
|
21
|
+
# directory for aggregated PDFs
|
|
22
|
+
# PDFs and sources organized per book directory
|
|
23
|
+
|
|
24
|
+
class Book:
|
|
25
|
+
"""Holder styr på all info om bildefiler til bok/avis/mm."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, digimedie, cli_mode=False):
|
|
28
|
+
# original passed ID; media_type and media_id will be set next
|
|
29
|
+
self.digimedie = digimedie
|
|
30
|
+
# run in CLI mode (flat dirs) vs. webapp mode (sources/, metadata/, pdf/)
|
|
31
|
+
self.cli_mode = bool(cli_mode)
|
|
32
|
+
if digimedie.find("plikt") > -1:
|
|
33
|
+
self.media_type = "plikt" + \
|
|
34
|
+
digimedie.split("plikt")[1].split("_")[0]
|
|
35
|
+
else:
|
|
36
|
+
self.media_type = "dig" + digimedie.split("dig")[1].split("_")[0]
|
|
37
|
+
self.media_id = digimedie.split(self.media_type + "_")[1]
|
|
38
|
+
# default folder_name = type + '_' + id
|
|
39
|
+
self.folder_name = f"{self.media_type}_{self.media_id}"
|
|
40
|
+
# store the original download folder name for canonical references
|
|
41
|
+
self.default_folder_name = self.folder_name
|
|
42
|
+
self.current_page = "0001"
|
|
43
|
+
self.max_workers = multiprocessing.cpu_count() * 4
|
|
44
|
+
self.covers = False
|
|
45
|
+
self.verbose = False
|
|
46
|
+
self.print_url = False
|
|
47
|
+
self.print_error = False
|
|
48
|
+
self.tile_width = 1024
|
|
49
|
+
self.tile_height = 1024
|
|
50
|
+
self.api_url = "https://api.nb.no/catalog/v1/iiif/URN:NBN:no-nb"
|
|
51
|
+
self.tilgang = ""
|
|
52
|
+
self.image_url = ""
|
|
53
|
+
self.title = "nbnopy"
|
|
54
|
+
self.folder_path = ""
|
|
55
|
+
self.existing_images = []
|
|
56
|
+
self.page_names = []
|
|
57
|
+
self.raw_metadata = []
|
|
58
|
+
self.page_data = {}
|
|
59
|
+
self.page_url = {}
|
|
60
|
+
self.num_pages = 0
|
|
61
|
+
# manifest-level thumbnail (from IIIF manifest), if any
|
|
62
|
+
self.manifest_thumbnail = None
|
|
63
|
+
self.resize = 0
|
|
64
|
+
self.session = session()
|
|
65
|
+
self.session.headers["User-Agent"] = "Mozilla/5.0"
|
|
66
|
+
self.adapter = HTTPAdapter(max_retries=(
|
|
67
|
+
Retry(total=5, backoff_factor=0.5)))
|
|
68
|
+
self.session.mount("https://", self.adapter)
|
|
69
|
+
self.session.mount("http://", self.adapter)
|
|
70
|
+
# whether to include a cover page (C1.jpg) first in PDF output
|
|
71
|
+
self.include_cover = False
|
|
72
|
+
# base output directory may be configured via DOWNLOAD_DIR env var
|
|
73
|
+
self.set_folder_path(os.path.join(BASE_DIR, self.folder_name) + os.path.sep)
|
|
74
|
+
# per-book storage layout differs: flat (CLI) or nested (webapp)
|
|
75
|
+
if self.cli_mode:
|
|
76
|
+
self.sources_dir = self.folder_path
|
|
77
|
+
self.pdf_dir = self.folder_path
|
|
78
|
+
self.meta_dir = self.folder_path
|
|
79
|
+
else:
|
|
80
|
+
self.sources_dir = os.path.join(self.folder_path, 'sources') + os.path.sep
|
|
81
|
+
self.pdf_dir = os.path.join(self.folder_path, 'pdf') + os.path.sep
|
|
82
|
+
self.meta_dir = os.path.join(self.folder_path, 'metadata') + os.path.sep
|
|
83
|
+
self.get_manifest()
|
|
84
|
+
self.image_lock = threading.Lock()
|
|
85
|
+
self.download_skipped = False
|
|
86
|
+
self._pdf_redownload_attempts = set()
|
|
87
|
+
|
|
88
|
+
def set_tile_sizes(self, width, height):
|
|
89
|
+
self.tile_width = width
|
|
90
|
+
self.tile_height = height
|
|
91
|
+
|
|
92
|
+
def _extract_label_text(self, label):
|
|
93
|
+
if isinstance(label, str):
|
|
94
|
+
return label
|
|
95
|
+
if isinstance(label, list):
|
|
96
|
+
for entry in label:
|
|
97
|
+
text = self._extract_label_text(entry)
|
|
98
|
+
if text:
|
|
99
|
+
return text
|
|
100
|
+
if isinstance(label, dict):
|
|
101
|
+
for entry in label.values():
|
|
102
|
+
text = self._extract_label_text(entry)
|
|
103
|
+
if text:
|
|
104
|
+
return text
|
|
105
|
+
return ""
|
|
106
|
+
|
|
107
|
+
def set_to_print_url(self):
|
|
108
|
+
self.print_url = True
|
|
109
|
+
|
|
110
|
+
def set_to_print_errors(self):
|
|
111
|
+
self.print_error = True
|
|
112
|
+
|
|
113
|
+
def set_title(self):
|
|
114
|
+
# rename folder to title string
|
|
115
|
+
self.folder_name = self.title
|
|
116
|
+
self.set_folder_path(os.path.join(BASE_DIR, self.folder_name) + os.path.sep)
|
|
117
|
+
|
|
118
|
+
def set_folder_name(self, folder_name):
|
|
119
|
+
"""Override the target folder/pdf name before downloading."""
|
|
120
|
+
self.folder_name = folder_name
|
|
121
|
+
self.set_folder_path(os.path.join(BASE_DIR, self.folder_name) + os.path.sep)
|
|
122
|
+
|
|
123
|
+
def verbose_print(self):
|
|
124
|
+
self.verbose = True
|
|
125
|
+
|
|
126
|
+
def set_folder_path(self, folder_path):
|
|
127
|
+
self.folder_path = folder_path
|
|
128
|
+
self.find_existing_files()
|
|
129
|
+
|
|
130
|
+
def load_cookie(self, file_path):
|
|
131
|
+
with open(file_path) as f:
|
|
132
|
+
for line in f:
|
|
133
|
+
if "=" in line:
|
|
134
|
+
key, value = map(str.strip, line.split("=", 1))
|
|
135
|
+
if key in ["authorization", "cookie"]:
|
|
136
|
+
self.session.headers[key] = value
|
|
137
|
+
if self.verbose:
|
|
138
|
+
print(self.session.headers)
|
|
139
|
+
|
|
140
|
+
def find_existing_files(self):
|
|
141
|
+
filelist = []
|
|
142
|
+
filelist.extend(glob(os.path.join(f"{self.folder_path}*.jpg")))
|
|
143
|
+
if filelist:
|
|
144
|
+
for file in filelist:
|
|
145
|
+
self.existing_images.extend(
|
|
146
|
+
[file.split(".jpg")[0].split(os.path.sep)[2]]
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
def set_resize(self, size):
|
|
150
|
+
self.resize = int(size) / 100
|
|
151
|
+
|
|
152
|
+
def set_from_page(self, frompage):
|
|
153
|
+
pages = []
|
|
154
|
+
for page in self.page_names:
|
|
155
|
+
if int(page) >= frompage:
|
|
156
|
+
pages.append(page)
|
|
157
|
+
self.page_names = pages
|
|
158
|
+
|
|
159
|
+
def set_to_page(self, topage):
|
|
160
|
+
pages = []
|
|
161
|
+
for page in self.page_names:
|
|
162
|
+
if int(page) <= topage:
|
|
163
|
+
pages.append(page)
|
|
164
|
+
self.page_names = pages
|
|
165
|
+
|
|
166
|
+
def get_manifest(self):
|
|
167
|
+
manifest_url = f"{self.api_url}_{self.media_type}_{self.media_id}/manifest"
|
|
168
|
+
try:
|
|
169
|
+
response = self.session.get(manifest_url)
|
|
170
|
+
response.raise_for_status()
|
|
171
|
+
json_data = response.json()
|
|
172
|
+
except RequestException as error:
|
|
173
|
+
print(error)
|
|
174
|
+
return
|
|
175
|
+
# retain raw metadata for preview
|
|
176
|
+
self.raw_metadata = json_data.get('metadata', [])
|
|
177
|
+
# grab manifest-level thumbnail if provided
|
|
178
|
+
thumb_val = json_data.get('thumbnail')
|
|
179
|
+
if thumb_val:
|
|
180
|
+
# thumbnail may be dict or list
|
|
181
|
+
if isinstance(thumb_val, list) and thumb_val:
|
|
182
|
+
t = thumb_val[0]
|
|
183
|
+
else:
|
|
184
|
+
t = thumb_val
|
|
185
|
+
# IIIF Presentation v3 uses 'id', v2 uses '@id'
|
|
186
|
+
if isinstance(t, dict):
|
|
187
|
+
self.manifest_thumbnail = t.get('id') or t.get('@id')
|
|
188
|
+
elif isinstance(t, str):
|
|
189
|
+
self.manifest_thumbnail = t
|
|
190
|
+
metadata = self.raw_metadata or []
|
|
191
|
+
if metadata:
|
|
192
|
+
first_entry = metadata[0]
|
|
193
|
+
tilgang_val = None
|
|
194
|
+
if isinstance(first_entry, dict):
|
|
195
|
+
tilgang_val = first_entry.get("value")
|
|
196
|
+
elif isinstance(first_entry, str):
|
|
197
|
+
tilgang_val = first_entry
|
|
198
|
+
if tilgang_val:
|
|
199
|
+
self.tilgang = tilgang_val
|
|
200
|
+
label_candidate = self._extract_label_text(json_data.get("label"))
|
|
201
|
+
if not label_candidate and len(metadata) > 1:
|
|
202
|
+
second_entry = metadata[1]
|
|
203
|
+
if isinstance(second_entry, dict):
|
|
204
|
+
label_candidate = second_entry.get("value", "")
|
|
205
|
+
elif isinstance(second_entry, str):
|
|
206
|
+
label_candidate = second_entry
|
|
207
|
+
if label_candidate:
|
|
208
|
+
self.title = re.sub(r"[^\w_. -]", "", label_candidate)
|
|
209
|
+
|
|
210
|
+
for page in json_data["sequences"][0]["canvases"]:
|
|
211
|
+
if self.media_type == "digavis":
|
|
212
|
+
page_name = page["@id"].split("_")[-2]
|
|
213
|
+
elif self.media_type == "digikart":
|
|
214
|
+
page_name = (
|
|
215
|
+
page["@id"].split("_")[-2] + "_" +
|
|
216
|
+
page["@id"].split("_")[-1]
|
|
217
|
+
)
|
|
218
|
+
else:
|
|
219
|
+
page_name = page["@id"].split("_")[-1]
|
|
220
|
+
page_dims = [page["width"], page["height"]]
|
|
221
|
+
if page_name.isdecimal():
|
|
222
|
+
self.page_names.append(page_name)
|
|
223
|
+
self.page_data[page_name] = page_dims
|
|
224
|
+
self.page_url[page_name] = page["images"][0]["resource"]["service"]["@id"]
|
|
225
|
+
self.page_names = sorted(self.page_names)
|
|
226
|
+
self.num_pages = len(self.page_names)
|
|
227
|
+
|
|
228
|
+
def fetch_new_image_url(self, side, column, row):
|
|
229
|
+
# compute region to request, clamped to page bounds to avoid oversize requests
|
|
230
|
+
orig_w, orig_h = self.page_data[side]
|
|
231
|
+
x = int(column) * self.tile_width
|
|
232
|
+
y = int(row) * self.tile_height
|
|
233
|
+
# clamp width/height for last tiles so as not to exceed page dimensions
|
|
234
|
+
region_w = min(self.tile_width, orig_w - x)
|
|
235
|
+
region_h = min(self.tile_height, orig_h - y)
|
|
236
|
+
image_url = (
|
|
237
|
+
f"{self.page_url[side]}/"
|
|
238
|
+
f"{x},{y},{region_w},{region_h}"
|
|
239
|
+
f"/full/0/native.jpg"
|
|
240
|
+
)
|
|
241
|
+
if self.print_url:
|
|
242
|
+
print(f"Side: {side}, Col: {column}, Row: {row}")
|
|
243
|
+
print(image_url)
|
|
244
|
+
return image_url
|
|
245
|
+
|
|
246
|
+
def update_column_row(self, side):
|
|
247
|
+
column_number, row_number = 0, 0
|
|
248
|
+
# use smaller tile sizes for 'plikt' resources to avoid access restrictions
|
|
249
|
+
if self.media_type.startswith("plikt"):
|
|
250
|
+
self.set_tile_sizes(300, 300)
|
|
251
|
+
elif self.media_type in ("digibok", "digitidsskrift"):
|
|
252
|
+
self.set_tile_sizes(1024, 1024)
|
|
253
|
+
else:
|
|
254
|
+
self.set_tile_sizes(4096, 4096)
|
|
255
|
+
column_number = ceil(self.page_data[side][0] / self.tile_width) - 1
|
|
256
|
+
row_number = ceil(self.page_data[side][1] / self.tile_height) - 1
|
|
257
|
+
return (int(column_number), int(row_number))
|
|
258
|
+
|
|
259
|
+
def download_covers(self):
|
|
260
|
+
# include cover pages in download AND PDF ordering
|
|
261
|
+
self.covers = True
|
|
262
|
+
self.include_cover = True
|
|
263
|
+
|
|
264
|
+
def set_include_cover(self, flag=True):
|
|
265
|
+
"""If True, put C1.jpg as first page in generated PDF."""
|
|
266
|
+
self.include_cover = bool(flag)
|
|
267
|
+
|
|
268
|
+
def download(self):
|
|
269
|
+
# prepare directories: sources (images), metadata, pdf
|
|
270
|
+
os.makedirs(self.sources_dir, exist_ok=True)
|
|
271
|
+
os.makedirs(self.meta_dir, exist_ok=True)
|
|
272
|
+
os.makedirs(self.pdf_dir, exist_ok=True)
|
|
273
|
+
# record original ID and title metadata for UI gallery
|
|
274
|
+
# write metadata for gallery
|
|
275
|
+
# write metadata for gallery UI only in webapp mode
|
|
276
|
+
if not self.cli_mode:
|
|
277
|
+
try:
|
|
278
|
+
with open(os.path.join(self.meta_dir, '.nbno_id'), 'w') as mf:
|
|
279
|
+
mf.write(self.digimedie)
|
|
280
|
+
# use manifest thumbnail URL (IIIF) rather than local preview logic
|
|
281
|
+
thumb = self.manifest_thumbnail
|
|
282
|
+
import datetime, json
|
|
283
|
+
meta = {
|
|
284
|
+
'orig': self.digimedie,
|
|
285
|
+
'title': self.title,
|
|
286
|
+
'type': self.media_type,
|
|
287
|
+
'pages': self.num_pages,
|
|
288
|
+
'thumbnail': thumb,
|
|
289
|
+
# use timezone-aware now() rather than deprecated utcnow()
|
|
290
|
+
'timestamp': int(datetime.datetime.now(datetime.timezone.utc).timestamp()),
|
|
291
|
+
}
|
|
292
|
+
if hasattr(self, 'custom_title'):
|
|
293
|
+
meta['custom_title'] = self.custom_title
|
|
294
|
+
with open(os.path.join(self.meta_dir, '.nbno_meta.json'), 'w', encoding='utf-8') as mf:
|
|
295
|
+
json.dump(meta, mf)
|
|
296
|
+
except Exception:
|
|
297
|
+
pass
|
|
298
|
+
self.download_skipped = False
|
|
299
|
+
imagelist = self.page_names
|
|
300
|
+
if self.covers:
|
|
301
|
+
# order covers same as GUI/web PDF: C1, I1, numbered pages, I3, C2, C3
|
|
302
|
+
front = []
|
|
303
|
+
back = []
|
|
304
|
+
for cover in ("C1", "I1"): # front covers
|
|
305
|
+
if cover in self.page_data:
|
|
306
|
+
front.append(cover)
|
|
307
|
+
# numeric pages (page_names are already sorted)
|
|
308
|
+
numeric = [p for p in imagelist if p.isdecimal()]
|
|
309
|
+
for cover in ("I3", "C2", "C3"): # back covers
|
|
310
|
+
if cover in self.page_data:
|
|
311
|
+
back.append(cover)
|
|
312
|
+
imagelist = front + numeric + back
|
|
313
|
+
if self.existing_images:
|
|
314
|
+
counter = 0
|
|
315
|
+
for image in self.existing_images:
|
|
316
|
+
if image in imagelist:
|
|
317
|
+
counter += 1
|
|
318
|
+
imagelist.remove(image)
|
|
319
|
+
print(f"{' '*5}Hopper over {counter} eksisterende sider.")
|
|
320
|
+
if len(imagelist) == 0:
|
|
321
|
+
if self.existing_images:
|
|
322
|
+
print(f"{' '*5}Alle bildene finnes allerede lokalt; hopper over nedlasting.")
|
|
323
|
+
self.download_skipped = True
|
|
324
|
+
return True
|
|
325
|
+
return False
|
|
326
|
+
else:
|
|
327
|
+
with cf.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
|
328
|
+
future_download = {
|
|
329
|
+
executor.submit(self.download_page, page): page
|
|
330
|
+
for page in imagelist
|
|
331
|
+
}
|
|
332
|
+
progress = 0
|
|
333
|
+
print("")
|
|
334
|
+
for future in cf.as_completed(future_download):
|
|
335
|
+
download = future.result()
|
|
336
|
+
if not download[0]:
|
|
337
|
+
if download[1] == 403:
|
|
338
|
+
print(f"HTTP 403 Forbidden: Får ikke tilgang til boken.")
|
|
339
|
+
print(f"Fra nb.nb - {self.tilgang}.")
|
|
340
|
+
elif download[1] == 408:
|
|
341
|
+
print(f"Connection Timeout ved forsøk på å laste sider.")
|
|
342
|
+
executor.shutdown(wait=False, cancel_futures=True)
|
|
343
|
+
break
|
|
344
|
+
else:
|
|
345
|
+
progress += 1
|
|
346
|
+
if not self.verbose:
|
|
347
|
+
print(
|
|
348
|
+
f"{' ' * 5}Lagrer side {progress} av {len(imagelist)}.",
|
|
349
|
+
end="\r",
|
|
350
|
+
)
|
|
351
|
+
if self.verbose:
|
|
352
|
+
print(f"\n{' '*5}Lagrer side {progress} av {len(imagelist)}.")
|
|
353
|
+
return download[0]
|
|
354
|
+
|
|
355
|
+
def download_page(self, page_number):
|
|
356
|
+
"""Laster ned og setter sammen bildedeler for side av boken"""
|
|
357
|
+
max_column, max_row = self.update_column_row(page_number)
|
|
358
|
+
image_parts = []
|
|
359
|
+
max_width, max_height = 0, 0
|
|
360
|
+
row_counter, column_counter = 0, 0
|
|
361
|
+
row_number = 0
|
|
362
|
+
HTTPerror = 0
|
|
363
|
+
while row_number <= max_row:
|
|
364
|
+
column_number = 0
|
|
365
|
+
while column_number <= max_column:
|
|
366
|
+
url = self.fetch_new_image_url(
|
|
367
|
+
page_number, column_number, row_number)
|
|
368
|
+
try:
|
|
369
|
+
response = self.session.get(url, timeout=10)
|
|
370
|
+
response.raise_for_status()
|
|
371
|
+
except RequestException as error:
|
|
372
|
+
try:
|
|
373
|
+
response
|
|
374
|
+
except:
|
|
375
|
+
HTTPerror = 408
|
|
376
|
+
break
|
|
377
|
+
else:
|
|
378
|
+
if response.status_code == 403:
|
|
379
|
+
HTTPerror = 403
|
|
380
|
+
break
|
|
381
|
+
if self.print_error:
|
|
382
|
+
print(error)
|
|
383
|
+
else:
|
|
384
|
+
try:
|
|
385
|
+
img = Image.open(io.BytesIO(response.content))
|
|
386
|
+
with self.image_lock:
|
|
387
|
+
image_parts.append(img)
|
|
388
|
+
if row_number == 0:
|
|
389
|
+
max_width += img.size[0]
|
|
390
|
+
column_counter += 1
|
|
391
|
+
if column_number == 0:
|
|
392
|
+
max_height += img.size[1]
|
|
393
|
+
row_counter += 1
|
|
394
|
+
except IOError as error:
|
|
395
|
+
if self.print_error:
|
|
396
|
+
print(error)
|
|
397
|
+
column_number += 1
|
|
398
|
+
row_number += 1
|
|
399
|
+
if HTTPerror != 0:
|
|
400
|
+
break
|
|
401
|
+
if HTTPerror == 403:
|
|
402
|
+
return False, 403
|
|
403
|
+
elif HTTPerror == 408:
|
|
404
|
+
return False, 408
|
|
405
|
+
elif not len(image_parts):
|
|
406
|
+
return False, 200
|
|
407
|
+
else:
|
|
408
|
+
if len(image_parts) == (column_number * row_number):
|
|
409
|
+
part_width, part_height = image_parts[0].size
|
|
410
|
+
full_page = Image.new("RGB", (max_width, max_height))
|
|
411
|
+
row_number = 0
|
|
412
|
+
part_counter = 0
|
|
413
|
+
row_counter = row_counter - 1
|
|
414
|
+
column_counter = column_counter - 1
|
|
415
|
+
while row_number <= row_counter:
|
|
416
|
+
column_number = 0
|
|
417
|
+
while column_number <= column_counter:
|
|
418
|
+
full_page.paste(
|
|
419
|
+
image_parts[part_counter],
|
|
420
|
+
((column_number * part_width),
|
|
421
|
+
(row_number * part_height)),
|
|
422
|
+
)
|
|
423
|
+
part_counter += 1
|
|
424
|
+
column_number += 1
|
|
425
|
+
row_number += 1
|
|
426
|
+
if self.resize != 0:
|
|
427
|
+
full_page = full_page.resize(
|
|
428
|
+
[int(self.resize * s) for s in full_page.size]
|
|
429
|
+
)
|
|
430
|
+
full_page.save(os.path.join(self.sources_dir, f"{page_number}.jpg"))
|
|
431
|
+
if self.verbose:
|
|
432
|
+
print(f"{' '*5}Lagret side {page_number}.jpg")
|
|
433
|
+
return True, 200
|
|
434
|
+
else:
|
|
435
|
+
if page_number in ["I3", "I1", "C3", "C2", "C1"]:
|
|
436
|
+
print(f"Feilet å laste ned side {page_number}.jpg - hopper over.")
|
|
437
|
+
return True, 200
|
|
438
|
+
else:
|
|
439
|
+
print(f"Feilet å laste ned side {page_number}.jpg - prøver igjen.")
|
|
440
|
+
return self.download_page(page_number)
|
|
441
|
+
|
|
442
|
+
def _attempt_redownload_page_for_pdf(self, path):
|
|
443
|
+
page_name = os.path.splitext(os.path.basename(path))[0]
|
|
444
|
+
if page_name in self._pdf_redownload_attempts:
|
|
445
|
+
return False
|
|
446
|
+
self._pdf_redownload_attempts.add(page_name)
|
|
447
|
+
try:
|
|
448
|
+
os.remove(path)
|
|
449
|
+
except OSError:
|
|
450
|
+
pass
|
|
451
|
+
print(
|
|
452
|
+
f"{' '*5}Korrupt bildefil oppdaget for {page_name} under PDF-generering; "
|
|
453
|
+
"sletter og laster inn siden på nytt."
|
|
454
|
+
)
|
|
455
|
+
success, status = self.download_page(page_name)
|
|
456
|
+
if not success:
|
|
457
|
+
print(
|
|
458
|
+
f"Kunne ikke laste ned {page_name} på nytt (HTTP {status}); "
|
|
459
|
+
"PDF-generering avbrytes."
|
|
460
|
+
)
|
|
461
|
+
return success
|
|
462
|
+
|
|
463
|
+
def _open_pdf_image(self, path):
|
|
464
|
+
try:
|
|
465
|
+
return Image.open(path)
|
|
466
|
+
except (UnidentifiedImageError, IOError) as error:
|
|
467
|
+
if self.print_error:
|
|
468
|
+
print(error)
|
|
469
|
+
if self._attempt_redownload_page_for_pdf(path):
|
|
470
|
+
try:
|
|
471
|
+
return Image.open(path)
|
|
472
|
+
except (UnidentifiedImageError, IOError) as second_error:
|
|
473
|
+
if self.print_error:
|
|
474
|
+
print(second_error)
|
|
475
|
+
return None
|
|
476
|
+
|
|
477
|
+
def make_pdf(self):
|
|
478
|
+
"""Build a PDF from all downloaded JPGs in the folder."""
|
|
479
|
+
# filter out any warnings for large images
|
|
480
|
+
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
|
481
|
+
|
|
482
|
+
# list all .jpg files in sources subdir (and optionally include cover first)
|
|
483
|
+
pattern = os.path.join(self.sources_dir, "*.jpg")
|
|
484
|
+
files = sorted(glob(pattern))
|
|
485
|
+
if self.include_cover:
|
|
486
|
+
# desired order: C1, I1, numbered pages, I3, C2, C3
|
|
487
|
+
basenames = [os.path.basename(f) for f in files]
|
|
488
|
+
lower = [b.lower() for b in basenames]
|
|
489
|
+
ordered = []
|
|
490
|
+
# front covers
|
|
491
|
+
for name in ('c1.jpg', 'i1.jpg'):
|
|
492
|
+
if name in lower:
|
|
493
|
+
idx = lower.index(name)
|
|
494
|
+
ordered.append(files[idx])
|
|
495
|
+
# numeric pages
|
|
496
|
+
for fpath, bname in zip(files, basenames):
|
|
497
|
+
if bname[:-4].isdigit():
|
|
498
|
+
ordered.append(fpath)
|
|
499
|
+
# back covers
|
|
500
|
+
for name in ('i3.jpg', 'c2.jpg', 'c3.jpg'):
|
|
501
|
+
if name in lower:
|
|
502
|
+
idx = lower.index(name)
|
|
503
|
+
ordered.append(files[idx])
|
|
504
|
+
# append any remaining pages
|
|
505
|
+
for fpath in files:
|
|
506
|
+
if fpath not in ordered:
|
|
507
|
+
ordered.append(fpath)
|
|
508
|
+
files = ordered
|
|
509
|
+
if not files:
|
|
510
|
+
print("No images found to build PDF.")
|
|
511
|
+
return False
|
|
512
|
+
|
|
513
|
+
# name PDF after the folder_name, not the original ID
|
|
514
|
+
# ensure PDF_DIR exists
|
|
515
|
+
# ensure per-book pdf directory exists
|
|
516
|
+
os.makedirs(self.pdf_dir, exist_ok=True)
|
|
517
|
+
output_pdf = os.path.join(self.pdf_dir, f"{self.folder_name}.pdf")
|
|
518
|
+
# try efficient PDF assembly via img2pdf to reduce peak memory use
|
|
519
|
+
try:
|
|
520
|
+
import img2pdf
|
|
521
|
+
|
|
522
|
+
with open(output_pdf, "wb") as f:
|
|
523
|
+
f.write(img2pdf.convert(files))
|
|
524
|
+
print(f"PDF created: {output_pdf}")
|
|
525
|
+
return True
|
|
526
|
+
except ImportError:
|
|
527
|
+
pass
|
|
528
|
+
except Exception as e:
|
|
529
|
+
print(f"Error creating PDF via img2pdf: {e}")
|
|
530
|
+
print("Falling back to PIL for PDF creation.")
|
|
531
|
+
|
|
532
|
+
# fallback to PIL-based assembly (may use more memory)
|
|
533
|
+
self._pdf_redownload_attempts.clear()
|
|
534
|
+
try:
|
|
535
|
+
images = []
|
|
536
|
+
for path in files:
|
|
537
|
+
image = self._open_pdf_image(path)
|
|
538
|
+
if image is None:
|
|
539
|
+
return False
|
|
540
|
+
images.append(image)
|
|
541
|
+
images[0].save(
|
|
542
|
+
output_pdf,
|
|
543
|
+
"PDF",
|
|
544
|
+
resolution=100.0,
|
|
545
|
+
save_all=True,
|
|
546
|
+
append_images=images[1:]
|
|
547
|
+
)
|
|
548
|
+
print(f"PDF created: {output_pdf}")
|
|
549
|
+
return True
|
|
550
|
+
except Exception as e:
|
|
551
|
+
print(f"Error creating PDF: {e}")
|
|
552
|
+
return False
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def f2pdf(image_location, pdf_name):
|
|
556
|
+
try:
|
|
557
|
+
Image.open(image_location).save(
|
|
558
|
+
pdf_name + ".pdf", "PDF", resolution=100.0, append=True
|
|
559
|
+
)
|
|
560
|
+
except OSError:
|
|
561
|
+
Image.open(image_location).save(
|
|
562
|
+
pdf_name + ".pdf", "PDF", resolution=100.0)
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def main():
|
|
566
|
+
parser = argparse.ArgumentParser()
|
|
567
|
+
optional = parser._action_groups.pop()
|
|
568
|
+
required = parser.add_argument_group("required arguments")
|
|
569
|
+
required.add_argument(
|
|
570
|
+
"--id",
|
|
571
|
+
metavar="<bokID>",
|
|
572
|
+
help="IDen på mediet som skal lastes ned",
|
|
573
|
+
default=False,
|
|
574
|
+
)
|
|
575
|
+
optional.add_argument(
|
|
576
|
+
"--cover", action="store_true", help="Settes for å laste covers", default=False
|
|
577
|
+
)
|
|
578
|
+
optional.add_argument(
|
|
579
|
+
"--title",
|
|
580
|
+
action="store_true",
|
|
581
|
+
help="Settes for å hente tittel på bok automatisk",
|
|
582
|
+
default=False,
|
|
583
|
+
)
|
|
584
|
+
optional.add_argument(
|
|
585
|
+
"--pdf",
|
|
586
|
+
action="store_true",
|
|
587
|
+
help="Settes for å lage pdf av bildene som lastes",
|
|
588
|
+
default=False,
|
|
589
|
+
)
|
|
590
|
+
optional.add_argument(
|
|
591
|
+
"--f2pdf",
|
|
592
|
+
action="store_true",
|
|
593
|
+
help="Settes for å lage pdf av bilder i eksisterende mappe",
|
|
594
|
+
default=False,
|
|
595
|
+
)
|
|
596
|
+
optional.add_argument(
|
|
597
|
+
"--url",
|
|
598
|
+
action="store_true",
|
|
599
|
+
help="Settes for å printe URL av hver del",
|
|
600
|
+
default=False,
|
|
601
|
+
)
|
|
602
|
+
optional.add_argument(
|
|
603
|
+
"--error",
|
|
604
|
+
action="store_true",
|
|
605
|
+
help="Settes for å printe feilkoder",
|
|
606
|
+
default=False,
|
|
607
|
+
)
|
|
608
|
+
optional.add_argument(
|
|
609
|
+
"--v",
|
|
610
|
+
action="store_true",
|
|
611
|
+
help="Settes for å printe mer info",
|
|
612
|
+
default=False,
|
|
613
|
+
)
|
|
614
|
+
optional.add_argument(
|
|
615
|
+
"--resize",
|
|
616
|
+
metavar="<int>",
|
|
617
|
+
help="Prosent av originalstørrelse på bilder",
|
|
618
|
+
default=False,
|
|
619
|
+
)
|
|
620
|
+
optional.add_argument(
|
|
621
|
+
"--start", metavar="<int>", help="Sidetall å starte på", default=False
|
|
622
|
+
)
|
|
623
|
+
optional.add_argument(
|
|
624
|
+
"--stop", metavar="<int>", help="Sidetall å stoppe på", default=False
|
|
625
|
+
)
|
|
626
|
+
optional.add_argument(
|
|
627
|
+
"--cookie",
|
|
628
|
+
metavar="<string>",
|
|
629
|
+
help="Sti til fil for autentisering",
|
|
630
|
+
default=False
|
|
631
|
+
)
|
|
632
|
+
parser._action_groups.append(optional)
|
|
633
|
+
args = parser.parse_args()
|
|
634
|
+
|
|
635
|
+
if args.id:
|
|
636
|
+
if args.f2pdf:
|
|
637
|
+
media_type = "dig" + args.id.split("dig")[1].split("_")[0]
|
|
638
|
+
media_id = str(args.id.split(media_type + "_")[1])
|
|
639
|
+
filelist = []
|
|
640
|
+
filelist.extend(
|
|
641
|
+
glob(os.path.join(str(media_id), ("[0-9]" * 4) + ".jpg")))
|
|
642
|
+
if not filelist:
|
|
643
|
+
filelist.extend(
|
|
644
|
+
glob(os.path.join(str(media_id), ("[0-9]" * 3) + ".jpg"))
|
|
645
|
+
)
|
|
646
|
+
filelist = sorted(filelist)
|
|
647
|
+
print(f"\nLager {media_id}.pdf\n")
|
|
648
|
+
if args.cover:
|
|
649
|
+
filelist = [f"{media_id}/C1.jpg",
|
|
650
|
+
*filelist, f"{media_id}/C3.jpg"]
|
|
651
|
+
for file in filelist:
|
|
652
|
+
f2pdf(file, str(media_id))
|
|
653
|
+
print(
|
|
654
|
+
f"{' '*5}{file.split(os.path.sep)[1]} --> {media_id}.pdf", end="\r"
|
|
655
|
+
)
|
|
656
|
+
if args.v:
|
|
657
|
+
print("")
|
|
658
|
+
print("\n\nFerdig med å lage pdf.")
|
|
659
|
+
exit()
|
|
660
|
+
# CLI mode: flat directory structure for direct downloads
|
|
661
|
+
book = Book(args.id, cli_mode=True)
|
|
662
|
+
if args.url:
|
|
663
|
+
book.set_to_print_url()
|
|
664
|
+
if args.error:
|
|
665
|
+
book.set_to_print_errors()
|
|
666
|
+
if args.v:
|
|
667
|
+
book.verbose_print()
|
|
668
|
+
if args.cookie:
|
|
669
|
+
if os.path.exists(args.cookie):
|
|
670
|
+
book.load_cookie(args.cookie)
|
|
671
|
+
else:
|
|
672
|
+
print(f"Fil for autentisering ikke funnet: {args.cookie}")
|
|
673
|
+
exit()
|
|
674
|
+
if args.resize:
|
|
675
|
+
book.set_resize(int(args.resize))
|
|
676
|
+
if args.start:
|
|
677
|
+
book.set_from_page(int(args.start))
|
|
678
|
+
if args.stop:
|
|
679
|
+
book.set_to_page(int(args.stop))
|
|
680
|
+
if int(args.stop) > book.num_pages:
|
|
681
|
+
print("Du har forsøkt å laste ned flere sider enn det eksisterer.")
|
|
682
|
+
print(
|
|
683
|
+
f"Det finnes kun {book.num_pages} sider, du får ikke flere enn dette."
|
|
684
|
+
)
|
|
685
|
+
print(f"Laster ned {book.media_type} med ID: {book.media_id}.")
|
|
686
|
+
if args.cover:
|
|
687
|
+
book.download_covers()
|
|
688
|
+
if args.title:
|
|
689
|
+
book.set_title()
|
|
690
|
+
download = book.download()
|
|
691
|
+
if download == False:
|
|
692
|
+
print(
|
|
693
|
+
f"\nNoe gikk galt, du prøvde å laste ned {book.media_type}, "
|
|
694
|
+
f"med id {book.media_id}, er dette korrekt?"
|
|
695
|
+
)
|
|
696
|
+
exit()
|
|
697
|
+
else:
|
|
698
|
+
if not book.download_skipped:
|
|
699
|
+
print(f"\n{' '*5}Ferdig med å laste ned alle sider.\n")
|
|
700
|
+
if args.pdf:
|
|
701
|
+
print(f"\nLager {book.media_id}.pdf")
|
|
702
|
+
savepdf = book.make_pdf()
|
|
703
|
+
if savepdf:
|
|
704
|
+
print(f"\n{' '*5}Ferdig med å lage pdf.\n")
|
|
705
|
+
exit()
|
|
706
|
+
else:
|
|
707
|
+
parser.print_help()
|
|
708
|
+
exit()
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
if __name__ == "__main__":
|
|
712
|
+
main()
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "nbno"
|
|
7
|
+
version = "1.2.3"
|
|
8
|
+
description = "NB.no nedlaster"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"Pillow>=12,<13",
|
|
13
|
+
"requests>=2.32,<3",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[project.optional-dependencies]
|
|
17
|
+
web = [
|
|
18
|
+
"Flask>=2,<4",
|
|
19
|
+
"ocrmypdf>=13.5",
|
|
20
|
+
"img2pdf>=0.4",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.scripts]
|
|
24
|
+
nbno = "nbno:main"
|
|
25
|
+
|
|
26
|
+
[tool.setuptools]
|
|
27
|
+
py-modules = ["nbno"]
|
nbno-1.2.3/setup.cfg
ADDED