Dragon-Ball-API-Alejo 0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dragon_ball_api_alejo-0.1/DragonBall/Constants/ScrapCharacter.py +53 -0
- dragon_ball_api_alejo-0.1/DragonBall/Constants/ScrapFusion.py +36 -0
- dragon_ball_api_alejo-0.1/DragonBall/Constants/ScrapGames.py +35 -0
- dragon_ball_api_alejo-0.1/DragonBall/Constants/ScrapPlanet.py +32 -0
- dragon_ball_api_alejo-0.1/DragonBall/Constants/ScrapSagas.py +36 -0
- dragon_ball_api_alejo-0.1/DragonBall/Constants/URLS.py +15 -0
- dragon_ball_api_alejo-0.1/DragonBall/Data/Characters.py +130 -0
- dragon_ball_api_alejo-0.1/DragonBall/Data/Fusions.py +122 -0
- dragon_ball_api_alejo-0.1/DragonBall/Data/OutInformation.py +28 -0
- dragon_ball_api_alejo-0.1/DragonBall/Data/Planets.py +123 -0
- dragon_ball_api_alejo-0.1/DragonBall/Data/Sagas.py +130 -0
- dragon_ball_api_alejo-0.1/DragonBall/Data/VideoGames.py +122 -0
- dragon_ball_api_alejo-0.1/DragonBall/Soup/section.py +17 -0
- dragon_ball_api_alejo-0.1/DragonBall/Soup/summary.py +18 -0
- dragon_ball_api_alejo-0.1/DragonBall/Soup/tags.py +53 -0
- dragon_ball_api_alejo-0.1/DragonBall/__init__.py +6 -0
- dragon_ball_api_alejo-0.1/Dragon_Ball_API_Alejo.egg-info/PKG-INFO +111 -0
- dragon_ball_api_alejo-0.1/Dragon_Ball_API_Alejo.egg-info/SOURCES.txt +24 -0
- dragon_ball_api_alejo-0.1/Dragon_Ball_API_Alejo.egg-info/dependency_links.txt +1 -0
- dragon_ball_api_alejo-0.1/Dragon_Ball_API_Alejo.egg-info/requires.txt +2 -0
- dragon_ball_api_alejo-0.1/Dragon_Ball_API_Alejo.egg-info/top_level.txt +1 -0
- dragon_ball_api_alejo-0.1/PKG-INFO +111 -0
- dragon_ball_api_alejo-0.1/README.md +85 -0
- dragon_ball_api_alejo-0.1/setup.cfg +4 -0
- dragon_ball_api_alejo-0.1/setup.py +35 -0
- dragon_ball_api_alejo-0.1/tests/test_smoke.py +38 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
class ScrapCharacter:
|
|
2
|
+
# these names have to be exactly the same as the asyde region
|
|
3
|
+
|
|
4
|
+
sections = {
|
|
5
|
+
"names": "Nombres",
|
|
6
|
+
"data": "Datos",
|
|
7
|
+
"debut": "Debut"
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
# font for character names section
|
|
11
|
+
|
|
12
|
+
names = {
|
|
13
|
+
"japan_name": "nombre-ja",
|
|
14
|
+
"latin_name": "nombre-ja_latino",
|
|
15
|
+
"pronunciation_name": "pronunciación",
|
|
16
|
+
"other_names": "otros nombres"
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
# font for character data section
|
|
20
|
+
|
|
21
|
+
data = {
|
|
22
|
+
"gender": "sexo",
|
|
23
|
+
"birthday": "fecha de nacimiento",
|
|
24
|
+
"deaths": "fecha de muerte",
|
|
25
|
+
"age": "edad",
|
|
26
|
+
"species": "raza",
|
|
27
|
+
"occupation": "ocupación",
|
|
28
|
+
"origin": "procedencia",
|
|
29
|
+
"residence": "residencia",
|
|
30
|
+
"transformations": "transformaciones",
|
|
31
|
+
"fusion": "fusiones",
|
|
32
|
+
"family": "familia",
|
|
33
|
+
"affiliates": "afiliados"
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
# font for character debut section
|
|
37
|
+
|
|
38
|
+
debut = {
|
|
39
|
+
"sagas": "saga",
|
|
40
|
+
"debut": "Primera aparición",
|
|
41
|
+
"manga": "manga",
|
|
42
|
+
"volume": "tomo",
|
|
43
|
+
"anime": "anime",
|
|
44
|
+
"introduction": "arco"
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
# list of names to use according to sections
|
|
48
|
+
|
|
49
|
+
list_character = {
|
|
50
|
+
"names": list(names.values()),
|
|
51
|
+
"data": list(data.values()),
|
|
52
|
+
"debut": list(debut.values())
|
|
53
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
class ScrapFusion:
|
|
2
|
+
# these names have to be exactly the same as the asyde region
|
|
3
|
+
|
|
4
|
+
sections = {
|
|
5
|
+
"names": "Nombres",
|
|
6
|
+
"data": "Otros datos",
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
# font for character names section
|
|
10
|
+
|
|
11
|
+
names = {
|
|
12
|
+
"japan_name": "nombre-ja",
|
|
13
|
+
"latin_name": "nombre-ja_latino",
|
|
14
|
+
"pronunciation_name": "pronunciación",
|
|
15
|
+
"other_names": "otros nombres"
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
# font for character data section
|
|
19
|
+
|
|
20
|
+
data = {
|
|
21
|
+
"gender": "sexo",
|
|
22
|
+
"date of origin": "fecha de origen",
|
|
23
|
+
"species": "raza",
|
|
24
|
+
"fusion": "fusionados",
|
|
25
|
+
"method_used": "método",
|
|
26
|
+
"transformations": "transformaciones",
|
|
27
|
+
"apparition": "aparición",
|
|
28
|
+
"debut": "Primera aparición"
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
# list of names to use according to sections
|
|
32
|
+
|
|
33
|
+
list_character = {
|
|
34
|
+
"names": list(names.values()),
|
|
35
|
+
"data": list(data.values()),
|
|
36
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
class ScrapGames:
|
|
2
|
+
# these names have to be exactly the same as the asyde region
|
|
3
|
+
|
|
4
|
+
sections = {
|
|
5
|
+
"names": "Nombres",
|
|
6
|
+
"data": "Datos",
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
# font for character names section
|
|
10
|
+
|
|
11
|
+
names = {
|
|
12
|
+
"japan_name": "nombre-ja",
|
|
13
|
+
"latin_name": "nombre-ja_latino",
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
# font for character data section
|
|
17
|
+
|
|
18
|
+
data = {
|
|
19
|
+
"engine_game": "Motor",
|
|
20
|
+
"platforms": "Plataforma",
|
|
21
|
+
"developers": "Desarrolladores",
|
|
22
|
+
"dealers": "Distribuidores",
|
|
23
|
+
"launch": "Lanzamiento",
|
|
24
|
+
"game_mode": "Modos de Juego",
|
|
25
|
+
"gender": "Género",
|
|
26
|
+
"classification": "Clasificaciones",
|
|
27
|
+
"sagas": "Sagas"
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
# list of names to use according to sections
|
|
31
|
+
|
|
32
|
+
list_character = {
|
|
33
|
+
"names": list(names.values()),
|
|
34
|
+
"data": list(data.values())
|
|
35
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
class ScrapPlanet:
|
|
2
|
+
# these names have to be exactly the same as the asyde region
|
|
3
|
+
|
|
4
|
+
sections = {
|
|
5
|
+
"names": "Nombres",
|
|
6
|
+
"data": "Datos",
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
# font for character names section
|
|
10
|
+
|
|
11
|
+
names = {
|
|
12
|
+
"japan_name": "nombre-ja",
|
|
13
|
+
"latin_name": "nombre-ja_latino",
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
# font for character data section
|
|
17
|
+
|
|
18
|
+
data = {
|
|
19
|
+
"localization": "localización",
|
|
20
|
+
"gobernant": "gobernante",
|
|
21
|
+
"residents": "residentes",
|
|
22
|
+
"especies": "especie",
|
|
23
|
+
"sagas": "saga",
|
|
24
|
+
"debut": "aparición",
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
# list of names to use according to sections
|
|
28
|
+
|
|
29
|
+
list_character = {
|
|
30
|
+
"names": list(names.values()),
|
|
31
|
+
"data": list(data.values())
|
|
32
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
class ScrapSagas:
|
|
2
|
+
# these names have to be exactly the same as the asyde region
|
|
3
|
+
|
|
4
|
+
sections = {
|
|
5
|
+
"names": "Títulos",
|
|
6
|
+
"data": "Datos",
|
|
7
|
+
"serialization": "Serialización"
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
# font for character names section
|
|
11
|
+
|
|
12
|
+
names = {
|
|
13
|
+
"japan_name": "nombre-ja",
|
|
14
|
+
"latin_name": "nombre-ja_latino",
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
# font for character data section
|
|
18
|
+
|
|
19
|
+
data = {
|
|
20
|
+
"episodes": "número de episodios (Anime)",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
# font for serialization data section
|
|
24
|
+
|
|
25
|
+
serialization = {
|
|
26
|
+
"first_emision": "primera emisión",
|
|
27
|
+
"last_emision": "última emisión",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
# list of names to use according to sections
|
|
31
|
+
|
|
32
|
+
list_character = {
|
|
33
|
+
"names": list(names.values()),
|
|
34
|
+
"data": list(data.values()),
|
|
35
|
+
"serialization": list(serialization.values())
|
|
36
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
class URL:
|
|
2
|
+
URL_BASE = "https://dragonball.fandom.com/"
|
|
3
|
+
URL_WIKI = "https://dragonball.fandom.com/es/wiki/"
|
|
4
|
+
|
|
5
|
+
HEADERS = {
|
|
6
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
|
|
7
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
8
|
+
"Accept-Language": "es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3"
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
URL_LIST_CHARACTERS = URL_WIKI + "Lista_de_personajes"
|
|
12
|
+
URL_LIST_FUSIONS = URL_WIKI + "Lista_de_fusiones"
|
|
13
|
+
URL_LIST_PLANETS = URL_WIKI + "Lista_de_planetas"
|
|
14
|
+
URL_LIST_CHRONO = URL_WIKI + "Cronología"
|
|
15
|
+
URL_LIST_GAMES = URL_WIKI + "Lista_de_videojuegos"
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import requests as requests
|
|
2
|
+
from bs4 import BeautifulSoup
|
|
3
|
+
|
|
4
|
+
from DragonBall.Constants.ScrapCharacter import ScrapCharacter
|
|
5
|
+
from DragonBall.Constants.URLS import URL
|
|
6
|
+
from DragonBall.Data.OutInformation import OutInformation
|
|
7
|
+
from DragonBall.Soup.section import Section
|
|
8
|
+
from DragonBall.Soup.summary import Summary
|
|
9
|
+
from DragonBall.Soup.tags import Tags
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Characters:
|
|
13
|
+
def __init__(self, name: str, all_data=False):
|
|
14
|
+
self.name = name
|
|
15
|
+
self.all_data = all_data
|
|
16
|
+
|
|
17
|
+
@classmethod
|
|
18
|
+
def character_list(cls, character_list: list):
|
|
19
|
+
return [Characters(i).character_information() for i in character_list]
|
|
20
|
+
|
|
21
|
+
@staticmethod
|
|
22
|
+
def list():
|
|
23
|
+
request = requests.get(URL.URL_LIST_CHARACTERS, headers=URL.HEADERS)
|
|
24
|
+
soup = BeautifulSoup(request.content.decode('utf-8'), "html.parser")
|
|
25
|
+
toc = soup.find("div", {"id": "toc"})
|
|
26
|
+
if not toc:
|
|
27
|
+
return {}
|
|
28
|
+
summary_data = toc.text.strip().splitlines()
|
|
29
|
+
return Summary.clean_list_summary(summary_data, [1, -1])
|
|
30
|
+
|
|
31
|
+
def character_information(self):
|
|
32
|
+
url = URL.URL_WIKI + self.name.replace(" ", "_")
|
|
33
|
+
request = requests.get(url, headers=URL.HEADERS)
|
|
34
|
+
soup = BeautifulSoup(request.content.decode('utf-8'), "html.parser")
|
|
35
|
+
list_section = Section.obtain_section_data(soup, 'aside')
|
|
36
|
+
basic_dict = self.__obtain_basic_data(soup, url)
|
|
37
|
+
section_dict = self.__evaluate_scrapper_section(list_section, list(list_section.keys()), ScrapCharacter)
|
|
38
|
+
return self.__show_information(basic_dict, section_dict)
|
|
39
|
+
|
|
40
|
+
def __obtain_basic_data(self, soup: BeautifulSoup, url: str):
|
|
41
|
+
name = self.__get_official_name(soup, "h1", "id", "firstHeading")
|
|
42
|
+
image = self.__get_image(soup, "a", "class", "image image-thumbnail")
|
|
43
|
+
description = self.__get_description_character(soup)
|
|
44
|
+
return {"url": url, "name": name, "image": image, "descripcion": description}
|
|
45
|
+
|
|
46
|
+
@staticmethod
|
|
47
|
+
def __get_official_name(soup: BeautifulSoup, tag: str, identification: str, element: str):
|
|
48
|
+
# Intento 1: Buscar el h1 con el id exacto
|
|
49
|
+
official_name = Tags.custom_catch_error(soup, tag, identification, element)
|
|
50
|
+
if official_name != OutInformation.RESOURCE_NOT_FOUND:
|
|
51
|
+
return official_name.get_text(strip=True)
|
|
52
|
+
|
|
53
|
+
# Intento 2: Buscar CUALQUIER etiqueta <h1> en la página
|
|
54
|
+
h1 = soup.find("h1")
|
|
55
|
+
if h1:
|
|
56
|
+
return h1.get_text(strip=True)
|
|
57
|
+
|
|
58
|
+
# Intento 3: Usar el título de la pestaña del navegador
|
|
59
|
+
if soup.title:
|
|
60
|
+
return soup.title.text.split("|")[0].strip()
|
|
61
|
+
|
|
62
|
+
return OutInformation.RESOURCE_NOT_FOUND
|
|
63
|
+
|
|
64
|
+
@staticmethod
|
|
65
|
+
def __get_image(soup: BeautifulSoup, tag: str, identification: str, name: str):
|
|
66
|
+
found = soup.find(tag, attrs={identification: name})
|
|
67
|
+
if found:
|
|
68
|
+
return found.get("href")
|
|
69
|
+
return OutInformation.RESOURCE_NOT_FOUND
|
|
70
|
+
|
|
71
|
+
@staticmethod
|
|
72
|
+
def __get_description_character(soup: BeautifulSoup):
|
|
73
|
+
[remove_sup.extract() for remove_sup in soup.find_all('sup')]
|
|
74
|
+
try:
|
|
75
|
+
return next(
|
|
76
|
+
p.text.strip() for header in soup.find_all(['h2', 'h3']) if (p := header.find_next_sibling('p')))
|
|
77
|
+
except StopIteration:
|
|
78
|
+
return OutInformation.RESOURCE_NOT_FOUND
|
|
79
|
+
|
|
80
|
+
def __evaluate_scrapper_section(self, list_dict, list_search, tag_found: ScrapCharacter):
|
|
81
|
+
empty = self.__return_empty_section(list_dict)
|
|
82
|
+
if empty:
|
|
83
|
+
return empty
|
|
84
|
+
|
|
85
|
+
section_names = list(tag_found.sections.values())
|
|
86
|
+
names = list_dict.get(section_names[0], OutInformation.RESOURCE_NOT_FOUND)
|
|
87
|
+
data = list_dict.get(section_names[1], OutInformation.RESOURCE_NOT_FOUND)
|
|
88
|
+
debut = list_dict.get(section_names[2], OutInformation.RESOURCE_NOT_FOUND)
|
|
89
|
+
|
|
90
|
+
return {
|
|
91
|
+
"nombres": self.__names_section(names, tag_found.names) if section_names[0] in list_search else names,
|
|
92
|
+
"data": self.__data_section(data, tag_found.data) if section_names[1] in list_search else data,
|
|
93
|
+
"debut": self.__debut_section(debut, tag_found.debut) if section_names[2] in list_search else debut,
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
@staticmethod
|
|
97
|
+
def __return_empty_section(list_dict: dict):
|
|
98
|
+
if not list_dict:
|
|
99
|
+
return {
|
|
100
|
+
"nombres": OutInformation.RESOURCE_NOT_FOUND,
|
|
101
|
+
"data": OutInformation.RESOURCE_NOT_FOUND,
|
|
102
|
+
"debut": OutInformation.RESOURCE_NOT_FOUND,
|
|
103
|
+
}
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
@staticmethod
|
|
107
|
+
def __names_section(name, lista_tags):
|
|
108
|
+
return {key: Tags.clean_text(Tags.catch_error(name, value)) for key, value in lista_tags.items()}
|
|
109
|
+
|
|
110
|
+
@staticmethod
|
|
111
|
+
def __data_section(data, lista_tags):
|
|
112
|
+
return {key: Tags.clean_text(Tags.catch_error(data, value)) for key, value in lista_tags.items()}
|
|
113
|
+
|
|
114
|
+
@staticmethod
|
|
115
|
+
def __debut_section(debut, lista_tags):
|
|
116
|
+
return {key: Tags.clean_text(Tags.catch_error(debut, value)) for key, value in lista_tags.items()}
|
|
117
|
+
|
|
118
|
+
def __show_information(self, dictionary_basic, dictionary_sections):
|
|
119
|
+
dictionary = {
|
|
120
|
+
"name": dictionary_basic['name'],
|
|
121
|
+
"image": dictionary_basic['image'],
|
|
122
|
+
"description": dictionary_basic['descripcion'],
|
|
123
|
+
"names": dictionary_sections['nombres'],
|
|
124
|
+
"data": dictionary_sections['data'],
|
|
125
|
+
"debut": dictionary_sections['debut'],
|
|
126
|
+
"url_scrap": dictionary_basic['url']
|
|
127
|
+
}
|
|
128
|
+
if self.all_data:
|
|
129
|
+
return dictionary
|
|
130
|
+
return OutInformation.clean_not_found_scrape(dictionary)
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import requests as requests
|
|
2
|
+
from bs4 import BeautifulSoup
|
|
3
|
+
|
|
4
|
+
from DragonBall.Constants.ScrapFusion import ScrapFusion
|
|
5
|
+
from DragonBall.Constants.URLS import URL
|
|
6
|
+
from DragonBall.Data.OutInformation import OutInformation
|
|
7
|
+
from DragonBall.Soup.section import Section
|
|
8
|
+
from DragonBall.Soup.summary import Summary
|
|
9
|
+
from DragonBall.Soup.tags import Tags
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Fusions:
|
|
13
|
+
def __init__(self, name: str, all_data=False):
|
|
14
|
+
self.name = name
|
|
15
|
+
self.all_data = all_data
|
|
16
|
+
|
|
17
|
+
@classmethod
|
|
18
|
+
def fusion_list(cls, fusion_list: list):
|
|
19
|
+
return [Fusions(i).fusion_information() for i in fusion_list]
|
|
20
|
+
|
|
21
|
+
@staticmethod
|
|
22
|
+
def list():
|
|
23
|
+
request = requests.get(URL.URL_LIST_FUSIONS, headers=URL.HEADERS)
|
|
24
|
+
soup = BeautifulSoup(request.content.decode('utf-8'), "html.parser")
|
|
25
|
+
toc = soup.find("div", {"id": "toc"})
|
|
26
|
+
if not toc:
|
|
27
|
+
return {}
|
|
28
|
+
summary_data = toc.text.strip().splitlines()
|
|
29
|
+
return Summary.clean_list_summary(summary_data, [1, -3])
|
|
30
|
+
|
|
31
|
+
def fusion_information(self):
|
|
32
|
+
url = URL.URL_WIKI + self.name.replace(" ", "_")
|
|
33
|
+
request = requests.get(url, headers=URL.HEADERS)
|
|
34
|
+
soup = BeautifulSoup(request.content.decode('utf-8'), "html.parser")
|
|
35
|
+
list_section = Section.obtain_section_data(soup, 'aside')
|
|
36
|
+
basic_dict = self.__obtain_basic_data(soup, url)
|
|
37
|
+
section_dict = self.__evaluate_scrapper_section(list_section, list(list_section.keys()), ScrapFusion)
|
|
38
|
+
return self.__show_information(basic_dict, section_dict)
|
|
39
|
+
|
|
40
|
+
def __obtain_basic_data(self, soup: BeautifulSoup, url: str):
|
|
41
|
+
name = self.__get_official_name(soup, "h1", "id", "firstHeading")
|
|
42
|
+
image = self.__get_image(soup, "a", "class", "image image-thumbnail")
|
|
43
|
+
description = self.__get_description_fusion(soup)
|
|
44
|
+
return {"url": url, "name": name, "image": image, "descripcion": description}
|
|
45
|
+
|
|
46
|
+
@staticmethod
|
|
47
|
+
def __get_official_name(soup: BeautifulSoup, tag: str, identification: str, element: str):
|
|
48
|
+
# Intento 1: Buscar el h1 con el id exacto
|
|
49
|
+
official_name = Tags.custom_catch_error(soup, tag, identification, element)
|
|
50
|
+
if official_name != OutInformation.RESOURCE_NOT_FOUND:
|
|
51
|
+
return official_name.get_text(strip=True)
|
|
52
|
+
|
|
53
|
+
# Intento 2: Buscar CUALQUIER etiqueta <h1> en la página
|
|
54
|
+
h1 = soup.find("h1")
|
|
55
|
+
if h1:
|
|
56
|
+
return h1.get_text(strip=True)
|
|
57
|
+
|
|
58
|
+
# Intento 3: Usar el título de la pestaña del navegador
|
|
59
|
+
if soup.title:
|
|
60
|
+
return soup.title.text.split("|")[0].strip()
|
|
61
|
+
|
|
62
|
+
return OutInformation.RESOURCE_NOT_FOUND
|
|
63
|
+
|
|
64
|
+
@staticmethod
|
|
65
|
+
def __get_image(soup: BeautifulSoup, tag: str, identification: str, name: str):
|
|
66
|
+
found = soup.find(tag, attrs={identification: name})
|
|
67
|
+
if found:
|
|
68
|
+
return found.get("href")
|
|
69
|
+
return OutInformation.RESOURCE_NOT_FOUND
|
|
70
|
+
|
|
71
|
+
@staticmethod
|
|
72
|
+
def __get_description_fusion(soup: BeautifulSoup):
|
|
73
|
+
[remove_sup.extract() for remove_sup in soup.find_all('sup')]
|
|
74
|
+
try:
|
|
75
|
+
return next(
|
|
76
|
+
p.text.strip() for header in soup.find_all(['h2', 'h3']) if (p := header.find_next_sibling('p')))
|
|
77
|
+
except StopIteration:
|
|
78
|
+
return OutInformation.RESOURCE_NOT_FOUND
|
|
79
|
+
|
|
80
|
+
def __evaluate_scrapper_section(self, list_dict, list_search, tag_found: ScrapFusion):
|
|
81
|
+
empty = self.__return_empty_section(list_dict)
|
|
82
|
+
if empty:
|
|
83
|
+
return empty
|
|
84
|
+
|
|
85
|
+
section_names = list(tag_found.sections.values())
|
|
86
|
+
names = list_dict.get(section_names[0], OutInformation.RESOURCE_NOT_FOUND)
|
|
87
|
+
data = list_dict.get(section_names[1], OutInformation.RESOURCE_NOT_FOUND)
|
|
88
|
+
|
|
89
|
+
return {
|
|
90
|
+
"nombres": self.__names_section(names, tag_found.names) if section_names[0] in list_search else names,
|
|
91
|
+
"data": self.__data_section(data, tag_found.data) if section_names[1] in list_search else data,
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
@staticmethod
|
|
95
|
+
def __return_empty_section(list_dict: dict):
|
|
96
|
+
if not list_dict:
|
|
97
|
+
return {
|
|
98
|
+
"nombres": OutInformation.RESOURCE_NOT_FOUND,
|
|
99
|
+
"data": OutInformation.RESOURCE_NOT_FOUND,
|
|
100
|
+
}
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
@staticmethod
|
|
104
|
+
def __names_section(name, lista_tags):
|
|
105
|
+
return {key: Tags.clean_text(Tags.catch_error(name, value)) for key, value in lista_tags.items()}
|
|
106
|
+
|
|
107
|
+
@staticmethod
|
|
108
|
+
def __data_section(data, lista_tags):
|
|
109
|
+
return {key: Tags.clean_text(Tags.catch_error(data, value)) for key, value in lista_tags.items()}
|
|
110
|
+
|
|
111
|
+
def __show_information(self, dictionary_basic, dictionary_sections):
|
|
112
|
+
dictionary = {
|
|
113
|
+
"name": dictionary_basic['name'],
|
|
114
|
+
"image": dictionary_basic['image'],
|
|
115
|
+
"description": dictionary_basic['descripcion'],
|
|
116
|
+
"names": dictionary_sections['nombres'],
|
|
117
|
+
"data": dictionary_sections['data'],
|
|
118
|
+
"url_scrap": dictionary_basic['url']
|
|
119
|
+
}
|
|
120
|
+
if self.all_data:
|
|
121
|
+
return dictionary
|
|
122
|
+
return OutInformation.clean_not_found_scrape(dictionary)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class OutInformation:
|
|
5
|
+
RESOURCE_NOT_FOUND = 'Not Found In Web'
|
|
6
|
+
|
|
7
|
+
@staticmethod
|
|
8
|
+
def beautiful_print_dict(json_dict: dict):
|
|
9
|
+
beautiful_print = json.dumps(json_dict, indent=4, ensure_ascii=False).encode('utf-8')
|
|
10
|
+
return beautiful_print.decode('utf-8')
|
|
11
|
+
|
|
12
|
+
@classmethod
|
|
13
|
+
def clean_not_found_scrape(cls, character: dict):
|
|
14
|
+
for key, value in character.copy().items():
|
|
15
|
+
if value == cls.RESOURCE_NOT_FOUND:
|
|
16
|
+
character.pop(key)
|
|
17
|
+
elif isinstance(value, dict):
|
|
18
|
+
cls.clean_not_found_scrape(value)
|
|
19
|
+
# si el dict interno quedó vacío tras la limpieza, elimínalo
|
|
20
|
+
if not character.get(key):
|
|
21
|
+
character.pop(key, None)
|
|
22
|
+
elif isinstance(value, list):
|
|
23
|
+
cleaned = [v for v in value if v != cls.RESOURCE_NOT_FOUND]
|
|
24
|
+
if cleaned:
|
|
25
|
+
character[key] = cleaned
|
|
26
|
+
else:
|
|
27
|
+
character.pop(key)
|
|
28
|
+
return character
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import requests as requests
|
|
2
|
+
from bs4 import BeautifulSoup
|
|
3
|
+
|
|
4
|
+
from DragonBall.Constants.ScrapPlanet import ScrapPlanet
|
|
5
|
+
from DragonBall.Constants.URLS import URL
|
|
6
|
+
from DragonBall.Data.OutInformation import OutInformation
|
|
7
|
+
from DragonBall.Soup.section import Section
|
|
8
|
+
from DragonBall.Soup.tags import Tags
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Planets:
|
|
12
|
+
def __init__(self, name: str, all_data=False):
|
|
13
|
+
self.name = name
|
|
14
|
+
self.all_data = all_data
|
|
15
|
+
|
|
16
|
+
@classmethod
|
|
17
|
+
def planet_list(cls, planet_list: list):
|
|
18
|
+
return [Planets(i).planet_information() for i in planet_list]
|
|
19
|
+
|
|
20
|
+
@staticmethod
|
|
21
|
+
def list():
|
|
22
|
+
request = requests.get(URL.URL_LIST_PLANETS, headers=URL.HEADERS)
|
|
23
|
+
soup = BeautifulSoup(request.content.decode('utf-8'), "html.parser")
|
|
24
|
+
ref = soup.find('div', {"class": "mw-parser-output"})
|
|
25
|
+
if not ref:
|
|
26
|
+
return []
|
|
27
|
+
list_planets = [planet.text for planet in ref.find_all('b')]
|
|
28
|
+
if list_planets:
|
|
29
|
+
list_planets.pop(0)
|
|
30
|
+
return list_planets
|
|
31
|
+
|
|
32
|
+
def planet_information(self):
|
|
33
|
+
url = URL.URL_WIKI + self.name.replace(" ", "_")
|
|
34
|
+
request = requests.get(url, headers=URL.HEADERS)
|
|
35
|
+
soup = BeautifulSoup(request.content.decode('utf-8'), "html.parser")
|
|
36
|
+
list_section = Section.obtain_section_data(soup, 'aside')
|
|
37
|
+
basic_dict = self.__obtain_basic_data(soup, url)
|
|
38
|
+
section_dict = self.__evaluate_scrapper_section(list_section, list(list_section.keys()), ScrapPlanet)
|
|
39
|
+
return self.__show_information(basic_dict, section_dict)
|
|
40
|
+
|
|
41
|
+
def __obtain_basic_data(self, soup: BeautifulSoup, url: str):
|
|
42
|
+
name = self.__get_official_name(soup, "h1", "id", "firstHeading")
|
|
43
|
+
image = self.__get_image(soup, "a", "class", "image image-thumbnail")
|
|
44
|
+
description = self.__get_description_planet(soup)
|
|
45
|
+
return {"url": url, "name": name, "image": image, "descripcion": description}
|
|
46
|
+
|
|
47
|
+
@staticmethod
|
|
48
|
+
def __get_official_name(soup: BeautifulSoup, tag: str, identification: str, element: str):
|
|
49
|
+
# Intento 1: Buscar el h1 con el id exacto
|
|
50
|
+
official_name = Tags.custom_catch_error(soup, tag, identification, element)
|
|
51
|
+
if official_name != OutInformation.RESOURCE_NOT_FOUND:
|
|
52
|
+
return official_name.get_text(strip=True)
|
|
53
|
+
|
|
54
|
+
# Intento 2: Buscar CUALQUIER etiqueta <h1> en la página
|
|
55
|
+
h1 = soup.find("h1")
|
|
56
|
+
if h1:
|
|
57
|
+
return h1.get_text(strip=True)
|
|
58
|
+
|
|
59
|
+
# Intento 3: Usar el título de la pestaña del navegador
|
|
60
|
+
if soup.title:
|
|
61
|
+
return soup.title.text.split("|")[0].strip()
|
|
62
|
+
|
|
63
|
+
return OutInformation.RESOURCE_NOT_FOUND
|
|
64
|
+
|
|
65
|
+
@staticmethod
|
|
66
|
+
def __get_image(soup: BeautifulSoup, tag: str, identification: str, name: str):
|
|
67
|
+
found = soup.find(tag, attrs={identification: name})
|
|
68
|
+
if found:
|
|
69
|
+
return found.get("href")
|
|
70
|
+
return OutInformation.RESOURCE_NOT_FOUND
|
|
71
|
+
|
|
72
|
+
@staticmethod
|
|
73
|
+
def __get_description_planet(soup: BeautifulSoup):
|
|
74
|
+
[remove_sup.extract() for remove_sup in soup.find_all('sup')]
|
|
75
|
+
try:
|
|
76
|
+
return next(
|
|
77
|
+
p.text.strip() for header in soup.find_all(['h2', 'h3']) if (p := header.find_next_sibling('p')))
|
|
78
|
+
except StopIteration:
|
|
79
|
+
return OutInformation.RESOURCE_NOT_FOUND
|
|
80
|
+
|
|
81
|
+
def __evaluate_scrapper_section(self, list_dict, list_search, tag_found: ScrapPlanet):
|
|
82
|
+
empty = self.__return_empty_section(list_dict)
|
|
83
|
+
if empty:
|
|
84
|
+
return empty
|
|
85
|
+
|
|
86
|
+
section_names = list(tag_found.sections.values())
|
|
87
|
+
names = list_dict.get(section_names[0], OutInformation.RESOURCE_NOT_FOUND)
|
|
88
|
+
data = list_dict.get(section_names[1], OutInformation.RESOURCE_NOT_FOUND)
|
|
89
|
+
|
|
90
|
+
return {
|
|
91
|
+
"nombres": self.__names_section(names, tag_found.names) if section_names[0] in list_search else names,
|
|
92
|
+
"data": self.__data_section(data, tag_found.data) if section_names[1] in list_search else data,
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
@staticmethod
|
|
96
|
+
def __return_empty_section(list_dict: dict):
|
|
97
|
+
if not list_dict:
|
|
98
|
+
return {
|
|
99
|
+
"nombres": OutInformation.RESOURCE_NOT_FOUND,
|
|
100
|
+
"data": OutInformation.RESOURCE_NOT_FOUND,
|
|
101
|
+
}
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
@staticmethod
|
|
105
|
+
def __names_section(name, lista_tags):
|
|
106
|
+
return {key: Tags.clean_text(Tags.catch_error(name, value)) for key, value in lista_tags.items()}
|
|
107
|
+
|
|
108
|
+
@staticmethod
|
|
109
|
+
def __data_section(data, lista_tags):
|
|
110
|
+
return {key: Tags.clean_text(Tags.catch_error(data, value)) for key, value in lista_tags.items()}
|
|
111
|
+
|
|
112
|
+
def __show_information(self, dictionary_basic, dictionary_sections):
|
|
113
|
+
dictionary = {
|
|
114
|
+
"name": dictionary_basic['name'],
|
|
115
|
+
"image": dictionary_basic['image'],
|
|
116
|
+
"description": dictionary_basic['descripcion'],
|
|
117
|
+
"names": dictionary_sections['nombres'],
|
|
118
|
+
"data": dictionary_sections['data'],
|
|
119
|
+
"url_scrap": dictionary_basic['url']
|
|
120
|
+
}
|
|
121
|
+
if self.all_data:
|
|
122
|
+
return dictionary
|
|
123
|
+
return OutInformation.clean_not_found_scrape(dictionary)
|