FranKGraphBench 0.0.1a0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- FranKGraphBench-0.0.1a0.dist-info/LICENSE +201 -0
- FranKGraphBench-0.0.1a0.dist-info/METADATA +189 -0
- FranKGraphBench-0.0.1a0.dist-info/RECORD +70 -0
- FranKGraphBench-0.0.1a0.dist-info/WHEEL +6 -0
- FranKGraphBench-0.0.1a0.dist-info/entry_points.txt +3 -0
- FranKGraphBench-0.0.1a0.dist-info/top_level.txt +2 -0
- data_integration/__init__.py +47 -0
- data_integration/__main__.py +4 -0
- data_integration/dataset.py +270 -0
- data_integration/dataset2class.py +34 -0
- data_integration/datasets/__init__.py +0 -0
- data_integration/datasets/amazon.py +140 -0
- data_integration/datasets/book-crossing.py +98 -0
- data_integration/datasets/douban-movie.py +75 -0
- data_integration/datasets/lastfm.py +215 -0
- data_integration/datasets/movielens.py +375 -0
- data_integration/datasets/steam.py +98 -0
- data_integration/datasets/worker.py +31 -0
- data_integration/datasets/yelp.py +156 -0
- framework/__init__.py +17 -0
- framework/__main__.py +4 -0
- framework/dataloader/__init__.py +0 -0
- framework/dataloader/dataloader.py +52 -0
- framework/dataloader/dataset/__init__.py +0 -0
- framework/dataloader/dataset/dataset.py +38 -0
- framework/dataloader/edge_splitter/__init__.py +0 -0
- framework/dataloader/edge_splitter/edge_splitter.py +163 -0
- framework/dataloader/graph/__init__.py +0 -0
- framework/dataloader/graph/graph.py +314 -0
- framework/dataloader/graph/node.py +80 -0
- framework/dataloader/preprocess/__init__.py +0 -0
- framework/dataloader/preprocess/methods.py +47 -0
- framework/evaluator/__init__.py +0 -0
- framework/evaluator/metric.py +39 -0
- framework/evaluator/metric2class.py +10 -0
- framework/evaluator/metrics/__init__.py +0 -0
- framework/evaluator/metrics/map.py +33 -0
- framework/evaluator/metrics/ndcg.py +49 -0
- framework/experiment.py +120 -0
- framework/recommender/__init__.py +0 -0
- framework/recommender/model2class.py +46 -0
- framework/recommender/models/__init__.py +0 -0
- framework/recommender/models/complEx/__init__.py +0 -0
- framework/recommender/models/complEx/model.py +193 -0
- framework/recommender/models/deep_walk_based/__init__.py +0 -0
- framework/recommender/models/deep_walk_based/model.py +149 -0
- framework/recommender/models/distMult/__init__.py +0 -0
- framework/recommender/models/distMult/model.py +187 -0
- framework/recommender/models/ePHEN/__init__.py +0 -0
- framework/recommender/models/ePHEN/model.py +159 -0
- framework/recommender/models/rESCAL/__init__.py +0 -0
- framework/recommender/models/rESCAL/model.py +184 -0
- framework/recommender/models/rotatE/__init__.py +0 -0
- framework/recommender/models/rotatE/model.py +195 -0
- framework/recommender/models/transD/__init__.py +0 -0
- framework/recommender/models/transD/model.py +190 -0
- framework/recommender/models/transE/__init__.py +0 -0
- framework/recommender/models/transE/model.py +194 -0
- framework/recommender/models/transH/__init__.py +0 -0
- framework/recommender/models/transH/model.py +193 -0
- framework/recommender/models/transR/__init__.py +0 -0
- framework/recommender/models/transR/model.py +190 -0
- framework/recommender/models/tuckER/__init__.py +0 -0
- framework/recommender/models/tuckER/model.py +199 -0
- framework/recommender/recommender.py +32 -0
- framework/recommender/utils/__init__.py +0 -0
- framework/recommender/utils/walker.py +185 -0
- framework/reporter/__init__.py +0 -0
- framework/reporter/report.py +101 -0
- framework/utils.py +10 -0
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import queue
|
|
4
|
+
from string import Template
|
|
5
|
+
from ..dataset import Dataset
|
|
6
|
+
|
|
7
|
+
from io import BytesIO
|
|
8
|
+
from collections import defaultdict
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
import pandas as pd
|
|
12
|
+
from tqdm import tqdm
|
|
13
|
+
from thefuzz import process
|
|
14
|
+
from SPARQLWrapper import CSV
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class LastFM(Dataset):
|
|
18
|
+
def __init__(self, input_path, output_path, n_workers=1):
|
|
19
|
+
super().__init__(input_path, output_path, n_workers)
|
|
20
|
+
self.dataset_name = "LastFM"
|
|
21
|
+
|
|
22
|
+
self.item_separator = "\t"
|
|
23
|
+
# self.user_separator = '|'
|
|
24
|
+
self.rating_separator = "\t"
|
|
25
|
+
self.social_separator = "\t"
|
|
26
|
+
|
|
27
|
+
self.item_fields = {"id": "item_id::string", "name": "name::string"}
|
|
28
|
+
self.user_fields = {"userID": "user_id::string"}
|
|
29
|
+
self.rating_fields = {
|
|
30
|
+
"userID": "user_id::string",
|
|
31
|
+
"artistID": "item_id::string",
|
|
32
|
+
"weight": "rating::number",
|
|
33
|
+
}
|
|
34
|
+
self.map_fields = {"item_id": "item_id::string", "URI": "URI::string"}
|
|
35
|
+
self.social_fields = {"userID": "user1::string", "friendID": "user2::string"}
|
|
36
|
+
|
|
37
|
+
self.map_query_template = Template(
|
|
38
|
+
"""
|
|
39
|
+
PREFIX dct: <http://purl.org/dc/terms/>
|
|
40
|
+
PREFIX dbo: <http://dbpedia.org/ontology/>
|
|
41
|
+
PREFIX dbr: <http://dbpedia.org/resource/>
|
|
42
|
+
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
|
43
|
+
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
|
44
|
+
SELECT DISTINCT ?artist WHERE {
|
|
45
|
+
{
|
|
46
|
+
?artist rdf:type dbo:MusicalArtist .
|
|
47
|
+
?artist rdfs:label ?label .
|
|
48
|
+
FILTER regex(?label, "$name_regex", "i")
|
|
49
|
+
}
|
|
50
|
+
UNION
|
|
51
|
+
{
|
|
52
|
+
?artist rdf:type dbo:MusicalArtist .
|
|
53
|
+
?tmp dbo:wikiPageRedirects ?artist .
|
|
54
|
+
?tmp rdfs:label ?label .
|
|
55
|
+
FILTER regex(?label, "$name_regex", "i") .
|
|
56
|
+
}
|
|
57
|
+
UNION
|
|
58
|
+
{
|
|
59
|
+
?artist rdf:type dbo:Band .
|
|
60
|
+
?artist rdfs:label ?label .
|
|
61
|
+
FILTER regex(?label, "$name_regex", "i")
|
|
62
|
+
}
|
|
63
|
+
UNION
|
|
64
|
+
{
|
|
65
|
+
?artist rdf:type dbo:Band .
|
|
66
|
+
?tmp dbo:wikiPageRedirects ?artist .
|
|
67
|
+
?tmp rdfs:label ?label .
|
|
68
|
+
FILTER regex(?label, "$name_regex", "i") .
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
"""
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
self.enrich_query_template = Template(
|
|
75
|
+
"""
|
|
76
|
+
PREFIX dct: <http://purl.org/dc/terms/>
|
|
77
|
+
PREFIX dbo: <http://dbpedia.org/ontology/>
|
|
78
|
+
PREFIX dbr: <http://dbpedia.org/resource/>
|
|
79
|
+
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
|
80
|
+
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
|
81
|
+
SELECT DISTINCT
|
|
82
|
+
?abstract
|
|
83
|
+
(GROUP_CONCAT(DISTINCT ?bandMember; SEPARATOR="::") AS ?bandMember)
|
|
84
|
+
(GROUP_CONCAT(DISTINCT ?genre; SEPARATOR="::") AS ?genre)
|
|
85
|
+
(GROUP_CONCAT(DISTINCT ?associatedMusicalArtist; SEPARATOR="::") AS ?associatedMusicalArtist)
|
|
86
|
+
(GROUP_CONCAT(DISTINCT ?awards; SEPARATOR="::") AS ?awards)
|
|
87
|
+
(GROUP_CONCAT(DISTINCT ?recordLabel; SEPARATOR="::") AS ?recordLabel)
|
|
88
|
+
(GROUP_CONCAT(DISTINCT ?associatedBand; SEPARATOR="::") AS ?associatedBand)
|
|
89
|
+
(GROUP_CONCAT(DISTINCT ?origin; SEPARATOR="::") AS ?origin)
|
|
90
|
+
WHERE {
|
|
91
|
+
OPTIONAL { <$URI> dbo:genre ?genre } .
|
|
92
|
+
OPTIONAL { <$URI> dbo:abstract ?abstract } .
|
|
93
|
+
OPTIONAL { <$URI> dbp:origin ?origin } .
|
|
94
|
+
OPTIONAL { <$URI> dbo:recordLabel ?recordLabel } .
|
|
95
|
+
OPTIONAL { <$URI> dbo:bandMember ?bandMember } .
|
|
96
|
+
OPTIONAL { <$URI> dbo:associatedMusicalArtist ?associatedMusicalArtist } .
|
|
97
|
+
OPTIONAL { <$URI> dbo:associatedBand ?associatedBand } .
|
|
98
|
+
OPTIONAL { <$URI> dbp:awards ?awards } .
|
|
99
|
+
|
|
100
|
+
FILTER(LANG(?abstract) = 'en')
|
|
101
|
+
}
|
|
102
|
+
"""
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
def load_item_data(self) -> pd.DataFrame():
|
|
106
|
+
filename = os.path.join(self.input_path, "artists.dat")
|
|
107
|
+
|
|
108
|
+
df = pd.read_csv(filename, sep=self.item_separator)
|
|
109
|
+
df = df.rename(self.item_fields, axis=1)
|
|
110
|
+
df = df[self.item_fields.values()]
|
|
111
|
+
return df
|
|
112
|
+
|
|
113
|
+
def entity_linking(self, df_item) -> pd.DataFrame():
|
|
114
|
+
q = queue.Queue()
|
|
115
|
+
for idx, row in df_item[["name", "item_id"]].iterrows():
|
|
116
|
+
query = self.get_map_query(row["name"])
|
|
117
|
+
q.put((row["item_id"], query))
|
|
118
|
+
|
|
119
|
+
if self.n_workers > 1:
|
|
120
|
+
responses = self.parallel_queries(q)
|
|
121
|
+
else:
|
|
122
|
+
responses = self.sequential_queries(q)
|
|
123
|
+
|
|
124
|
+
URI_mapping = {}
|
|
125
|
+
for response in tqdm(responses, desc="Disambiguating query return"):
|
|
126
|
+
candidate_URIs = []
|
|
127
|
+
idx, result = response
|
|
128
|
+
for binding in result["results"]["bindings"]:
|
|
129
|
+
URI = binding["artist"]["value"]
|
|
130
|
+
candidate_URIs.append(URI)
|
|
131
|
+
|
|
132
|
+
expected_URI = f'http://dbpedia.org/resource/{df_item.loc[df_item.item_id == idx]["name"]}'
|
|
133
|
+
str_matching_result = process.extractOne(expected_URI, candidate_URIs)
|
|
134
|
+
|
|
135
|
+
if str_matching_result is not None:
|
|
136
|
+
URI, _ = str_matching_result
|
|
137
|
+
URI_mapping[idx] = URI
|
|
138
|
+
|
|
139
|
+
df_map = pd.DataFrame({"item_id": df_item["item_id"]})
|
|
140
|
+
df_map.set_index("item_id")
|
|
141
|
+
df_map["URI"] = df_map["item_id"].apply(lambda id: URI_mapping.get(id))
|
|
142
|
+
df_map = df_map.rename(self.map_fields, axis=1)
|
|
143
|
+
|
|
144
|
+
return df_map
|
|
145
|
+
|
|
146
|
+
def get_map_query(self, name) -> str:
|
|
147
|
+
name = name.translate(self._special_chars_map)
|
|
148
|
+
name = name.replace(" ", ".*")
|
|
149
|
+
name = "^" + name
|
|
150
|
+
name = name + "$"
|
|
151
|
+
|
|
152
|
+
params = {"name_regex": name}
|
|
153
|
+
query = self.map_query_template.substitute(**params)
|
|
154
|
+
return query
|
|
155
|
+
|
|
156
|
+
def enrich(self, df_map):
|
|
157
|
+
df_map = df_map[df_map["URI"].notna()]
|
|
158
|
+
|
|
159
|
+
q = queue.Queue()
|
|
160
|
+
for _, row in df_map[["URI", "item_id"]].iterrows():
|
|
161
|
+
query = self.get_enrich_query(row["URI"])
|
|
162
|
+
q.put((row["item_id"], query))
|
|
163
|
+
|
|
164
|
+
if self.n_workers > 1:
|
|
165
|
+
responses = self.parallel_queries(q, CSV)
|
|
166
|
+
else:
|
|
167
|
+
responses = self.sequential_queries(q, CSV)
|
|
168
|
+
|
|
169
|
+
item_enriching = defaultdict(dict)
|
|
170
|
+
for response in responses:
|
|
171
|
+
idx, result = response
|
|
172
|
+
df = pd.read_csv(BytesIO(result))
|
|
173
|
+
|
|
174
|
+
if df.shape[0] > 1:
|
|
175
|
+
print("At least one property has more than one value!")
|
|
176
|
+
print(df.value_counts(dropna=False))
|
|
177
|
+
|
|
178
|
+
item_enriching[idx] = df.iloc[0] # getting pd.Series
|
|
179
|
+
|
|
180
|
+
df_enrich = pd.DataFrame.from_dict(item_enriching, orient="index")
|
|
181
|
+
df_enrich.index.name = "item_id"
|
|
182
|
+
|
|
183
|
+
return df_enrich
|
|
184
|
+
|
|
185
|
+
def get_enrich_query(self, URI) -> str:
|
|
186
|
+
params = {"URI": URI}
|
|
187
|
+
query = self.enrich_query_template.substitute(**params)
|
|
188
|
+
return query
|
|
189
|
+
|
|
190
|
+
def load_rating_data(self) -> pd.DataFrame():
|
|
191
|
+
filename = os.path.join(self.input_path, "user_artists.dat")
|
|
192
|
+
df = pd.read_csv(filename, sep=self.rating_separator)
|
|
193
|
+
df = df.rename(self.rating_fields, axis=1)
|
|
194
|
+
df = df[self.rating_fields.values()]
|
|
195
|
+
|
|
196
|
+
return df
|
|
197
|
+
|
|
198
|
+
def load_social_data(self) -> pd.DataFrame():
|
|
199
|
+
filename = os.path.join(self.input_path, "user_friends.dat")
|
|
200
|
+
df = pd.read_csv(filename, sep=self.social_separator)
|
|
201
|
+
df = df.rename(self.social_fields, axis=1)
|
|
202
|
+
df = df[self.social_fields.values()]
|
|
203
|
+
|
|
204
|
+
return df
|
|
205
|
+
|
|
206
|
+
def load_user_data(self) -> pd.DataFrame():
|
|
207
|
+
# There is no .dat only for user (none aditional user info)
|
|
208
|
+
# Will extract the users from the rating data
|
|
209
|
+
filename = os.path.join(self.input_path, "user_artists.dat")
|
|
210
|
+
df = pd.read_csv(filename, sep=self.rating_separator)
|
|
211
|
+
users = df["userID"].unique()
|
|
212
|
+
df = df.rename(self.user_fields)
|
|
213
|
+
df = pd.DataFrame(pd.Series(users), columns=self.user_fields.values())
|
|
214
|
+
|
|
215
|
+
return df
|
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import queue
|
|
4
|
+
from io import BytesIO
|
|
5
|
+
from collections import defaultdict
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from string import Template
|
|
9
|
+
from ..dataset import Dataset
|
|
10
|
+
|
|
11
|
+
import pandas as pd
|
|
12
|
+
from tqdm import tqdm
|
|
13
|
+
from thefuzz import process
|
|
14
|
+
from SPARQLWrapper import CSV
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class MovieLens(Dataset):
|
|
18
|
+
"""
|
|
19
|
+
General MovieLens(Dataset) class for data integration between DBpedia and MovieLens
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, input_path, output_path, n_workers=1):
|
|
23
|
+
super().__init__(input_path, output_path, n_workers)
|
|
24
|
+
self.map_query_template = Template(
|
|
25
|
+
"""
|
|
26
|
+
PREFIX dct: <http://purl.org/dc/terms/>
|
|
27
|
+
PREFIX dbo: <http://dbpedia.org/ontology/>
|
|
28
|
+
PREFIX dbr: <http://dbpedia.org/resource/>
|
|
29
|
+
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
|
30
|
+
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
|
31
|
+
SELECT DISTINCT ?film WHERE {
|
|
32
|
+
{
|
|
33
|
+
?film rdf:type dbo:Film .
|
|
34
|
+
?film dct:subject $year_category .
|
|
35
|
+
?film rdfs:label ?label .
|
|
36
|
+
FILTER regex(?label, "$name_regex", "i")
|
|
37
|
+
}
|
|
38
|
+
UNION
|
|
39
|
+
{
|
|
40
|
+
?film rdf:type dbo:Film .
|
|
41
|
+
?film dct:subject $year_category .
|
|
42
|
+
?tmp dbo:wikiPageRedirects ?film .
|
|
43
|
+
?tmp rdfs:label ?label .
|
|
44
|
+
FILTER regex(?label, "$name_regex", "i") .
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
"""
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
self.enrich_query_template = Template(
|
|
51
|
+
"""
|
|
52
|
+
PREFIX dct: <http://purl.org/dc/terms/>
|
|
53
|
+
PREFIX dbo: <http://dbpedia.org/ontology/>
|
|
54
|
+
PREFIX dbr: <http://dbpedia.org/resource/>
|
|
55
|
+
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
|
56
|
+
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
|
57
|
+
SELECT DISTINCT
|
|
58
|
+
?abstract
|
|
59
|
+
(GROUP_CONCAT(DISTINCT ?producer; SEPARATOR="::") AS ?producer)
|
|
60
|
+
(GROUP_CONCAT(DISTINCT ?distributor; SEPARATOR="::") AS ?distributor)
|
|
61
|
+
(GROUP_CONCAT(DISTINCT ?writer; SEPARATOR="::") AS ?writer)
|
|
62
|
+
(GROUP_CONCAT(DISTINCT ?cinematography; SEPARATOR="::") AS ?cinematography)
|
|
63
|
+
(GROUP_CONCAT(DISTINCT ?subject; SEPARATOR="::") AS ?subject)
|
|
64
|
+
(GROUP_CONCAT(DISTINCT ?starring; SEPARATOR="::") AS ?starring)
|
|
65
|
+
(GROUP_CONCAT(DISTINCT ?director; SEPARATOR="::") AS ?director)
|
|
66
|
+
WHERE {
|
|
67
|
+
OPTIONAL { <$URI> dct:subject ?subject } .
|
|
68
|
+
OPTIONAL { <$URI> dbo:starring ?starring } .
|
|
69
|
+
OPTIONAL { <$URI> dbo:director ?director } .
|
|
70
|
+
OPTIONAL { <$URI> dbo:abstract ?abstract } .
|
|
71
|
+
OPTIONAL { <$URI> dbo:producer ?producer } .
|
|
72
|
+
OPTIONAL { <$URI> dbo:distributor ?distributor } .
|
|
73
|
+
OPTIONAL { <$URI> dbo:writer ?writer } .
|
|
74
|
+
OPTIONAL { <$URI> dbo:cinematography ?cinematography } .
|
|
75
|
+
|
|
76
|
+
FILTER(LANG(?abstract) = 'en')
|
|
77
|
+
}
|
|
78
|
+
"""
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
def _extract_title(self, title):
|
|
82
|
+
regex = re.compile(r"\((\d{4})\)")
|
|
83
|
+
rgx_match = regex.search(title)
|
|
84
|
+
year_start, _ = rgx_match.span()
|
|
85
|
+
movie_title = title[: year_start - 1].strip()
|
|
86
|
+
|
|
87
|
+
# removing parenthesis (name of the film in other languages)
|
|
88
|
+
movie_title = [s.split(")")[-1] for s in movie_title.split("(")][0]
|
|
89
|
+
|
|
90
|
+
# Reversing titles with X, The
|
|
91
|
+
movie_title = movie_title.split(",")
|
|
92
|
+
movie_title = [movie_title[-1].strip()] + [
|
|
93
|
+
title.strip() for title in movie_title[:-1]
|
|
94
|
+
]
|
|
95
|
+
movie_title = " ".join(movie_title)
|
|
96
|
+
|
|
97
|
+
movie_title = movie_title.strip()
|
|
98
|
+
return movie_title
|
|
99
|
+
|
|
100
|
+
def _extract_year(self, title):
|
|
101
|
+
regex = re.compile(r"\((\d{4})\)")
|
|
102
|
+
rgx_match = regex.search(title)
|
|
103
|
+
movie_year = rgx_match.group()
|
|
104
|
+
movie_year = int(movie_year[1:-1])
|
|
105
|
+
return movie_year
|
|
106
|
+
|
|
107
|
+
def entity_linking(self, df_item) -> pd.DataFrame():
|
|
108
|
+
q = queue.Queue()
|
|
109
|
+
for idx, row in df_item[["movie_title", "movie_year", "item_id"]].iterrows():
|
|
110
|
+
query = self.get_map_query(row["movie_title"], row["movie_year"])
|
|
111
|
+
q.put((row["item_id"], query))
|
|
112
|
+
|
|
113
|
+
if self.n_workers > 1:
|
|
114
|
+
responses = self.parallel_queries(q)
|
|
115
|
+
else:
|
|
116
|
+
responses = self.sequential_queries(q)
|
|
117
|
+
|
|
118
|
+
URI_mapping = defaultdict(dict)
|
|
119
|
+
for response in tqdm(responses, desc="Disambiguating query return"):
|
|
120
|
+
candidate_URIs = []
|
|
121
|
+
idx, result = response
|
|
122
|
+
for binding in result["results"]["bindings"]:
|
|
123
|
+
URI = binding["film"]["value"]
|
|
124
|
+
candidate_URIs.append(URI)
|
|
125
|
+
|
|
126
|
+
expected_URI = f"http://dbpedia.org/resource/{df_item.loc[df_item.item_id == idx]['movie_title']}"
|
|
127
|
+
str_matching_result = process.extractOne(expected_URI, candidate_URIs)
|
|
128
|
+
|
|
129
|
+
if str_matching_result is not None:
|
|
130
|
+
URI, _ = str_matching_result
|
|
131
|
+
URI_mapping[idx] = URI
|
|
132
|
+
|
|
133
|
+
df_map = pd.DataFrame({"item_id": df_item["item_id"]})
|
|
134
|
+
df_map.set_index("item_id")
|
|
135
|
+
df_map["URI"] = df_map["item_id"].apply(lambda id: URI_mapping.get(id))
|
|
136
|
+
df_map = df_map.rename(self.map_fields, axis=1)
|
|
137
|
+
|
|
138
|
+
return df_map
|
|
139
|
+
|
|
140
|
+
def get_map_query(self, title, year) -> str:
|
|
141
|
+
# title = title.translate(self._special_chars_map)
|
|
142
|
+
title = title.replace(" ", ".*")
|
|
143
|
+
title = "^" + title
|
|
144
|
+
|
|
145
|
+
params = {
|
|
146
|
+
"name_regex": title,
|
|
147
|
+
"year_category": f"dbr:Category:{str(year)}_films",
|
|
148
|
+
}
|
|
149
|
+
query = self.map_query_template.substitute(**params)
|
|
150
|
+
return query
|
|
151
|
+
|
|
152
|
+
def enrich(self, df_map) -> pd.DataFrame():
|
|
153
|
+
df_map = df_map[df_map["URI"].notna()]
|
|
154
|
+
|
|
155
|
+
q = queue.Queue()
|
|
156
|
+
for _, row in df_map[["URI", "item_id"]].iterrows():
|
|
157
|
+
query = self.get_enrich_query(row["URI"])
|
|
158
|
+
q.put((row["item_id"], query))
|
|
159
|
+
|
|
160
|
+
if self.n_workers > 1:
|
|
161
|
+
responses = self.parallel_queries(q, CSV)
|
|
162
|
+
else:
|
|
163
|
+
responses = self.sequential_queries(q, CSV)
|
|
164
|
+
|
|
165
|
+
item_enriching = defaultdict(dict)
|
|
166
|
+
for response in responses:
|
|
167
|
+
idx, result = response
|
|
168
|
+
df = pd.read_csv(BytesIO(result))
|
|
169
|
+
|
|
170
|
+
if df.shape[0] > 1:
|
|
171
|
+
print("At least one property has more than one value!")
|
|
172
|
+
print(df.value_counts(dropna=False))
|
|
173
|
+
|
|
174
|
+
item_enriching[idx] = df.iloc[0] # getting pd.Series
|
|
175
|
+
|
|
176
|
+
df_enrich = pd.DataFrame.from_dict(item_enriching, orient="index")
|
|
177
|
+
df_enrich.index.name = "item_id"
|
|
178
|
+
|
|
179
|
+
return df_enrich
|
|
180
|
+
|
|
181
|
+
def get_enrich_query(self, URI) -> str:
|
|
182
|
+
params = {"URI": URI}
|
|
183
|
+
query = self.enrich_query_template.substitute(**params)
|
|
184
|
+
return query
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class MovieLens100k(MovieLens):
|
|
188
|
+
"""
|
|
189
|
+
Dataset class for data integration between DBpedia and MovieLens-100k
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
def __init__(self, input_path, output_path, n_workers=1):
|
|
193
|
+
super().__init__(input_path, output_path, n_workers)
|
|
194
|
+
self.dataset_name = "MovieLens100k"
|
|
195
|
+
|
|
196
|
+
self.item_separator = "|"
|
|
197
|
+
self.user_separator = "|"
|
|
198
|
+
self.rating_separator = "\t"
|
|
199
|
+
self.item_fields = {
|
|
200
|
+
"movie id": "item_id::string",
|
|
201
|
+
"movie_title": "movie_title::string",
|
|
202
|
+
"movie_year": "movie_year::string", # extracted from movie_title
|
|
203
|
+
}
|
|
204
|
+
self.map_fields = {"item_id": "item_id::string", "URI": "URI::string"}
|
|
205
|
+
self.rating_fields = {
|
|
206
|
+
"user id": "user_id::string",
|
|
207
|
+
"item id": "item_id::string",
|
|
208
|
+
"rating": "rating::number",
|
|
209
|
+
"timestamp": "timestamp::number",
|
|
210
|
+
}
|
|
211
|
+
self.user_fields = {
|
|
212
|
+
"user id": "user_id::string",
|
|
213
|
+
"age": "age::string",
|
|
214
|
+
"gender": "gender::string",
|
|
215
|
+
"occupation": "occupation::string",
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
def load_item_data(self) -> pd.DataFrame():
|
|
219
|
+
# Reading item file of MovieLens100k dataset
|
|
220
|
+
filename = os.path.join(self.input_path, "u.item")
|
|
221
|
+
column_names = """movie id | movie title | release date | video release date |
|
|
222
|
+
IMDb URL | unknown | Action | Adventure | Animation |
|
|
223
|
+
Children's | Comedy | Crime | Documentary | Drama | Fantasy |
|
|
224
|
+
Film-Noir | Horror | Musical | Mystery | Romance | Sci-Fi |
|
|
225
|
+
Thriller | War | Western"""
|
|
226
|
+
column_names = [name.strip() for name in column_names.split("|")]
|
|
227
|
+
df = pd.read_csv(
|
|
228
|
+
filename,
|
|
229
|
+
header=None,
|
|
230
|
+
sep=self.item_separator,
|
|
231
|
+
encoding="latin-1",
|
|
232
|
+
names=column_names,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
# Pre processing item
|
|
236
|
+
# 1- Dropping unused fields
|
|
237
|
+
df.set_index("movie id")
|
|
238
|
+
df = df.drop(["video release date", "unknown", "IMDb URL"], axis=1)
|
|
239
|
+
df = df.dropna()
|
|
240
|
+
|
|
241
|
+
# 2- Extracting title and year
|
|
242
|
+
df["movie_title"] = df["movie title"].apply(self._extract_title)
|
|
243
|
+
df["movie_year"] = df["movie title"].apply(self._extract_year)
|
|
244
|
+
|
|
245
|
+
df = df.rename(self.item_fields, axis=1)
|
|
246
|
+
df = df[self.item_fields.values()]
|
|
247
|
+
|
|
248
|
+
return df
|
|
249
|
+
|
|
250
|
+
def load_user_data(self) -> pd.DataFrame():
|
|
251
|
+
filename = os.path.join(self.input_path, "u.user")
|
|
252
|
+
column_names = "user id | age | gender | occupation | zip code"
|
|
253
|
+
column_names = [name.strip() for name in column_names.split("|")]
|
|
254
|
+
|
|
255
|
+
df = pd.read_csv(
|
|
256
|
+
filename,
|
|
257
|
+
header=None,
|
|
258
|
+
sep=self.user_separator,
|
|
259
|
+
encoding="latin-1",
|
|
260
|
+
names=column_names,
|
|
261
|
+
)
|
|
262
|
+
df = df.rename(self.user_fields, axis=1)
|
|
263
|
+
df = df[self.user_fields.values()]
|
|
264
|
+
|
|
265
|
+
return df
|
|
266
|
+
|
|
267
|
+
def load_rating_data(self) -> pd.DataFrame():
|
|
268
|
+
filename = os.path.join(self.input_path, "u.data")
|
|
269
|
+
column_names = "user id | item id | rating | timestamp"
|
|
270
|
+
column_names = [name.strip() for name in column_names.split("|")]
|
|
271
|
+
|
|
272
|
+
df = pd.read_csv(
|
|
273
|
+
filename,
|
|
274
|
+
header=None,
|
|
275
|
+
sep=self.rating_separator,
|
|
276
|
+
encoding="latin-1",
|
|
277
|
+
names=column_names,
|
|
278
|
+
)
|
|
279
|
+
df = df.rename(self.rating_fields, axis=1)
|
|
280
|
+
df = df[self.rating_fields.values()]
|
|
281
|
+
|
|
282
|
+
return df
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
class MovieLens1M(MovieLens):
|
|
286
|
+
"""
|
|
287
|
+
Dataset class for data integration between DBpedia and MovieLens 1M
|
|
288
|
+
"""
|
|
289
|
+
|
|
290
|
+
def __init__(self, input_path, output_path, n_workers=1):
|
|
291
|
+
super().__init__(input_path, output_path, n_workers)
|
|
292
|
+
self.dataset_name = "MovieLens1M"
|
|
293
|
+
|
|
294
|
+
self.item_separator = "::"
|
|
295
|
+
self.user_separator = "::"
|
|
296
|
+
self.rating_separator = "::"
|
|
297
|
+
self.item_fields = {
|
|
298
|
+
"MovieID": "item_id::string",
|
|
299
|
+
"movie_title": "movie_title::string",
|
|
300
|
+
"movie_year": "movie_year::string", # extracted from movie_title
|
|
301
|
+
}
|
|
302
|
+
self.map_fields = {"item_id": "item_id::string", "URI": "URI::string"}
|
|
303
|
+
self.rating_fields = {
|
|
304
|
+
"UserID": "user_id::string",
|
|
305
|
+
"MovieID": "item_id::string",
|
|
306
|
+
"Rating": "rating::number",
|
|
307
|
+
"Timestamp": "timestamp::number",
|
|
308
|
+
}
|
|
309
|
+
self.user_fields = {
|
|
310
|
+
"UserID": "user_id::string",
|
|
311
|
+
"Gender": "gender::string",
|
|
312
|
+
"Age": "age::string",
|
|
313
|
+
"Occupation": "occupation::string",
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
def load_item_data(self) -> pd.DataFrame():
|
|
317
|
+
# Reading item file of MovieLens100k dataset
|
|
318
|
+
filename = os.path.join(self.input_path, "movies.dat")
|
|
319
|
+
column_names = "MovieID::Title::Genres"
|
|
320
|
+
column_names = [name.strip() for name in column_names.split("::")]
|
|
321
|
+
df = pd.read_csv(
|
|
322
|
+
filename,
|
|
323
|
+
header=None,
|
|
324
|
+
sep=self.item_separator,
|
|
325
|
+
encoding="latin-1",
|
|
326
|
+
names=column_names,
|
|
327
|
+
engine="python",
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
# Pre processing item
|
|
331
|
+
# Extracting title and year
|
|
332
|
+
df["movie_title"] = df["Title"].apply(self._extract_title)
|
|
333
|
+
df["movie_year"] = df["Title"].apply(self._extract_year)
|
|
334
|
+
|
|
335
|
+
df = df.rename(self.item_fields, axis=1)
|
|
336
|
+
df = df[self.item_fields.values()]
|
|
337
|
+
|
|
338
|
+
return df
|
|
339
|
+
|
|
340
|
+
def load_user_data(self) -> pd.DataFrame():
|
|
341
|
+
filename = os.path.join(self.input_path, "users.dat")
|
|
342
|
+
column_names = "UserID::Gender::Age::Occupation::Zip-code"
|
|
343
|
+
column_names = [name.strip() for name in column_names.split("::")]
|
|
344
|
+
|
|
345
|
+
df = pd.read_csv(
|
|
346
|
+
filename,
|
|
347
|
+
header=None,
|
|
348
|
+
sep=self.user_separator,
|
|
349
|
+
encoding="latin-1",
|
|
350
|
+
names=column_names,
|
|
351
|
+
engine="python",
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
df = df.rename(self.user_fields, axis=1)
|
|
355
|
+
df = df[self.user_fields.values()]
|
|
356
|
+
|
|
357
|
+
return df
|
|
358
|
+
|
|
359
|
+
def load_rating_data(self) -> pd.DataFrame():
|
|
360
|
+
filename = os.path.join(self.input_path, "ratings.dat")
|
|
361
|
+
column_names = "UserID::MovieID::Rating::Timestamp"
|
|
362
|
+
column_names = [name.strip() for name in column_names.split("::")]
|
|
363
|
+
|
|
364
|
+
df = pd.read_csv(
|
|
365
|
+
filename,
|
|
366
|
+
header=None,
|
|
367
|
+
sep=self.rating_separator,
|
|
368
|
+
encoding="latin-1",
|
|
369
|
+
names=column_names,
|
|
370
|
+
engine="python",
|
|
371
|
+
)
|
|
372
|
+
df = df.rename(self.rating_fields, axis=1)
|
|
373
|
+
df = df[self.rating_fields.values()]
|
|
374
|
+
|
|
375
|
+
return df
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import queue
|
|
4
|
+
from string import Template
|
|
5
|
+
from ..dataset import Dataset
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
from tqdm import tqdm
|
|
9
|
+
from thefuzz import process
|
|
10
|
+
|
|
11
|
+
class Steam(Dataset):
|
|
12
|
+
def __init__(self, input_path, output_path, n_workers=1):
|
|
13
|
+
super().__init__(input_path, output_path, n_workers)
|
|
14
|
+
self.dataset_name = 'Steam'
|
|
15
|
+
|
|
16
|
+
self.item_separator = ','
|
|
17
|
+
# self.user_separator = '|'
|
|
18
|
+
# self.rating_separator = '\t'
|
|
19
|
+
|
|
20
|
+
self.item_fields = ['item_id', 'title', 'date_release']
|
|
21
|
+
# self.user_fields = ['user_id', 'age', 'gender', 'occupation']
|
|
22
|
+
# self.rating_fields = ['user_id', 'item_id', 'rating']
|
|
23
|
+
# self.map_fields = ['item_id', 'URI']
|
|
24
|
+
|
|
25
|
+
self.map_query_template = Template('''
|
|
26
|
+
PREFIX dct: <http://purl.org/dc/terms/>
|
|
27
|
+
PREFIX dbo: <http://dbpedia.org/ontology/>
|
|
28
|
+
PREFIX dbr: <http://dbpedia.org/resource/>
|
|
29
|
+
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
|
30
|
+
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
|
31
|
+
SELECT DISTINCT ?game WHERE {
|
|
32
|
+
{
|
|
33
|
+
?game rdf:type dbo:VideoGame .
|
|
34
|
+
?game rdfs:label ?label .
|
|
35
|
+
FILTER regex(?label, "$name_regex", "i")
|
|
36
|
+
}
|
|
37
|
+
UNION
|
|
38
|
+
{
|
|
39
|
+
?game rdf:type dbo:VideoGame .
|
|
40
|
+
?tmp dbo:wikiPageRedirects ?game .
|
|
41
|
+
?tmp rdfs:label ?label .
|
|
42
|
+
FILTER regex(?label, "$name_regex", "i") .
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
''')
|
|
46
|
+
|
|
47
|
+
def load_item_data(self) -> pd.DataFrame():
|
|
48
|
+
filename = os.path.join(self.input_path, 'steam_games.csv')
|
|
49
|
+
used_columns = ['app_id', 'title', 'date_release']
|
|
50
|
+
|
|
51
|
+
df = pd.read_csv(filename, sep=self.item_separator)
|
|
52
|
+
df = df[used_columns]
|
|
53
|
+
df.columns = self.item_fields
|
|
54
|
+
|
|
55
|
+
return df
|
|
56
|
+
|
|
57
|
+
def entity_linking(self, df_item) -> pd.DataFrame():
|
|
58
|
+
q = queue.Queue()
|
|
59
|
+
for idx, row in df_item[['title', 'item_id']].iterrows():
|
|
60
|
+
query = self.get_map_query(row['title'])
|
|
61
|
+
q.put((row['item_id'], query))
|
|
62
|
+
|
|
63
|
+
if self.n_workers > 1:
|
|
64
|
+
responses = self.parallel_queries(q)
|
|
65
|
+
else:
|
|
66
|
+
responses = self.sequential_queries(q)
|
|
67
|
+
|
|
68
|
+
URI_mapping = {}
|
|
69
|
+
for response in tqdm(responses, desc='Disambiguating query return'):
|
|
70
|
+
candidate_URIs = []
|
|
71
|
+
idx, result = response
|
|
72
|
+
for binding in result['results']['bindings']:
|
|
73
|
+
URI = binding['game']['value']
|
|
74
|
+
candidate_URIs.append(URI)
|
|
75
|
+
|
|
76
|
+
expected_URI = f'http://dbpedia.org/resource/{df_item.loc[df_item.item_id == idx]["title"]}'
|
|
77
|
+
str_matching_result = process.extractOne(expected_URI, candidate_URIs)
|
|
78
|
+
|
|
79
|
+
if str_matching_result is not None:
|
|
80
|
+
URI, _ = str_matching_result
|
|
81
|
+
URI_mapping[idx] = URI
|
|
82
|
+
|
|
83
|
+
df_map = pd.DataFrame({'item_id': df_item['item_id']})
|
|
84
|
+
df_map.set_index('item_id')
|
|
85
|
+
df_map['URI'] = df_map['item_id'].apply(lambda id: URI_mapping.get(id))
|
|
86
|
+
|
|
87
|
+
return df_map
|
|
88
|
+
|
|
89
|
+
def get_map_query(self, title) -> str:
|
|
90
|
+
title = title.encode('ascii', 'ignore').decode()
|
|
91
|
+
title = title.translate(self._special_chars_map)
|
|
92
|
+
title = title.replace(' ', '.*')
|
|
93
|
+
title = '^' + title + '$'
|
|
94
|
+
|
|
95
|
+
params = {'name_regex': title}
|
|
96
|
+
query = self.map_query_template.substitute(**params)
|
|
97
|
+
return query
|
|
98
|
+
|