FranKGraphBench 0.0.1a0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. FranKGraphBench-0.0.1a0.dist-info/LICENSE +201 -0
  2. FranKGraphBench-0.0.1a0.dist-info/METADATA +189 -0
  3. FranKGraphBench-0.0.1a0.dist-info/RECORD +70 -0
  4. FranKGraphBench-0.0.1a0.dist-info/WHEEL +6 -0
  5. FranKGraphBench-0.0.1a0.dist-info/entry_points.txt +3 -0
  6. FranKGraphBench-0.0.1a0.dist-info/top_level.txt +2 -0
  7. data_integration/__init__.py +47 -0
  8. data_integration/__main__.py +4 -0
  9. data_integration/dataset.py +270 -0
  10. data_integration/dataset2class.py +34 -0
  11. data_integration/datasets/__init__.py +0 -0
  12. data_integration/datasets/amazon.py +140 -0
  13. data_integration/datasets/book-crossing.py +98 -0
  14. data_integration/datasets/douban-movie.py +75 -0
  15. data_integration/datasets/lastfm.py +215 -0
  16. data_integration/datasets/movielens.py +375 -0
  17. data_integration/datasets/steam.py +98 -0
  18. data_integration/datasets/worker.py +31 -0
  19. data_integration/datasets/yelp.py +156 -0
  20. framework/__init__.py +17 -0
  21. framework/__main__.py +4 -0
  22. framework/dataloader/__init__.py +0 -0
  23. framework/dataloader/dataloader.py +52 -0
  24. framework/dataloader/dataset/__init__.py +0 -0
  25. framework/dataloader/dataset/dataset.py +38 -0
  26. framework/dataloader/edge_splitter/__init__.py +0 -0
  27. framework/dataloader/edge_splitter/edge_splitter.py +163 -0
  28. framework/dataloader/graph/__init__.py +0 -0
  29. framework/dataloader/graph/graph.py +314 -0
  30. framework/dataloader/graph/node.py +80 -0
  31. framework/dataloader/preprocess/__init__.py +0 -0
  32. framework/dataloader/preprocess/methods.py +47 -0
  33. framework/evaluator/__init__.py +0 -0
  34. framework/evaluator/metric.py +39 -0
  35. framework/evaluator/metric2class.py +10 -0
  36. framework/evaluator/metrics/__init__.py +0 -0
  37. framework/evaluator/metrics/map.py +33 -0
  38. framework/evaluator/metrics/ndcg.py +49 -0
  39. framework/experiment.py +120 -0
  40. framework/recommender/__init__.py +0 -0
  41. framework/recommender/model2class.py +46 -0
  42. framework/recommender/models/__init__.py +0 -0
  43. framework/recommender/models/complEx/__init__.py +0 -0
  44. framework/recommender/models/complEx/model.py +193 -0
  45. framework/recommender/models/deep_walk_based/__init__.py +0 -0
  46. framework/recommender/models/deep_walk_based/model.py +149 -0
  47. framework/recommender/models/distMult/__init__.py +0 -0
  48. framework/recommender/models/distMult/model.py +187 -0
  49. framework/recommender/models/ePHEN/__init__.py +0 -0
  50. framework/recommender/models/ePHEN/model.py +159 -0
  51. framework/recommender/models/rESCAL/__init__.py +0 -0
  52. framework/recommender/models/rESCAL/model.py +184 -0
  53. framework/recommender/models/rotatE/__init__.py +0 -0
  54. framework/recommender/models/rotatE/model.py +195 -0
  55. framework/recommender/models/transD/__init__.py +0 -0
  56. framework/recommender/models/transD/model.py +190 -0
  57. framework/recommender/models/transE/__init__.py +0 -0
  58. framework/recommender/models/transE/model.py +194 -0
  59. framework/recommender/models/transH/__init__.py +0 -0
  60. framework/recommender/models/transH/model.py +193 -0
  61. framework/recommender/models/transR/__init__.py +0 -0
  62. framework/recommender/models/transR/model.py +190 -0
  63. framework/recommender/models/tuckER/__init__.py +0 -0
  64. framework/recommender/models/tuckER/model.py +199 -0
  65. framework/recommender/recommender.py +32 -0
  66. framework/recommender/utils/__init__.py +0 -0
  67. framework/recommender/utils/walker.py +185 -0
  68. framework/reporter/__init__.py +0 -0
  69. framework/reporter/report.py +101 -0
  70. framework/utils.py +10 -0
@@ -0,0 +1,215 @@
1
+ import os
2
+ import re
3
+ import queue
4
+ from string import Template
5
+ from ..dataset import Dataset
6
+
7
+ from io import BytesIO
8
+ from collections import defaultdict
9
+
10
+
11
+ import pandas as pd
12
+ from tqdm import tqdm
13
+ from thefuzz import process
14
+ from SPARQLWrapper import CSV
15
+
16
+
17
+ class LastFM(Dataset):
18
+ def __init__(self, input_path, output_path, n_workers=1):
19
+ super().__init__(input_path, output_path, n_workers)
20
+ self.dataset_name = "LastFM"
21
+
22
+ self.item_separator = "\t"
23
+ # self.user_separator = '|'
24
+ self.rating_separator = "\t"
25
+ self.social_separator = "\t"
26
+
27
+ self.item_fields = {"id": "item_id::string", "name": "name::string"}
28
+ self.user_fields = {"userID": "user_id::string"}
29
+ self.rating_fields = {
30
+ "userID": "user_id::string",
31
+ "artistID": "item_id::string",
32
+ "weight": "rating::number",
33
+ }
34
+ self.map_fields = {"item_id": "item_id::string", "URI": "URI::string"}
35
+ self.social_fields = {"userID": "user1::string", "friendID": "user2::string"}
36
+
37
+ self.map_query_template = Template(
38
+ """
39
+ PREFIX dct: <http://purl.org/dc/terms/>
40
+ PREFIX dbo: <http://dbpedia.org/ontology/>
41
+ PREFIX dbr: <http://dbpedia.org/resource/>
42
+ PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
43
+ PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
44
+ SELECT DISTINCT ?artist WHERE {
45
+ {
46
+ ?artist rdf:type dbo:MusicalArtist .
47
+ ?artist rdfs:label ?label .
48
+ FILTER regex(?label, "$name_regex", "i")
49
+ }
50
+ UNION
51
+ {
52
+ ?artist rdf:type dbo:MusicalArtist .
53
+ ?tmp dbo:wikiPageRedirects ?artist .
54
+ ?tmp rdfs:label ?label .
55
+ FILTER regex(?label, "$name_regex", "i") .
56
+ }
57
+ UNION
58
+ {
59
+ ?artist rdf:type dbo:Band .
60
+ ?artist rdfs:label ?label .
61
+ FILTER regex(?label, "$name_regex", "i")
62
+ }
63
+ UNION
64
+ {
65
+ ?artist rdf:type dbo:Band .
66
+ ?tmp dbo:wikiPageRedirects ?artist .
67
+ ?tmp rdfs:label ?label .
68
+ FILTER regex(?label, "$name_regex", "i") .
69
+ }
70
+ }
71
+ """
72
+ )
73
+
74
+ self.enrich_query_template = Template(
75
+ """
76
+ PREFIX dct: <http://purl.org/dc/terms/>
77
+ PREFIX dbo: <http://dbpedia.org/ontology/>
78
+ PREFIX dbr: <http://dbpedia.org/resource/>
79
+ PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
80
+ PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
81
+ SELECT DISTINCT
82
+ ?abstract
83
+ (GROUP_CONCAT(DISTINCT ?bandMember; SEPARATOR="::") AS ?bandMember)
84
+ (GROUP_CONCAT(DISTINCT ?genre; SEPARATOR="::") AS ?genre)
85
+ (GROUP_CONCAT(DISTINCT ?associatedMusicalArtist; SEPARATOR="::") AS ?associatedMusicalArtist)
86
+ (GROUP_CONCAT(DISTINCT ?awards; SEPARATOR="::") AS ?awards)
87
+ (GROUP_CONCAT(DISTINCT ?recordLabel; SEPARATOR="::") AS ?recordLabel)
88
+ (GROUP_CONCAT(DISTINCT ?associatedBand; SEPARATOR="::") AS ?associatedBand)
89
+ (GROUP_CONCAT(DISTINCT ?origin; SEPARATOR="::") AS ?origin)
90
+ WHERE {
91
+ OPTIONAL { <$URI> dbo:genre ?genre } .
92
+ OPTIONAL { <$URI> dbo:abstract ?abstract } .
93
+ OPTIONAL { <$URI> dbp:origin ?origin } .
94
+ OPTIONAL { <$URI> dbo:recordLabel ?recordLabel } .
95
+ OPTIONAL { <$URI> dbo:bandMember ?bandMember } .
96
+ OPTIONAL { <$URI> dbo:associatedMusicalArtist ?associatedMusicalArtist } .
97
+ OPTIONAL { <$URI> dbo:associatedBand ?associatedBand } .
98
+ OPTIONAL { <$URI> dbp:awards ?awards } .
99
+
100
+ FILTER(LANG(?abstract) = 'en')
101
+ }
102
+ """
103
+ )
104
+
105
+ def load_item_data(self) -> pd.DataFrame():
106
+ filename = os.path.join(self.input_path, "artists.dat")
107
+
108
+ df = pd.read_csv(filename, sep=self.item_separator)
109
+ df = df.rename(self.item_fields, axis=1)
110
+ df = df[self.item_fields.values()]
111
+ return df
112
+
113
+ def entity_linking(self, df_item) -> pd.DataFrame():
114
+ q = queue.Queue()
115
+ for idx, row in df_item[["name", "item_id"]].iterrows():
116
+ query = self.get_map_query(row["name"])
117
+ q.put((row["item_id"], query))
118
+
119
+ if self.n_workers > 1:
120
+ responses = self.parallel_queries(q)
121
+ else:
122
+ responses = self.sequential_queries(q)
123
+
124
+ URI_mapping = {}
125
+ for response in tqdm(responses, desc="Disambiguating query return"):
126
+ candidate_URIs = []
127
+ idx, result = response
128
+ for binding in result["results"]["bindings"]:
129
+ URI = binding["artist"]["value"]
130
+ candidate_URIs.append(URI)
131
+
132
+ expected_URI = f'http://dbpedia.org/resource/{df_item.loc[df_item.item_id == idx]["name"]}'
133
+ str_matching_result = process.extractOne(expected_URI, candidate_URIs)
134
+
135
+ if str_matching_result is not None:
136
+ URI, _ = str_matching_result
137
+ URI_mapping[idx] = URI
138
+
139
+ df_map = pd.DataFrame({"item_id": df_item["item_id"]})
140
+ df_map.set_index("item_id")
141
+ df_map["URI"] = df_map["item_id"].apply(lambda id: URI_mapping.get(id))
142
+ df_map = df_map.rename(self.map_fields, axis=1)
143
+
144
+ return df_map
145
+
146
+ def get_map_query(self, name) -> str:
147
+ name = name.translate(self._special_chars_map)
148
+ name = name.replace(" ", ".*")
149
+ name = "^" + name
150
+ name = name + "$"
151
+
152
+ params = {"name_regex": name}
153
+ query = self.map_query_template.substitute(**params)
154
+ return query
155
+
156
+ def enrich(self, df_map):
157
+ df_map = df_map[df_map["URI"].notna()]
158
+
159
+ q = queue.Queue()
160
+ for _, row in df_map[["URI", "item_id"]].iterrows():
161
+ query = self.get_enrich_query(row["URI"])
162
+ q.put((row["item_id"], query))
163
+
164
+ if self.n_workers > 1:
165
+ responses = self.parallel_queries(q, CSV)
166
+ else:
167
+ responses = self.sequential_queries(q, CSV)
168
+
169
+ item_enriching = defaultdict(dict)
170
+ for response in responses:
171
+ idx, result = response
172
+ df = pd.read_csv(BytesIO(result))
173
+
174
+ if df.shape[0] > 1:
175
+ print("At least one property has more than one value!")
176
+ print(df.value_counts(dropna=False))
177
+
178
+ item_enriching[idx] = df.iloc[0] # getting pd.Series
179
+
180
+ df_enrich = pd.DataFrame.from_dict(item_enriching, orient="index")
181
+ df_enrich.index.name = "item_id"
182
+
183
+ return df_enrich
184
+
185
+ def get_enrich_query(self, URI) -> str:
186
+ params = {"URI": URI}
187
+ query = self.enrich_query_template.substitute(**params)
188
+ return query
189
+
190
+ def load_rating_data(self) -> pd.DataFrame():
191
+ filename = os.path.join(self.input_path, "user_artists.dat")
192
+ df = pd.read_csv(filename, sep=self.rating_separator)
193
+ df = df.rename(self.rating_fields, axis=1)
194
+ df = df[self.rating_fields.values()]
195
+
196
+ return df
197
+
198
+ def load_social_data(self) -> pd.DataFrame():
199
+ filename = os.path.join(self.input_path, "user_friends.dat")
200
+ df = pd.read_csv(filename, sep=self.social_separator)
201
+ df = df.rename(self.social_fields, axis=1)
202
+ df = df[self.social_fields.values()]
203
+
204
+ return df
205
+
206
+ def load_user_data(self) -> pd.DataFrame():
207
+ # There is no .dat only for user (none aditional user info)
208
+ # Will extract the users from the rating data
209
+ filename = os.path.join(self.input_path, "user_artists.dat")
210
+ df = pd.read_csv(filename, sep=self.rating_separator)
211
+ users = df["userID"].unique()
212
+ df = df.rename(self.user_fields)
213
+ df = pd.DataFrame(pd.Series(users), columns=self.user_fields.values())
214
+
215
+ return df
@@ -0,0 +1,375 @@
1
+ import os
2
+ import re
3
+ import queue
4
+ from io import BytesIO
5
+ from collections import defaultdict
6
+
7
+
8
+ from string import Template
9
+ from ..dataset import Dataset
10
+
11
+ import pandas as pd
12
+ from tqdm import tqdm
13
+ from thefuzz import process
14
+ from SPARQLWrapper import CSV
15
+
16
+
17
+ class MovieLens(Dataset):
18
+ """
19
+ General MovieLens(Dataset) class for data integration between DBpedia and MovieLens
20
+ """
21
+
22
+ def __init__(self, input_path, output_path, n_workers=1):
23
+ super().__init__(input_path, output_path, n_workers)
24
+ self.map_query_template = Template(
25
+ """
26
+ PREFIX dct: <http://purl.org/dc/terms/>
27
+ PREFIX dbo: <http://dbpedia.org/ontology/>
28
+ PREFIX dbr: <http://dbpedia.org/resource/>
29
+ PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
30
+ PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
31
+ SELECT DISTINCT ?film WHERE {
32
+ {
33
+ ?film rdf:type dbo:Film .
34
+ ?film dct:subject $year_category .
35
+ ?film rdfs:label ?label .
36
+ FILTER regex(?label, "$name_regex", "i")
37
+ }
38
+ UNION
39
+ {
40
+ ?film rdf:type dbo:Film .
41
+ ?film dct:subject $year_category .
42
+ ?tmp dbo:wikiPageRedirects ?film .
43
+ ?tmp rdfs:label ?label .
44
+ FILTER regex(?label, "$name_regex", "i") .
45
+ }
46
+ }
47
+ """
48
+ )
49
+
50
+ self.enrich_query_template = Template(
51
+ """
52
+ PREFIX dct: <http://purl.org/dc/terms/>
53
+ PREFIX dbo: <http://dbpedia.org/ontology/>
54
+ PREFIX dbr: <http://dbpedia.org/resource/>
55
+ PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
56
+ PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
57
+ SELECT DISTINCT
58
+ ?abstract
59
+ (GROUP_CONCAT(DISTINCT ?producer; SEPARATOR="::") AS ?producer)
60
+ (GROUP_CONCAT(DISTINCT ?distributor; SEPARATOR="::") AS ?distributor)
61
+ (GROUP_CONCAT(DISTINCT ?writer; SEPARATOR="::") AS ?writer)
62
+ (GROUP_CONCAT(DISTINCT ?cinematography; SEPARATOR="::") AS ?cinematography)
63
+ (GROUP_CONCAT(DISTINCT ?subject; SEPARATOR="::") AS ?subject)
64
+ (GROUP_CONCAT(DISTINCT ?starring; SEPARATOR="::") AS ?starring)
65
+ (GROUP_CONCAT(DISTINCT ?director; SEPARATOR="::") AS ?director)
66
+ WHERE {
67
+ OPTIONAL { <$URI> dct:subject ?subject } .
68
+ OPTIONAL { <$URI> dbo:starring ?starring } .
69
+ OPTIONAL { <$URI> dbo:director ?director } .
70
+ OPTIONAL { <$URI> dbo:abstract ?abstract } .
71
+ OPTIONAL { <$URI> dbo:producer ?producer } .
72
+ OPTIONAL { <$URI> dbo:distributor ?distributor } .
73
+ OPTIONAL { <$URI> dbo:writer ?writer } .
74
+ OPTIONAL { <$URI> dbo:cinematography ?cinematography } .
75
+
76
+ FILTER(LANG(?abstract) = 'en')
77
+ }
78
+ """
79
+ )
80
+
81
+ def _extract_title(self, title):
82
+ regex = re.compile(r"\((\d{4})\)")
83
+ rgx_match = regex.search(title)
84
+ year_start, _ = rgx_match.span()
85
+ movie_title = title[: year_start - 1].strip()
86
+
87
+ # removing parenthesis (name of the film in other languages)
88
+ movie_title = [s.split(")")[-1] for s in movie_title.split("(")][0]
89
+
90
+ # Reversing titles with X, The
91
+ movie_title = movie_title.split(",")
92
+ movie_title = [movie_title[-1].strip()] + [
93
+ title.strip() for title in movie_title[:-1]
94
+ ]
95
+ movie_title = " ".join(movie_title)
96
+
97
+ movie_title = movie_title.strip()
98
+ return movie_title
99
+
100
+ def _extract_year(self, title):
101
+ regex = re.compile(r"\((\d{4})\)")
102
+ rgx_match = regex.search(title)
103
+ movie_year = rgx_match.group()
104
+ movie_year = int(movie_year[1:-1])
105
+ return movie_year
106
+
107
+ def entity_linking(self, df_item) -> pd.DataFrame():
108
+ q = queue.Queue()
109
+ for idx, row in df_item[["movie_title", "movie_year", "item_id"]].iterrows():
110
+ query = self.get_map_query(row["movie_title"], row["movie_year"])
111
+ q.put((row["item_id"], query))
112
+
113
+ if self.n_workers > 1:
114
+ responses = self.parallel_queries(q)
115
+ else:
116
+ responses = self.sequential_queries(q)
117
+
118
+ URI_mapping = defaultdict(dict)
119
+ for response in tqdm(responses, desc="Disambiguating query return"):
120
+ candidate_URIs = []
121
+ idx, result = response
122
+ for binding in result["results"]["bindings"]:
123
+ URI = binding["film"]["value"]
124
+ candidate_URIs.append(URI)
125
+
126
+ expected_URI = f"http://dbpedia.org/resource/{df_item.loc[df_item.item_id == idx]['movie_title']}"
127
+ str_matching_result = process.extractOne(expected_URI, candidate_URIs)
128
+
129
+ if str_matching_result is not None:
130
+ URI, _ = str_matching_result
131
+ URI_mapping[idx] = URI
132
+
133
+ df_map = pd.DataFrame({"item_id": df_item["item_id"]})
134
+ df_map.set_index("item_id")
135
+ df_map["URI"] = df_map["item_id"].apply(lambda id: URI_mapping.get(id))
136
+ df_map = df_map.rename(self.map_fields, axis=1)
137
+
138
+ return df_map
139
+
140
+ def get_map_query(self, title, year) -> str:
141
+ # title = title.translate(self._special_chars_map)
142
+ title = title.replace(" ", ".*")
143
+ title = "^" + title
144
+
145
+ params = {
146
+ "name_regex": title,
147
+ "year_category": f"dbr:Category:{str(year)}_films",
148
+ }
149
+ query = self.map_query_template.substitute(**params)
150
+ return query
151
+
152
+ def enrich(self, df_map) -> pd.DataFrame():
153
+ df_map = df_map[df_map["URI"].notna()]
154
+
155
+ q = queue.Queue()
156
+ for _, row in df_map[["URI", "item_id"]].iterrows():
157
+ query = self.get_enrich_query(row["URI"])
158
+ q.put((row["item_id"], query))
159
+
160
+ if self.n_workers > 1:
161
+ responses = self.parallel_queries(q, CSV)
162
+ else:
163
+ responses = self.sequential_queries(q, CSV)
164
+
165
+ item_enriching = defaultdict(dict)
166
+ for response in responses:
167
+ idx, result = response
168
+ df = pd.read_csv(BytesIO(result))
169
+
170
+ if df.shape[0] > 1:
171
+ print("At least one property has more than one value!")
172
+ print(df.value_counts(dropna=False))
173
+
174
+ item_enriching[idx] = df.iloc[0] # getting pd.Series
175
+
176
+ df_enrich = pd.DataFrame.from_dict(item_enriching, orient="index")
177
+ df_enrich.index.name = "item_id"
178
+
179
+ return df_enrich
180
+
181
+ def get_enrich_query(self, URI) -> str:
182
+ params = {"URI": URI}
183
+ query = self.enrich_query_template.substitute(**params)
184
+ return query
185
+
186
+
187
+ class MovieLens100k(MovieLens):
188
+ """
189
+ Dataset class for data integration between DBpedia and MovieLens-100k
190
+ """
191
+
192
+ def __init__(self, input_path, output_path, n_workers=1):
193
+ super().__init__(input_path, output_path, n_workers)
194
+ self.dataset_name = "MovieLens100k"
195
+
196
+ self.item_separator = "|"
197
+ self.user_separator = "|"
198
+ self.rating_separator = "\t"
199
+ self.item_fields = {
200
+ "movie id": "item_id::string",
201
+ "movie_title": "movie_title::string",
202
+ "movie_year": "movie_year::string", # extracted from movie_title
203
+ }
204
+ self.map_fields = {"item_id": "item_id::string", "URI": "URI::string"}
205
+ self.rating_fields = {
206
+ "user id": "user_id::string",
207
+ "item id": "item_id::string",
208
+ "rating": "rating::number",
209
+ "timestamp": "timestamp::number",
210
+ }
211
+ self.user_fields = {
212
+ "user id": "user_id::string",
213
+ "age": "age::string",
214
+ "gender": "gender::string",
215
+ "occupation": "occupation::string",
216
+ }
217
+
218
+ def load_item_data(self) -> pd.DataFrame():
219
+ # Reading item file of MovieLens100k dataset
220
+ filename = os.path.join(self.input_path, "u.item")
221
+ column_names = """movie id | movie title | release date | video release date |
222
+ IMDb URL | unknown | Action | Adventure | Animation |
223
+ Children's | Comedy | Crime | Documentary | Drama | Fantasy |
224
+ Film-Noir | Horror | Musical | Mystery | Romance | Sci-Fi |
225
+ Thriller | War | Western"""
226
+ column_names = [name.strip() for name in column_names.split("|")]
227
+ df = pd.read_csv(
228
+ filename,
229
+ header=None,
230
+ sep=self.item_separator,
231
+ encoding="latin-1",
232
+ names=column_names,
233
+ )
234
+
235
+ # Pre processing item
236
+ # 1- Dropping unused fields
237
+ df.set_index("movie id")
238
+ df = df.drop(["video release date", "unknown", "IMDb URL"], axis=1)
239
+ df = df.dropna()
240
+
241
+ # 2- Extracting title and year
242
+ df["movie_title"] = df["movie title"].apply(self._extract_title)
243
+ df["movie_year"] = df["movie title"].apply(self._extract_year)
244
+
245
+ df = df.rename(self.item_fields, axis=1)
246
+ df = df[self.item_fields.values()]
247
+
248
+ return df
249
+
250
+ def load_user_data(self) -> pd.DataFrame():
251
+ filename = os.path.join(self.input_path, "u.user")
252
+ column_names = "user id | age | gender | occupation | zip code"
253
+ column_names = [name.strip() for name in column_names.split("|")]
254
+
255
+ df = pd.read_csv(
256
+ filename,
257
+ header=None,
258
+ sep=self.user_separator,
259
+ encoding="latin-1",
260
+ names=column_names,
261
+ )
262
+ df = df.rename(self.user_fields, axis=1)
263
+ df = df[self.user_fields.values()]
264
+
265
+ return df
266
+
267
+ def load_rating_data(self) -> pd.DataFrame():
268
+ filename = os.path.join(self.input_path, "u.data")
269
+ column_names = "user id | item id | rating | timestamp"
270
+ column_names = [name.strip() for name in column_names.split("|")]
271
+
272
+ df = pd.read_csv(
273
+ filename,
274
+ header=None,
275
+ sep=self.rating_separator,
276
+ encoding="latin-1",
277
+ names=column_names,
278
+ )
279
+ df = df.rename(self.rating_fields, axis=1)
280
+ df = df[self.rating_fields.values()]
281
+
282
+ return df
283
+
284
+
285
+ class MovieLens1M(MovieLens):
286
+ """
287
+ Dataset class for data integration between DBpedia and MovieLens 1M
288
+ """
289
+
290
+ def __init__(self, input_path, output_path, n_workers=1):
291
+ super().__init__(input_path, output_path, n_workers)
292
+ self.dataset_name = "MovieLens1M"
293
+
294
+ self.item_separator = "::"
295
+ self.user_separator = "::"
296
+ self.rating_separator = "::"
297
+ self.item_fields = {
298
+ "MovieID": "item_id::string",
299
+ "movie_title": "movie_title::string",
300
+ "movie_year": "movie_year::string", # extracted from movie_title
301
+ }
302
+ self.map_fields = {"item_id": "item_id::string", "URI": "URI::string"}
303
+ self.rating_fields = {
304
+ "UserID": "user_id::string",
305
+ "MovieID": "item_id::string",
306
+ "Rating": "rating::number",
307
+ "Timestamp": "timestamp::number",
308
+ }
309
+ self.user_fields = {
310
+ "UserID": "user_id::string",
311
+ "Gender": "gender::string",
312
+ "Age": "age::string",
313
+ "Occupation": "occupation::string",
314
+ }
315
+
316
+ def load_item_data(self) -> pd.DataFrame():
317
+ # Reading item file of MovieLens100k dataset
318
+ filename = os.path.join(self.input_path, "movies.dat")
319
+ column_names = "MovieID::Title::Genres"
320
+ column_names = [name.strip() for name in column_names.split("::")]
321
+ df = pd.read_csv(
322
+ filename,
323
+ header=None,
324
+ sep=self.item_separator,
325
+ encoding="latin-1",
326
+ names=column_names,
327
+ engine="python",
328
+ )
329
+
330
+ # Pre processing item
331
+ # Extracting title and year
332
+ df["movie_title"] = df["Title"].apply(self._extract_title)
333
+ df["movie_year"] = df["Title"].apply(self._extract_year)
334
+
335
+ df = df.rename(self.item_fields, axis=1)
336
+ df = df[self.item_fields.values()]
337
+
338
+ return df
339
+
340
+ def load_user_data(self) -> pd.DataFrame():
341
+ filename = os.path.join(self.input_path, "users.dat")
342
+ column_names = "UserID::Gender::Age::Occupation::Zip-code"
343
+ column_names = [name.strip() for name in column_names.split("::")]
344
+
345
+ df = pd.read_csv(
346
+ filename,
347
+ header=None,
348
+ sep=self.user_separator,
349
+ encoding="latin-1",
350
+ names=column_names,
351
+ engine="python",
352
+ )
353
+
354
+ df = df.rename(self.user_fields, axis=1)
355
+ df = df[self.user_fields.values()]
356
+
357
+ return df
358
+
359
+ def load_rating_data(self) -> pd.DataFrame():
360
+ filename = os.path.join(self.input_path, "ratings.dat")
361
+ column_names = "UserID::MovieID::Rating::Timestamp"
362
+ column_names = [name.strip() for name in column_names.split("::")]
363
+
364
+ df = pd.read_csv(
365
+ filename,
366
+ header=None,
367
+ sep=self.rating_separator,
368
+ encoding="latin-1",
369
+ names=column_names,
370
+ engine="python",
371
+ )
372
+ df = df.rename(self.rating_fields, axis=1)
373
+ df = df[self.rating_fields.values()]
374
+
375
+ return df
@@ -0,0 +1,98 @@
1
+ import os
2
+ import re
3
+ import queue
4
+ from string import Template
5
+ from ..dataset import Dataset
6
+
7
+ import pandas as pd
8
+ from tqdm import tqdm
9
+ from thefuzz import process
10
+
11
+ class Steam(Dataset):
12
+ def __init__(self, input_path, output_path, n_workers=1):
13
+ super().__init__(input_path, output_path, n_workers)
14
+ self.dataset_name = 'Steam'
15
+
16
+ self.item_separator = ','
17
+ # self.user_separator = '|'
18
+ # self.rating_separator = '\t'
19
+
20
+ self.item_fields = ['item_id', 'title', 'date_release']
21
+ # self.user_fields = ['user_id', 'age', 'gender', 'occupation']
22
+ # self.rating_fields = ['user_id', 'item_id', 'rating']
23
+ # self.map_fields = ['item_id', 'URI']
24
+
25
+ self.map_query_template = Template('''
26
+ PREFIX dct: <http://purl.org/dc/terms/>
27
+ PREFIX dbo: <http://dbpedia.org/ontology/>
28
+ PREFIX dbr: <http://dbpedia.org/resource/>
29
+ PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
30
+ PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
31
+ SELECT DISTINCT ?game WHERE {
32
+ {
33
+ ?game rdf:type dbo:VideoGame .
34
+ ?game rdfs:label ?label .
35
+ FILTER regex(?label, "$name_regex", "i")
36
+ }
37
+ UNION
38
+ {
39
+ ?game rdf:type dbo:VideoGame .
40
+ ?tmp dbo:wikiPageRedirects ?game .
41
+ ?tmp rdfs:label ?label .
42
+ FILTER regex(?label, "$name_regex", "i") .
43
+ }
44
+ }
45
+ ''')
46
+
47
+ def load_item_data(self) -> pd.DataFrame():
48
+ filename = os.path.join(self.input_path, 'steam_games.csv')
49
+ used_columns = ['app_id', 'title', 'date_release']
50
+
51
+ df = pd.read_csv(filename, sep=self.item_separator)
52
+ df = df[used_columns]
53
+ df.columns = self.item_fields
54
+
55
+ return df
56
+
57
+ def entity_linking(self, df_item) -> pd.DataFrame():
58
+ q = queue.Queue()
59
+ for idx, row in df_item[['title', 'item_id']].iterrows():
60
+ query = self.get_map_query(row['title'])
61
+ q.put((row['item_id'], query))
62
+
63
+ if self.n_workers > 1:
64
+ responses = self.parallel_queries(q)
65
+ else:
66
+ responses = self.sequential_queries(q)
67
+
68
+ URI_mapping = {}
69
+ for response in tqdm(responses, desc='Disambiguating query return'):
70
+ candidate_URIs = []
71
+ idx, result = response
72
+ for binding in result['results']['bindings']:
73
+ URI = binding['game']['value']
74
+ candidate_URIs.append(URI)
75
+
76
+ expected_URI = f'http://dbpedia.org/resource/{df_item.loc[df_item.item_id == idx]["title"]}'
77
+ str_matching_result = process.extractOne(expected_URI, candidate_URIs)
78
+
79
+ if str_matching_result is not None:
80
+ URI, _ = str_matching_result
81
+ URI_mapping[idx] = URI
82
+
83
+ df_map = pd.DataFrame({'item_id': df_item['item_id']})
84
+ df_map.set_index('item_id')
85
+ df_map['URI'] = df_map['item_id'].apply(lambda id: URI_mapping.get(id))
86
+
87
+ return df_map
88
+
89
+ def get_map_query(self, title) -> str:
90
+ title = title.encode('ascii', 'ignore').decode()
91
+ title = title.translate(self._special_chars_map)
92
+ title = title.replace(' ', '.*')
93
+ title = '^' + title + '$'
94
+
95
+ params = {'name_regex': title}
96
+ query = self.map_query_template.substitute(**params)
97
+ return query
98
+