FranKGraphBench 0.0.1a0__py2.py3-none-any.whl

data_integration/dataset.py

@@ -0,0 +1,270 @@
+import os
+import queue
+import string
+import pandas as pd
+from string import Template
+from .datasets.worker import Worker
+
+from SPARQLWrapper import SPARQLWrapper, JSON
+from tqdm import tqdm
+
+
+class Dataset:
+    def __init__(self, input_path, output_path, n_workers=1):
+        r"""
+        Base class for Recommender System's datasets
+        """
+        self.input_path = input_path
+        self.output_path = output_path
+        self.n_workers = n_workers
+
+        self.sparql_endpoint = "http://dbpedia.org/sparql"
+        self.timeout = 180
+
+        # Output files
+        self.item_filename = os.path.join(self.output_path, "item.csv")
+        self.user_filename = os.path.join(self.output_path, "user.csv")
+        self.rating_filename = os.path.join(self.output_path, "rating.csv")
+        self.social_filename = os.path.join(self.output_path, "social.csv")
+        self.map_filename = os.path.join(self.output_path, "map.csv")
+        self.enriched_filename = os.path.join(self.output_path, "enriched.csv")
+
+        # create output path if doesn't exist
+        self.has_output_path()
+        # Mapping regex special chars
+        self._special_chars_map = str.maketrans({x: "" for x in string.punctuation})
+
+        # The following variables need to be overwritten by the subclasses of Dataset()
+        self.dataset_name = ""
+        self.item_separator = ""
+        self.user_separator = ""
+        self.rating_separator = ""
+        self.social_separator = ""
+        # Chosen fields to be extracted from dataset and mapping between the original column name and
+        # their standardized name
+        # Example: {"movie_title": "movie_title::string"}
+        self.reverse_dict = lambda x: {v: k for k, v in x.items()}
+        self.item_fields = {}
+        self.user_fields = {}
+        self.rating_fields = {}
+        self.map_query_template = Template(None)
+        self.enrich_query_template = Template(None)
+
+    def has_output_path(self):
+        if not os.path.isdir(self.output_path):
+            os.mkdir(self.output_path)
+
+    def load_item_data(self) -> pd.DataFrame:
+        """
+        Loads item info of Dataset
+        :return: returns pd.Dataframe containing each item info.
+        """
+        raise NotImplementedError
+
+    def load_user_data(self) -> pd.DataFrame:
+        """
+        Loads user info of Dataset
+        :return: returns pd.Dataframe containing each user info.
+        """
+        raise NotImplementedError
+
+    def load_rating_data(self) -> pd.DataFrame:
+        """
+        Loads rating interactions of Dataset
+        :return: returns pd.Dataframe containing each rating interaction.
+        """
+        raise NotImplementedError
+
+    def load_social_data(self) -> pd.DataFrame:
+        """
+        Loads social links of Dataset
+        :return: returns pd.Dataframe containing each social link between users.
+        """
+        raise NotImplementedError
+
+    def entity_linking(self, df_item) -> pd.DataFrame:
+        """
+        Entity link each item to their corresponding DBpedia's URI
+        :return: returns pd.Dataframe containing each item_id and their mapped URI
+        """
+        raise NotImplementedError
+
+    def enrich(self, df_map):
+        raise NotImplementedError
+
+    def get_map_query(self, *args, **kwargs) -> str:
+        """
+        Returns query parameters to be substituted in the map query template.
+        :returns: returns string query
+        """
+        raise NotImplementedError
+
+    def get_enrich_query(self, *args, **kwargs) -> str:
+        """
+        Returns query parameters to be substituted in the enrich query template.
+        :returns: returns string query
+        """
+        raise NotImplementedError
+
+    def parallel_queries(self, queue, return_type=JSON):
+        """
+        Parallel query SPARQL endpoint using Threads
+        :arguments:
+            queue: queue of tuples indicating the item_id and SPARQL query string
+        :returns: list containing tuples with item_id and response result in JSON
+        """
+        n_iters = queue.qsize()
+        pbar = tqdm(total=n_iters)
+        pbar.set_description("Requesting SPARQL endpoint for each item")
+
+        # instantiating each Worker
+        workers = []
+        for _ in range(self.n_workers):
+            worker = Worker(queue, lambda q: self._query(q, return_type), pbar)
+            worker.start()
+            workers.append(worker)
+
+        for worker in workers:
+            worker.join()
+        pbar.close()
+
+        # combining results
+        responses = []
+        for worker in workers:
+            responses.extend(worker.local_results)
+
+        return responses
+
+    def sequential_queries(self, q, return_type=JSON):
+        """
+        Sequential query SPARQL endpoint
+        :arguments:
+            queue: queue of tuples indicating the item_id and SPARQL query string
+        :returns: list containing tuples with item_id and response result in JSON
+        """
+        pbar = tqdm(total=q.qsize())
+        pbar.set_description("Requesting SPARQL endpoint for each item")
+        responses = []
+
+        while True:
+            try:
+                idx, query = q.get(block=False)
+                response = self._query(query, return_type)
+                responses.append((idx, response))
+                pbar.update(n=1)
+            except queue.Empty:
+                break
+            except Exception as e:
+                print(f"Exception:")
+                print(e)
+
+        return responses
+
+    def _query(self, query, return_type=JSON) -> dict:
+        sparql = SPARQLWrapper(self.sparql_endpoint)
+        sparql.setTimeout(self.timeout)
+        sparql.setQuery(query)
+        sparql.setReturnFormat(return_type)
+
+        try:
+            return sparql.query().convert()
+        except Exception as e:
+            raise e
+
+    def convert_item_data(self):
+        """
+        Converts loaded item data to a processed csv (item.csv).
+        """
+        try:
+            print(f"Creating file: {self.item_filename}.")
+            df_item = self.load_item_data()
+            print(f"{df_item.shape[0]} items with {df_item.shape[1]} Fields")
+            print("Fields: " + ", ".join(df_item.columns))
+            df_item.to_csv(self.item_filename, index=False)
+        except NotImplementedError:
+            print("Override load_item_data() method of your Dataset subclass.")
+
+    def convert_user_data(self):
+        """
+        Converts loaded user data to a processed csv (user.csv).
+        """
+        try:
+            print(f"Creating file: {self.user_filename}")
+            df_user = self.load_user_data()
+            print(f"{df_user.shape[0]} users with {df_user.shape[1]} Fields")
+            print("Fields: " + ", ".join(df_user.columns))
+            df_user.to_csv(self.user_filename, index=False)
+        except NotImplementedError:
+            print(f"Override load_user_data() of your Dataset subclass.")
+
+    def convert_rating_data(self):
+        """
+        Converts loaded rating data to a processed csv (rating.csv)
+        """
+        try:
+            print(f"Creating file: {self.rating_filename}")
+            df_rating = self.load_rating_data()
+            print(f"{df_rating.shape[0]} ratings with {df_rating.shape[1]} Fields")
+            print("Fields: " + ", ".join(df_rating.columns))
+            df_rating.to_csv(self.rating_filename, index=False)
+        except NotImplementedError:
+            print(f"Override load_rating_data() of your Dataset subclass.")
+
+    def convert_social_data(self):
+        """
+        Converts loaded social data to a processed csv (social.csv)
+        """
+        try:
+            print(f"Creating file: {self.social_filename}")
+            df_social = self.load_social_data()
+            print(f"{df_social.shape[0]} social link between users")
+            df_social.to_csv(self.social_filename, index=False)
+        except NotImplementedError:
+            print(f"Override load_rating_data() of your Dataset subclass.")
+
+    def map_URIs(self):
+        """
+        Maps each item to their corresponding DBpedia's URI.
+        """
+        try:
+            df_item = pd.read_csv(self.item_filename)
+            df_item.columns = [col.split("::")[0] for col in df_item.columns]
+            print(f"Mapping each dataset item with DBpedia: {self.map_filename}")
+            df_map = self.entity_linking(df_item)
+            df_map.to_csv(self.map_filename, index=False)
+
+            # print mapping statistics
+            n_items = df_map.shape[0]
+            n_unmatched = df_map[self.map_fields["URI"]].isna().sum()
+            print(
+                f"{n_unmatched} items weren't matched, corresponding to {n_unmatched/n_items*100:.2f}% of a total of {n_items}."
+            )
+
+        except NotImplementedError:
+            print("Override entity_linking() method of your Dataset subclass.")
+
+    def enrich_data(self):
+        """
+        Enrich each mapped item using DBpedia's resources.
+        """
+
+        try:
+            df_map = pd.read_csv(self.map_filename)
+            print(
+                f"Enriching each item with DBpedia resources: {self.enriched_filename}"
+            )
+            df_enrich = self.enrich(df_map)
+            df_enrich.to_csv(self.enriched_filename)
+
+            self.report_enriching(df_enrich)
+
+        except NotImplementedError:
+            print("Override enrich() method of your Dataset subclass.")
+
+    def report_enriching(self, df):
+        n_rows = df.shape[0]
+        for col in df.columns:
+            n_rows_with = df[col].notna().sum()
+            print(
+                f"number of entities with the property {col}: {n_rows_with} ({n_rows_with/n_rows*100:.2f}%)"
+            )

data_integration/dataset2class.py

@@ -0,0 +1,34 @@
+dataset2class = {
+    'ml-100k': {
+        'submodule': 'datasets.movielens',
+        'class': 'MovieLens100k'
+    },
+    'ml-1m': {
+        'submodule': 'datasets.movielens',
+        'class': 'MovieLens1M'
+    },
+    'lastfm': {
+        'submodule': 'datasets.lastfm',
+        'class': 'LastFM'
+    },
+    'book-crossing': {
+        'submodule': 'datasets.book-crossing',
+        'class': 'BookCrossing'
+    },
+    'steam': {
+        'submodule': 'datasets.steam',
+        'class': 'Steam'
+    },
+    'yelp': {
+        'submodule': 'datasets.yelp',
+        'class': 'Yelp'
+    },
+    'douban-movie': {
+        'submodule': 'datasets.douban-movie',
+        'class': 'DoubanMovie'
+    },
+    'amazon-video_games-5': {
+        'submodule': 'datasets.amazon',
+        'class': 'AmazonVideoGames5'
+    }
+}

data_integration/datasets/amazon.py

@@ -0,0 +1,140 @@
+import os
+import gzip
+import json
+import pandas as pd
+import re
+from collections import defaultdict
+from bs4 import BeautifulSoup
+
+from ..dataset import Dataset
+
+class Amazon(Dataset):
+    """
+    General Amazon(Dataset) class for data loading all different Amazon datasets
+    """
+    def __init__(self, input_path, output_path, n_workers=1):
+        super().__init__(input_path, output_path, n_workers)
+    
+    def _parse(self, path):
+        g = gzip.open(path, 'rb')
+        for l in g:
+            yield json.loads(l)
+
+    def _getDF(self, path):
+        i = 0
+        df = {}
+        for d in self._parse(path):
+            df[i] = d
+            i += 1
+        return pd.DataFrame.from_dict(df, orient='index')
+    
+class AmazonVideoGames5(Amazon):
+    def __init__(self, input_path, output_path, n_workers=1):
+        super().__init__(input_path, output_path, n_workers)
+        self.dataset_name = "Amazon Video Games 5-core"
+
+        self.string_list_separator = "::"
+
+        self.item_fields = {
+            "asin": "item_id::string",
+            "title": "name::string",
+            "feature": "features::string_list",
+            "description": "description::string",
+            "price": "price::number",
+            "also_buy": "also_buy::string_list",
+            "also_view": "also_view::string_list",
+            "brand": "brand::string",
+            "category": "categories::string_list"
+        }
+
+        self.user_fields = {
+            "reviewerID": "user_id::string",
+            "reviewerName": "name::string",
+        }
+
+        self.rating_fields = {
+            "reviewerID": "user_id::string",
+            "asin": "item_id::string",
+            "reviewText": "text::string",
+            "overall": "rating::number",
+            "summary": "summary::string",
+            "unixReviewTime": "timestamp::number"
+        }
+
+    def load_item_data(self) -> pd.DataFrame():
+        filename = os.path.join(self.input_path, "Video_Games_5.json.gz")
+        print(filename)
+        df = self._getDF(filename)
+
+        filename = os.path.join(self.input_path, "meta_Video_Games.json.gz")
+        print(filename)
+        meta_df = self._getDF(filename)
+        meta_df = meta_df[meta_df['asin'].isin(df['asin'].unique())].reset_index(drop=True)
+
+        features = defaultdict(list)
+        for index, row in meta_df['feature'].dropna().items():
+            for feature in row:
+                soup = BeautifulSoup(feature, 'html.parser')
+                feature = soup.text
+                if "" != feature:
+                    features[index].append(feature)
+        meta_df['feature'] = pd.Series(features).apply(lambda x: self.string_list_separator.join(x))
+        
+        descriptions = defaultdict(list)
+        for index, row in meta_df['description'].dropna().items():
+            for description in row:
+                soup = BeautifulSoup(description, 'html.parser')
+                description = soup.text
+                if "" != description:
+                    descriptions[index].append(description)
+        meta_df['description'] = pd.Series(descriptions).apply(lambda x: " ".join(x))
+
+        numeric_const_pattern = '[-+]? (?: (?: \d* \. \d+ ) | (?: \d+ \.? ) )(?: [Ee] [+-]? \d+ ) ?'
+        rx = re.compile(numeric_const_pattern, re.VERBOSE)
+        meta_df['price'] = meta_df['price'].apply(lambda x: rx.findall(x))
+        meta_df['price'] = meta_df['price'].apply(lambda x: x[0] if len(x) > 0 else None)
+        meta_df['price'] = meta_df['price'].astype('float64')
+
+        also_buys = defaultdict(list)
+        for index, row in meta_df['also_buy'].dropna().items():
+            for also_buy in row:
+                also_buys[index].append(also_buy)
+        meta_df['also_buy'] = pd.Series(also_buys).apply(lambda x: self.string_list_separator.join(x))
+
+        also_views = defaultdict(list)
+        for index, row in meta_df['also_view'].dropna().items():
+            for also_view in row:
+                also_views[index].append(also_view)
+        meta_df['also_view'] = pd.Series(also_views).apply(lambda x: self.string_list_separator.join(x))
+
+        categories = defaultdict(list)
+        for index, row in meta_df['category'].dropna().items():
+            for category in row:
+                soup = BeautifulSoup(category, 'html.parser')
+                category = soup.text
+                if "" != category:
+                    categories[index].append(category)
+        meta_df['category'] = pd.Series(categories).apply(lambda x: self.string_list_separator.join(x))
+
+        meta_df = meta_df.rename(self.item_fields, axis=1)
+        meta_df = meta_df[self.item_fields.values()]
+        return meta_df
+    
+    def load_user_data(self) -> pd.DataFrame():
+        filename = os.path.join(self.input_path, "Video_Games_5.json.gz")
+        print(filename)
+        df = self._getDF(filename)
+
+        df = df.rename(self.user_fields, axis=1)
+        df = df[self.user_fields.values()]
+        return df
+    
+    def load_rating_data(self) -> pd.DataFrame():
+        filename = os.path.join(self.input_path, "Video_Games_5.json.gz")
+        print(filename)
+        df = self._getDF(filename)
+
+        df = df.rename(self.rating_fields, axis=1)
+        df = df[self.rating_fields.values()]
+        return df
+

data_integration/datasets/book-crossing.py

@@ -0,0 +1,98 @@
+import os
+import re
+import queue
+from string import Template
+from ..dataset import Dataset
+
+import pandas as pd
+from tqdm import tqdm
+from thefuzz import process
+
+class BookCrossing(Dataset):
+    def __init__(self, input_path, output_path, n_workers=1):
+        super().__init__(input_path, output_path, n_workers)
+        self.dataset_name = 'Book-Crossing'
+
+        self.item_separator = ';'
+        self.user_separator = ';'
+        self.rating_separator = ';'
+
+        self.item_fields = ['item_id', 'title', 'author', 'year', 'publisher']
+        # self.user_fields = ['user_id', 'age', 'gender', 'occupation']
+        self.rating_fields = ['user_id', 'item_id', 'rating']
+        # self.map_fields = ['item_id', 'URI']
+
+        self.map_query_template = Template('''
+            PREFIX dct:  <http://purl.org/dc/terms/>
+            PREFIX dbo:  <http://dbpedia.org/ontology/>
+            PREFIX dbr:  <http://dbpedia.org/resource/>
+            PREFIX rdf:	 <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+            PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+            SELECT DISTINCT ?book WHERE {
+                {
+                    ?book rdf:type dbo:WrittenWork .
+                    ?book rdfs:label ?label .
+                    FILTER regex(?label, "$name_regex", "i")
+                }
+                UNION
+                {
+                    ?book rdf:type dbo:WrittenWork .
+                    ?tmp dbo:wikiPageRedirects ?book .
+                    ?tmp rdfs:label ?label .
+                    FILTER regex(?label, "$name_regex", "i") .
+                }
+            }
+        ''')
+
+
+    def load_item_data(self) -> pd.DataFrame():
+        filename = os.path.join(self.input_path, 'BX-Books.csv')
+
+        df = pd.read_csv(filename, sep=self.item_separator, encoding='CP1252', escapechar='\\')
+        df = df.drop(df.columns[-3:], axis=1) # Will not use picture URL related fields
+        df.columns = self.item_fields
+        return df
+    
+    def entity_linking(self, df_item) -> pd.DataFrame():
+        q = queue.Queue()
+        for idx, row in df_item[['title', 'item_id']].iterrows():
+            query = self.get_map_query(row['title'])
+            q.put((row['item_id'], query))
+        
+        if self.n_workers > 1:
+            responses = self.parallel_queries(q)
+        else:
+            responses = self.sequential_queries(q)
+        
+        URI_mapping = {}
+        for response in tqdm(responses, desc='Disambiguating query return'):
+            candidate_URIs = []
+            idx, result = response
+            for binding in result['results']['bindings']:
+                URI = binding['book']['value']
+                candidate_URIs.append(URI)
+            
+            expected_URI = f'http://dbpedia.org/resource/{df_item.loc[df_item.item_id == idx]["title"]}'
+            str_matching_result = process.extractOne(expected_URI, candidate_URIs)
+
+            if str_matching_result is not None:
+                URI, _ = str_matching_result
+                URI_mapping[idx] = URI
+
+        df_map = pd.DataFrame({'item_id': df_item['item_id']})
+        df_map.set_index('item_id')
+        df_map['URI'] = df_map['item_id'].apply(lambda id: URI_mapping.get(id))
+
+        return df_map
+    
+    def get_map_query(self, book_title) -> str:
+        book_title = book_title.translate(self._special_chars_map)
+        book_title = book_title.replace(' ', '.*')
+        book_title = '^' + book_title
+   
+        params = {'name_regex': book_title}
+        query = self.map_query_template.substitute(**params)
+        return query
+
+
+   

data_integration/datasets/douban-movie.py

@@ -0,0 +1,75 @@
+import os
+import pandas as pd
+
+from ..dataset import Dataset
+
+class DoubanMovie(Dataset):
+    def __init__(self, input_path, output_path, n_workers=1):
+        super().__init__(input_path, output_path, n_workers)
+        self.dataset_name = "Douban Movie Short"
+
+        self.item_fields_extract = ["Movie_Name_EN", "Movie_Name_CN"]
+        self.user_fields_extract = ["Username"]
+        self.rating_fields_extract = ["ID", "Date", "Star", "Comment", "Like", "Movie_Name_EN", "Username"]
+
+        self.item_fields = {
+            "movie_id": "item_id::string",
+            "Movie_Name_EN": "name_EN::string",
+            "Movie_Name_CN": "name_CN::string"
+        }
+
+        self.user_fields = {
+            "user_id": "user_id::string",
+            "Username": "name::string"
+        }
+
+        self.rating_fields = {
+            "user_id": "user_id::string",
+            "movie_id": "item_id::string",
+            "Star": "rating::number",
+            "Date": "date::string",
+            "Comment": "comment::string",
+            "Like": "like_count::number"
+        }
+
+    def load_item_data(self) -> pd.DataFrame():
+        filename = os.path.join(self.input_path, "DMSC.csv")
+        df = pd.read_csv(filename)
+        
+        df = df[self.item_fields_extract].drop_duplicates().reset_index(drop=True)
+        df['movie_id'] = df.index
+
+        df = df.rename(self.item_fields, axis=1)
+        df = df[self.item_fields.values()]
+        return df
+    
+    def load_user_data(self) -> pd.DataFrame():
+        filename = os.path.join(self.input_path, "DMSC.csv")
+        df = pd.read_csv(filename)
+        
+        df = df[self.user_fields_extract].drop_duplicates().reset_index(drop=True)
+        df['user_id'] = df.index
+
+        df = df.rename(self.user_fields, axis=1)
+        df = df[self.user_fields.values()]
+        return df
+    
+    def load_rating_data(self) -> pd.DataFrame():
+        filename = os.path.join(self.input_path, "DMSC.csv")
+        df = pd.read_csv(filename)
+
+        item_df = df[self.item_fields_extract].drop_duplicates().reset_index(drop=True)
+        item_df['movie_id'] = item_df.index
+
+        user_df = df[self.user_fields_extract].drop_duplicates().reset_index(drop=True)
+        user_df['user_id'] = user_df.index
+        
+        df = df[self.rating_fields_extract].drop_duplicates().reset_index(drop=True)
+        
+        df = df.join(item_df.set_index('Movie_Name_EN'), on='Movie_Name_EN')
+        df = df.join(user_df.set_index('Username'), on='Username')
+
+        df = df.rename(self.rating_fields, axis=1)
+        df = df[self.rating_fields.values()]
+        return df
+