FranKGraphBench 0.0.1a0__py2.py3-none-any.whl

data_integration/datasets/worker.py

@@ -0,0 +1,31 @@
+from threading import Thread
+import queue
+
+class Worker(Thread):
+    def __init__(self, request_queue, query_function, pbar=None):
+        """
+        Parameters:
+            request_queue: queue containing requests to be consumed by workers
+            query_function: function to make the query to the endpoint
+            pbar: tqdm progressbar to be updated after each query
+        """
+        super().__init__()
+        self.queue = request_queue
+        self.local_results = []
+        self.query = query_function
+        self.pbar = pbar
+    
+    def run(self):
+        while True:
+            try:
+                idx, query = self.queue.get(block=False)
+                response = self.query(query)
+                self.local_results.append((idx, response))
+                if self.pbar is not None:
+                    self.pbar.update(n=1)
+                self.queue.task_done()
+            except queue.Empty:
+                break
+            except Exception as e:
+                print(f'Exception:')
+                print(e)    

data_integration/datasets/yelp.py

@@ -0,0 +1,156 @@
+import os
+import json
+import pandas as pd
+from collections import defaultdict
+
+from ..dataset import Dataset
+
+class Yelp(Dataset):
+    def __init__(self, input_path, output_path, n_workers=1):
+        super().__init__(input_path, output_path, n_workers)
+        self.dataset_name = "Yelp"
+
+        self.categories_separator = ", "
+        self.friends_separator = ", "
+        self.elites_separator = ","
+
+        self.string_list_separator = "::"
+
+        self.item_fields = {
+            "business_id": "item_id::string",
+            "name": "name::string",
+            "address": "address::string",
+            "city": "city::string",
+            "state": "state::string",
+            "postal_code": "postal_code::string",
+            "latitude": "latitute::number",
+            "longitude": "longitude::number",
+            "stars": "stars::number",
+            "review_count": "review_count::number",
+            "is_open": "is_open::number",
+            "attributes": "attributes::string_list",
+            "categories": "categories::string_list",
+        }
+
+        self.user_fields = {
+            "user_id": "user_id::string",
+            "name": "name::string",
+            "review_count": "review_count::number",
+            "yelping_since": "yelping_since::string",
+            "useful": "useful_count::number",
+            "funny": "funny_count::number",
+            "cool": "cool_count::number",
+            "elite": "elite_years::string_list",
+            "fans": "fans_count::number",
+            "average_stars": "average_stars::number",
+            "compliment_hot": "compliment_hot_count::number",
+            "compliment_more": "compliment_more_count::number",
+            "compliment_profile": "compliment_cute_count::number",
+            "compliment_list": "compliment_list_count::number",
+            "compliment_note": "compliment_note_count::number",
+            "compliment_plain": "compliment_plain_count::number",
+            "compliment_cool": "compliment_cool_count::number",
+            "compliment_funny": "compliment_funny_count::number",
+            "compliment_writer": "compliment_writer_count::number",
+            "compliment_photos": "compliment_photos_count::number"
+        }
+
+        self.rating_fields = {
+            "user_id": "user_id::string",
+            "business_id": "item_id::string",
+            "stars": "rating::number",
+            "useful": "useful_count::number",
+            "funny": "funny_count::number",
+            "cool": "cool_count::number",
+            "text": "text::string",
+            "date": "date::string"
+        }
+
+        self.social_fields = {"user_id": "user1::string", "friend_id": "user2::string"}
+
+    def load_item_data(self) -> pd.DataFrame():
+        filename = os.path.join(self.input_path, "yelp_academic_dataset_business.json")
+
+        print(filename)
+        file = open(filename, encoding="utf8")
+        data = []
+        for line in file:
+            data.append(json.loads(line))
+        df = pd.DataFrame(data)
+        file.close()
+
+        attributes = defaultdict(list)
+        for index, row in df['attributes'].dropna().items():
+            for key, value in row.items():
+                if value == 'True':
+                    attributes[index].append(key)
+        df['attributes'] = pd.Series(attributes).apply(lambda x: self.string_list_separator.join(x))
+
+        categories = defaultdict(list)
+        for index, row in df['categories'].dropna().items():
+            for category in row.split(self.categories_separator):
+                categories[index].append(category)
+        df['categories'] = pd.Series(categories).apply(lambda x: self.string_list_separator.join(x))
+
+        df = df.rename(self.item_fields, axis=1)
+        df = df[self.item_fields.values()]
+        return df
+    
+    def load_user_data(self) -> pd.DataFrame:
+        filename = os.path.join(self.input_path, "yelp_academic_dataset_user.json")
+
+        file = open(filename, encoding="utf8")
+        data = []
+        for line in file:
+            data.append(json.loads(line))
+        df = pd.DataFrame(data)
+        file.close()
+
+        elites = defaultdict(list)
+        for index, row in df['elite'].dropna().items():
+            for elite in row.split(self.elites_separator):
+                elites[index].append(elite)
+        df['elite'] = pd.Series(elites).apply(lambda x: self.string_list_separator.join(x))
+
+        df = df.rename(self.user_fields, axis=1)
+        df = df[self.user_fields.values()]
+        return df
+    
+    def load_rating_data(self) -> pd.DataFrame:
+        filename = os.path.join(self.input_path, "yelp_academic_dataset_review.json")
+
+        file = open(filename, encoding="utf8")
+        data = []
+        for line in file:
+            data.append(json.loads(line))
+        df = pd.DataFrame(data)
+        file.close()
+        df = df.rename(self.rating_fields, axis=1)
+        df = df[self.rating_fields.values()]
+        return df
+    
+    def load_social_data(self) -> pd.DataFrame:
+        filename = os.path.join(self.input_path, "yelp_academic_dataset_user.json")
+
+        file = open(filename, encoding="utf8")
+        data = []
+        for line in file:
+            data.append(json.loads(line))
+        user_df = pd.DataFrame(data)
+        file.close()
+
+        friends = defaultdict(list)
+        for index, row in user_df['friends'].dropna().items():
+            for friend in row.split(self.friends_separator):
+                friends[index].append(friend)
+
+        social = {'user_id': [], 'friend_id': []}
+        for index, friends_list in friends.items():
+            for friend in friends_list:
+                social["user_id"].append(user_df['user_id'].iloc[index])
+                social["friend_id"].append(friend)
+        
+        df = pd.DataFrame(social)
+        df = df.rename(self.social_fields, axis=1)
+        df = df[self.social_fields.values()]
+        return df

framework/__init__.py

@@ -0,0 +1,17 @@
+import argparse
+from framework.experiment import run
+
+__version__ = "0.0.1-alpha"
+__author__ = "Paulo do Carmo and Alvaro Lopes"
+
+def framework():
+    parser = argparse.ArgumentParser(
+        description="Script to run framework for reproducible experiment."
+    )
+
+    parser.add_argument('-c', '--config',  type=str, required=True, help='.yml config filepath to setup experiment')
+
+    args = parser.parse_args()
+    config_path = args.config
+    print(f'Running experiment...')
+    run(config_path)

framework/__main__.py

@@ -0,0 +1,4 @@
+from . import framework
+
+if __name__ == '__main__':
+    framework()

framework/dataloader/dataloader.py

@@ -0,0 +1,52 @@
+from ..utils import get_optional_argument
+from .graph.graph import Graph
+from .preprocess.methods import apply_method
+from .edge_splitter.edge_splitter import EdgeSplitter
+from .dataset.dataset import Dataset
+
+"""
+    Loads, preprocess, filter and split dataset, then return a dataset
+    :arguments: 
+        dataset: dict specifying the dataset configuration 
+    :returns: Networkx graph (train, val, test)
+"""
+def load(**data_config):
+    # G = Graph(**data_config)
+    G = Graph()
+    G.build(**data_config)
+    return G
+
+def preprocess(G, methods: list):
+    for method in methods:
+        apply_method(G, **method)
+
+def split(G, **split_config):
+    seed = get_optional_argument(split_config, 'seed', 42)
+    edge_splitter_test = EdgeSplitter(G, seed=seed)
+    for (G_train, ratings_test) in edge_splitter_test.split(**split_config['test']):
+        dataset = Dataset()
+        dataset.set_test_data(ratings_test)
+        print(f'\tDisturbed graph info after splitting test data: {G_train.info()}')
+        print(f'\tNumber of test instances: {len_instances(ratings_test)}')
+
+        if get_optional_argument(split_config, 'validation', False):
+            if split_config['test']['method'] == 'k_fold' and split_config['validation']['method'] == 'k_fold':
+                raise ValueError("Validation split does not support k_fold method.")
+            
+            edge_splitter_val = EdgeSplitter(G_train, seed=seed)
+            for G_train, ratings_val in edge_splitter_val.split(**split_config['validation']):
+                dataset.set_val_data(ratings_val)
+                print(f'\tGraph info after splitting validation data: {G_train.info()}')
+                print(f'\tNumber of validation instances: {len_instances(ratings_val)}')
+
+        ratings_train = G_train.get_ratings_with_labels()
+        dataset.set_train_data(G_train, ratings_train)
+        print(f'Number of training instances: {len_instances(ratings_train)}')
+
+        yield dataset
+
+def len_instances(data):
+    return sum([len(items) for items in data.values()])
+
+
+    

framework/dataloader/dataset/dataset.py

@@ -0,0 +1,38 @@
+class Dataset():
+    def __init__(self):
+        self.G_train = None
+        self.ratings_train = None # {user: [(item1, rating), ... } Unordered items
+        self.ratings_test = None
+        self.has_val = False
+        self.ratings_val = None
+    
+    def set_train_data(self, G, ratings):
+        self.G_train = G
+        self._sort_ratings(ratings)
+        self.ratings_train = ratings
+    
+    def set_test_data(self, ratings):
+        self._sort_ratings(ratings)
+        self.ratings_test = ratings
+
+    def set_val_data(self, ratings):
+        self.has_val = True
+        self._sort_ratings(ratings)
+        self.ratings_val = ratings
+
+    def get_train_data(self):
+        return self.G_train, self.ratings_train
+
+    def get_test_data(self):
+        return self.ratings_test
+
+    def get_val_data(self):
+        return self.ratings_val
+    
+    def has_val_data(self):
+        return self.has_val
+    
+    def _sort_ratings(self, ratings):
+        # inplace sorting
+        for u in ratings.keys():
+            ratings[u].sort(reverse=True, key=lambda x: x[1])

framework/dataloader/edge_splitter/edge_splitter.py

@@ -0,0 +1,163 @@
+from typing import Tuple, Dict, List
+
+import random
+import math
+from copy import deepcopy
+from collections import defaultdict
+
+from ..graph.graph import Graph
+from ..graph.node import UserNode, ItemNode
+
+import numpy as np
+from sklearn.model_selection import KFold
+
+
+class EdgeSplitter():
+    def __init__(self, G: Graph, seed : int = 42):
+        self.G = deepcopy(G)
+        self.random_state = seed
+        random.seed(seed)
+        np.random.seed(seed)
+
+        self.supported_methods = [
+            'random_by_ratio',
+            'timestamp_by_ratio',
+            'fixed_timestamp',
+            'k_fold'
+        ]
+    
+    def split(self, **config) -> Tuple[Graph, Dict[UserNode, List[Tuple[ItemNode, float]]]]:
+        method = config['method']
+        if method not in self.supported_methods:
+            raise ValueError(f'Invalid split method provided. So far, the supported ones are:\n{self.supported_methods}')
+
+        if method in ['random_by_ratio', 'timestamp_by_ratio']:
+            p = config['p']
+            level = config["level"]
+            if p <= 0 or p >= 1:
+                raise ValueError('The parameter p must be in the interval (0,1)')
+
+            if level not in ['user', 'global']:
+                raise ValueError('Invalid level parameter for splitting. Choose between user and global level.')
+            
+            print(f'Splitting data using {method} method at {level} level, where p={p} ...')
+            
+            if method == 'random_by_ratio':
+                test = self._random_by_ratio(p, level)
+            elif method == 'timestamp_by_ratio':
+                test = self._timestamp_by_ratio(p, level)
+            
+            yield self._extract_dataset(test)
+
+        elif method == 'fixed_timestamp':
+            print(f'Splitting data using {method} method, where timestamp={config["timestamp"]} ...')
+
+            test = []
+            for (u,v) in self.G.get_rating_edges():
+                data = self.G.get_edge_data(u,v)
+                if data['timestamp'] > config['timestamp']:
+                    test.append((u,v))
+            yield self._extract_dataset(test)
+
+        elif method == 'k_fold':
+            k = config['k']
+            if k < 2:
+                raise ValueError('k parameter for K-fold must be at least 2.')
+            
+            level = config['level']
+            if level not in ['user', 'global']:
+                raise ValueError('Invalid level parameter for splitting. Choose between user and global level.')
+            
+            
+            for test in self._kfold(k, level):
+                yield self._extract_dataset(test)
+
+    def _extract_dataset(self, test):
+        # Returns Graph, {user: [(item1, rating1), (item2, rating2) ...]}
+        ratings = defaultdict(list)
+        for (u, v) in test:
+            data = self.G.get_edge_data(u,v)
+            self.G.remove_edge(u,v)
+            ratings[u].append((v, data['rating']))
+
+        return self.G, ratings 
+    
+    def _random_by_ratio(self, p: float, level: str) -> np.array:
+        if level == 'global':
+            ratings = np.array(list(self.G.get_rating_edges()))
+            np.random.shuffle(ratings)
+            n_test = math.ceil(ratings.shape[0] * p)
+
+            return ratings[-n_test:]
+        elif level == 'user':
+            test = []
+            for user, items in self.G.rating_edges.items():
+                # Number of test items for user
+                items = np.array(list(items))
+                np.random.shuffle(items)
+                n_test = math.ceil(items.shape[0] * p)
+                test += [(user, item) for item in items[-n_test:]]
+
+            return np.array(test)
+        
+    def _timestamp_by_ratio(self, p: float, level: str) -> np.array:
+        if level == 'global':
+            ratings = list(self.G.get_rating_edges())
+            ratings = self._sort_by_timestamp(ratings)
+            ratings = np.array(ratings)
+            n_test = math.ceil(ratings.shape[0] * p)
+
+            return ratings[-n_test:]
+        elif level == 'user':
+            test = []
+            for user, items in self.G.rating_edges.items():
+                ratings = [(user, item) for item in items]
+                ratings = self._sort_by_timestamp(ratings)
+                n_test = math.ceil(len(ratings) * p)    
+                test += ratings[-n_test:]
+
+            return np.array(test) 
+        
+    def _kfold(self, k : int, level : str):
+        G_Copy = deepcopy(self.G)
+        
+        if level == 'global':
+            ratings = list(self.G.get_rating_edges())
+            ratings = np.array(ratings)
+            np.random.shuffle(ratings)
+            kfold = KFold(n_splits=k)
+            for _, test_index in kfold.split(ratings):
+                yield ratings[test_index]
+                self.G = deepcopy(G_Copy)
+
+        elif level == 'user':
+            kfold_users, ratings_per_user = [], []
+            for user, items in self.G.rating_edges.items():
+                items = np.array(list(items))
+                np.random.shuffle(items)
+                kfold = KFold(n_splits=k).split(items)
+                kfold_users.append(kfold)
+                edges = np.array([(user, item) for item in items])
+                ratings_per_user.append(edges)
+
+            for _ in range(k):
+                test = []
+                for idx, kfold in enumerate(kfold_users):
+                    _, test_index = next(kfold)
+                    test_ratings = ratings_per_user[idx][test_index].tolist()
+                    test += test_ratings
+                test = np.array(test)
+                yield test
+                self.G = deepcopy(G_Copy)
+
+    def _sort_by_timestamp(self, ratings: list) -> list:
+        # globally sort the ratings by timestamp
+        f = lambda edge: self.G.get_edge_data(edge[0], edge[1])['timestamp']
+        return sorted(ratings, key=f)
+
+
+
+
+
+
+