opsci-toolbox 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

opsci_toolbox/apis/reddit.py

@@ -2,214 +2,53 @@ import praw
 import datetime 
 import pandas as pd
 from tqdm import tqdm
+import time
 
-def parse_author(author):
-    print(author)
-    if author:
-        if author.comment_karma:
-            author_comment_karma=author.comment_karma
-        else:
-            author_comment_karma = None
-        if author.created_utc:
-            author_created_utc=datetime.datetime.fromtimestamp(int(author.created_utc)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
-        else:
-            author_created_utc = datetime.datetime(1970,1,1,0,0,0)
-        if author.icon_img:
-            author_icon_img=author.icon_img
-        else:
-            author_icon_img = None
-        if author.id:
-            author_id=author.id
-        else:
-            author_id = None
-        if author.is_employee:
-            author_is_employee=author.is_employee
-        else:
-            author_is_employee = None
-        if author.is_mod:
-            author_is_mod=author.is_mod
-        else:
-            author_is_mod = None
-
-        if author.is_gold:
-            author_is_gold=author.is_gold
-        else:
-            author_is_gold = None
-
-        if author.link_karma:
-            author_link_karma=author.link_karma
-        else:
-            author_link_karma = None
-
-        if author.name:
-            author_name=author.name
-        else:
-            author_name = None
-        record = (author_id, author_name, author_link_karma, author_comment_karma, author_created_utc, author_icon_img, author_is_employee, author_is_mod, author_is_gold)
-    else:
-        record = (None, None, None, None, None, None, None, None, None)
-    return record
-
-
-def getSubmissions(reddit_client, lst_ids, subreddit_filter, subreddit_items, time_filter):
-    
-    all_records = []
-    for url in tqdm(lst_ids, total=len(lst_ids), desc="Récupération des soumissions"):
-        subreddit = reddit_client.subreddit(str(url))
-
-        sub_record = parse_subreddit(subreddit)
-
-        if subreddit_filter == "top":
-            subreddit_selection = subreddit.top(limit=subreddit_items, time_filter= time_filter)
-        elif subreddit_filter == "hot":
-            subreddit_selection = subreddit.hot(limit=subreddit_items, time_filter= time_filter)
-        elif subreddit_filter == "controversial":
-            subreddit_selection = subreddit.controversial(limit=subreddit_items, time_filter= time_filter)
-        else:
-            subreddit_selection = subreddit.new(limit=subreddit_items, time_filter= time_filter)
-
-        for submission in subreddit_selection:
-            author = submission.author
-            author_record  = parse_author(author)
-            submission_record = parse_submission(submission)    
-
-            record = sub_record + author_record + submission_record
-            all_records.append(record)
-
-            df = pd.DataFrame.from_records(all_records, 
-                                           columns = ["subreddit_id", "subreddit_name", "subreddit_display_name", "subreddit_subscribers", "subreddit_date", "subreddit_description", "subreddit_public_description", "subreddit_over18", 
-                                                      "subreddit_spoilers_enabled", "subreddit_can_assign_user_flair", "subreddit_can_assign_link_flair", "subreddit_lang", "subreddit_active_user_count",
-                                                      "author_id", "author_name", "author_link_karma", "author_comment_karma", "author_created_utc", "author_icon_img", "author_is_employee", "author_is_mod", "author_is_gold",
-                                                      "submission_id", "submission_title", "submission_name", "submission_created_utc", "submission_distinguished", "submission_edited", "submission_is_self", "submission_link_flair_template_id", 
-                                                      "submission_link_flair_text", "submission_locked", "submission_num_comments", "submission_over_18", "submission_permalink", "submission_score", "submission_selftext", "submission_spoiler", 
-                                                      "submission_stickied", "submission_url", "submission_upvote_ratio", "submission_downs", "submission_ups", "submission_num_crossposts", "submission_num_reports", "submission_score", 
-                                                      "submission_total_awards_received", "submission_view_count"]
-            )
-            
-    return df
-
-
-def parse_submission(submission):
-    if submission.id:
-        submission_id=submission.id
-    else:
-        submission_id = None
-    if submission.title:
-        submission_title=submission.title
-    else:
-        submission_title = None
-    if submission.name:
-        submission_name=submission.name
-    else:
-        submission_name = None
-    if submission.created_utc:
-        submission_created_utc=datetime.datetime.fromtimestamp(int(submission.created_utc)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
-    else:
-        submission_created_utc = datetime.datetime(1970,1,1,0,0,0)
-    if submission.distinguished:
-        submission_distinguished=submission.distinguished
-    else:
-        submission_distinguished = None
-    if submission.edited:
-        submission_edited=submission.edited
-    else:
-        submission_edited = None
-    if submission.is_self:
-        submission_is_self=submission.is_self
-    else:
-        submission_is_self = None
-    if submission.link_flair_template_id:
-        submission_link_flair_template_id=submission.link_flair_template_id
-    else:
-        submission_link_flair_template_id = None
-    if submission.link_flair_text:
-        submission_link_flair_text=submission.link_flair_text
-    else:
-        submission_link_flair_text = None
-    if submission.locked:
-        submission_locked=submission.locked
-    else:
-        submission_locked = None
-    if submission.num_comments:
-        submission_num_comments=submission.num_comments
-    else:
-        submission_num_comments = None
-    if submission.over_18:
-        submission_over_18=submission.over_18
-    else:
-        submission_over_18 = None
-    if submission.permalink:
-        submission_permalink=submission.permalink
-    else:
-        submission_permalink = None
-    if submission.score:
-        submission_score=submission.score
-    else:
-        submission_score = None
-    if submission.selftext:
-        submission_selftext=submission.selftext
-    else:
-        submission_selftext = None
-    if submission.spoiler:
-        submission_spoiler=submission.spoiler
-    else:
-        submission_spoiler = None
-    if submission.stickied:
-        submission_stickied=submission.stickied
-    else:
-        submission_stickied = None
-    if submission.upvote_ratio:
-        submission_upvote_ratio=submission.upvote_ratio
-    else:
-        submission_upvote_ratio = None
-    if submission.url:
-        submission_url=submission.url
-    else:
-        submission_url = None
-    if submission.downs:
-        submission_downs=submission.downs
-    else:
-        submission_downs = None
-    if submission.num_crossposts:
-        submission_num_crossposts=submission.num_crossposts 
-    else:
-        submission_num_crossposts = None
-    if submission.num_reports:
-        submission_num_reports=submission.num_reports 
-    else:
-        submission_num_reports = None
-
-    if submission.score:
-        submission_score=submission.score 
-    else:
-        submission_score = None
-
-    if submission.ups:
-        submission_ups=submission.ups 
-    else:
-        submission_ups = None
+def check_limit(reddit_client : praw.Reddit) -> tuple:
+    """
+    Check Reddit Client rate limit and wait if necessary.
 
-    if submission.total_awards_received:
-        submission_total_awards_received=submission.total_awards_received 
-    else:
-        submission_total_awards_received = None
+    Args:
+        reddit_client (praw.Reddit): current reddit client
 
-    if submission.view_count:
-        submission_view_count=submission.view_count 
-    else:
-        submission_view_count = None
+    Returns:
+        tuple containing the following information:
+            - remaining: remaining queries.
+            - reset_timestamp: time before reset.
+            - used: number of sent queries.
 
+    """
+    headers = reddit_client.auth.limits
+    remaining = headers.get('remaining')
+    reset_timestamp = headers.get('reset_timestamp')
+    used = headers.get('used')
+
+    if remaining and reset_timestamp :
+        if remaining <= 10:
+            # Calculate the time to wait until reset
+            current_time = time.time()
+            wait_time = reset_timestamp - current_time
+
+            if wait_time > 0:
+                # Convert wait_time to seconds and wait
+                print(f"Waiting for {wait_time:.2f} seconds until the next reset...")
+                time.sleep(wait_time)
+            else:
+                print("Reset time is in the past. No need to wait.")
+        # else:
+        #     print(f"{remaining} requests remaining. No need to wait.")
+    else : 
+        print("Missing required header information. Cannot determine wait time.")
+
+    return remaining, reset_timestamp, used
 
-    record = (submission_id, submission_title, submission_name, submission_created_utc, submission_distinguished, submission_edited, submission_is_self, submission_link_flair_template_id, submission_link_flair_text, submission_locked, submission_num_comments, submission_over_18, submission_permalink, submission_score, submission_selftext, submission_spoiler, submission_stickied, submission_url, submission_upvote_ratio, submission_downs, submission_ups, submission_num_crossposts, submission_num_reports, submission_score, submission_total_awards_received, submission_view_count)
-    return record
-
-    
 
 def get_subreddit_info(reddit_client : praw.Reddit, lst_ids: list) -> pd.DataFrame:
     """
     Retrieves information about subreddits based on a list of subreddit IDs.
 
     Args:
+        reddit_client (praw.Reddit): current reddit client
         lst_ids (list): A list of subreddit IDs.
 
     Returns:
@@ -228,172 +67,341 @@ def get_subreddit_info(reddit_client : praw.Reddit, lst_ids: list) -> pd.DataFra
     """
     all_records = []
     for reddit_id in lst_ids:
+        remaining, reset_timestamp, used = check_limit(reddit_client)
         subreddit = reddit_client.subreddit(str(reddit_id))
-        record = parse_subreddit(subreddit)
+        record = parse_subreddit(reddit_client, subreddit)
   
         all_records.append(record)
 
     df = pd.DataFrame.from_records(all_records, columns=["subreddit_id", "subreddit_name", "subreddit_display_name", "subreddit_subscribers", "subreddit_date", "subreddit_description", "subreddit_public_description", "subreddit_over18", "subreddit_spoilers_enabled", "subreddit_can_assign_user_flair", "subreddit_can_assign_link_flair", "subreddit_lang", "subreddit_active_user_count"])
     return df
 
-def parse_subreddit(subreddit):
-    if subreddit.id:
-        subreddit_id = subreddit.id
-    else:
-        subreddit_id = None
+
+def getSubmissions(reddit_client : praw.Reddit, sub_id : str, subreddit_filter : str, subreddit_items : int, time_filter : str) -> pd.DataFrame:
+    """
+    Retrieves submission from a subreddit ID.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        sub_id (str): a subreddit ID.
+        subreddit_filter (str): the filter to apply to the subreddit (top, hot, new, controversial).
+        subreddit_items (int): the number of items to retrieve. None to retrieve all items.
+        time_filter (str): the time filter to apply to the subreddit (hour, day, week, month, year, all).
+
+    Returns:
+        pd.DataFrame: A DataFrame containing submissions metadata.
+    """
+    
+    all_records = []
+    # for sub_id in tqdm(lst_ids, total=len(lst_ids), desc="Récupération des soumissions"):
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+    subreddit = reddit_client.subreddit(str(sub_id))
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+    if not vars(subreddit).get('_fetched'):
+        subreddit._fetch()
+        remaining, reset_timestamp, used = check_limit(reddit_client)
+
+    sub_record = parse_subreddit(reddit_client, subreddit)
+
+    if subreddit_filter == "top":
+        subreddit_selection = subreddit.top(limit=subreddit_items, time_filter= time_filter)
+    elif subreddit_filter == "hot":
+        subreddit_selection = subreddit.hot(limit=subreddit_items)
+    elif subreddit_filter == "controversial":
+        subreddit_selection = subreddit.controversial(limit=subreddit_items, time_filter= time_filter)
+    elif subreddit_filter == "new":
+        subreddit_selection = subreddit.new(limit=subreddit_items)
+    elif subreddit_filter == "gilded":
+        subreddit_selection = subreddit.gilded(limit=subreddit_items)
+    elif subreddit_filter == "rising":
+        subreddit_selection = subreddit.rising(limit=subreddit_items)
+    else:
+        return pd.DataFrame()
         
-    if subreddit.name:
-        name = subreddit.name
-    else:
-        name = None
+    
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+    for i, submission in enumerate(subreddit_selection):
+        try:
+            remaining, reset_timestamp, used = check_limit(reddit_client)
+            if not vars(submission).get('_fetched'):
+                submission._fetch()
+                remaining, reset_timestamp, used = check_limit(reddit_client)
 
-    if subreddit.display_name:
-        display_name = subreddit.display_name
-    else:
-        display_name = None
+            author = submission.author
+            author_record  = parse_author(reddit_client, author)
+            submission_record = parse_submission(reddit_client, submission)    
 
-    if subreddit.subscribers:
-        subscribers = subreddit.subscribers
-    else:
-        subscribers = None    
+            record = sub_record + author_record + submission_record
+            all_records.append(record)
+           
+        except Exception as e:
+            pass
+            print(e)
+
+        df = pd.DataFrame.from_records(all_records, 
+                                        columns = ["subreddit_id", "subreddit_name", "subreddit_display_name", "subreddit_subscribers", "subreddit_date", "subreddit_description", "subreddit_public_description", "subreddit_over18", 
+                                                    "subreddit_spoilers_enabled", "subreddit_can_assign_user_flair", "subreddit_can_assign_link_flair", "subreddit_lang", "subreddit_active_user_count",
+                                                    "author_id", "author_name", "author_link_karma", "author_comment_karma", "author_created_utc", "author_icon_img", "author_is_employee", "author_is_mod", "author_is_gold",
+                                                    "submission_id", "submission_title", "submission_name", "submission_created_utc", "submission_distinguished", "submission_edited", "submission_is_self", "submission_link_flair_template_id", 
+                                                    "submission_link_flair_text", "submission_locked", "submission_num_comments", "submission_over_18", "submission_permalink", "submission_selftext", "submission_spoiler", 
+                                                    "submission_stickied", "submission_url", "submission_upvote_ratio", "submission_downs", "submission_ups", "submission_num_crossposts", "submission_num_reports", "submission_score", 
+                                                    "submission_total_awards_received", "submission_view_count"]
+        )
 
-    if subreddit.created_utc:
-        date=datetime.datetime.fromtimestamp(int(subreddit.created_utc)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
-    else:
-        date = datetime.datetime(1970,1,1,0,0,0)
-    if subreddit.description:
-        description=subreddit.description
-    else:
-        description=None
-    if subreddit.public_description:
-        public_description = subreddit.public_description
-    else:
-        public_description = None
-    if subreddit.over18:
-        over18 = subreddit.over18
-    else:
-        over18 = None
-    if subreddit.lang:
-        lang = subreddit.lang
-    else:
-        lang = None
-    if subreddit.active_user_count:
-        active_user_count = subreddit.active_user_count
-    else:
-        active_user_count = None
+    return df
 
-    if subreddit.spoilers_enabled:
-        spoilers_enabled = subreddit.spoilers_enabled
-    else:
-        spoilers_enabled = None
 
-    if subreddit.can_assign_user_flair:
-        can_assign_user_flair = subreddit.can_assign_user_flair
-    else:
-        can_assign_user_flair = None
 
-    if subreddit.can_assign_link_flair:
-        can_assign_link_flair = subreddit.can_assign_link_flair     
-    else:
-        can_assign_link_flair = None
+def getComments(reddit_client : praw.Reddit, submission_id : str) -> pd.DataFrame:
+    """
+    Retrieves all comments from a submission ID.
 
-    record = (subreddit_id, name, display_name, subscribers, date, description, public_description, over18, spoilers_enabled, can_assign_user_flair, can_assign_link_flair, lang, active_user_count)
-    return record
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        submission_id (str): a submission ID.
 
+    Returns:
+        pd.DataFrame: A DataFrame containing comments metadata.
+    
+    """
+    
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+    submission = reddit_client.submission(str(submission_id))
+    if not vars(submission).get('_fetched'):
+        submission._fetch()
+
+    submission.comments.replace_more(limit=None)
+    remaining, reset_timestamp, used = check_limit(reddit_client)
 
-def getComments(reddit, lst_ids):
     all_records = []
-    for i, submission_id in tqdm(enumerate(lst_ids), total=len(lst_ids), desc="Récupération des commentaires"):
-        submission = reddit.submission(str(submission_id))
-        for comment in submission.comments.list():
-            record = (submission_id,) + parse_comments(comment)
-            
-            all_records.append(record)
+    for comment in tqdm(submission.comments.list(), total=len(submission.comments.list()), desc="Récupération des commentaires"):
+        remaining, reset_timestamp, used = check_limit(reddit_client)
+        record = (submission_id,) + parse_comments(reddit_client, comment)
+        all_records.append(record)
 
-    df = pd.DataFrame.from_records(all_records, columns=["submission_id", "comment_id", "comment_body", "comment_date", "comment_distinguished", "comment_edited", "comment_is_submitter", "comment_link_id", "comment_parent_id", "comment_permalink", 
-                                                         "comment_controversiality", "comment_depth", "comment_downs", "comment_likes", "comment_num_reports", "comment_score", "comment_total_awards_received", "comment_ups",
+    df = pd.DataFrame.from_records(all_records, columns=["submission_id", "comment_id", "comment_body", "comment_date", "comment_distinguished", "comment_is_submitter", "comment_link_id", "comment_parent_id", "comment_permalink", 
+                                                         "comment_controversiality", "comment_depth", "comment_score", "comment_total_awards_received", "comment_ups",
                                                          "author_id", "author_name", "author_link_karma", "author_comment_karma", "author_created_utc", "author_icon_img", "author_is_employee", "author_is_mod", "author_is_gold"
                                                          ])
 
     return df
 
-def parse_comments(comment):
-    if comment.id:
-        comment_id=comment.id
-    else:
-        comment_id = None
-    if comment.body:
-        comment_body=comment.body
-    else:
-        comment_body = None
-    if comment.created_utc:
-        comment_date=datetime.datetime.fromtimestamp(int(comment.created_utc)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
-    else:
-        comment_date = None
-    if comment.distinguished:
-        comment_distinguished=comment.distinguished
-    else:
-        comment_distinguished = None
-    if comment.edited:
-        comment_edited=comment.edited
-    else:
-        comment_edited = None
-    if comment.is_submitter:
-        comment_is_submitter=comment.is_submitter
-    else:
-        comment_is_submitter = None
+def get_top_level_comments(reddit_client : praw.Reddit, submission_id : str) -> pd.DataFrame:
+    """
+    Retrieves top level comments from a submission ID.
 
-    if comment.link_id:
-        comment_link_id=comment.link_id
-    else:
-        comment_link_id = None
-    if comment.parent_id:
-        comment_parent_id=comment.parent_id
-    else:
-        comment_parent_id = None
-    if comment.permalink:
-        comment_permalink=comment.permalink
-    else:
-        comment_permalink = None
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        submission_id (str): a submission ID.
 
-    if comment.controversiality:
-        comment_controversiality=comment.controversiality
-    else:
-        comment_controversiality = None
-    if comment.depth:
-        comment_depth=comment.depth
-    else:
-        comment_depth = None
-    if comment.downs:
-        comment_downs=comment.downs
-    else:
-        comment_downs = None
-    if comment.likes:
-        comment_likes=comment.likes
-    else:
-        comment_likes = None
-    if comment.num_reports:
-        comment_num_reports=comment.num_reports
-    else:
-        comment_num_reports = None
-    if comment.score:
-        comment_score=comment.score
-    else:
-        comment_score = None
-    if comment.total_awards_received:
-        comment_total_awards_received=comment.total_awards_received
-    else:
-        comment_total_awards_received = None
-    if comment.ups:
-        comment_ups=comment.ups
+    Returns:
+        pd.DataFrame: A DataFrame containing comments metadata.
+    
+    """
+
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+    submission = reddit_client.submission(str(submission_id))
+    if not vars(submission).get('_fetched'):
+        submission._fetch()
+
+    submission.comments.replace_more(limit=None)
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+
+    all_records = []
+    for comment in tqdm(submission.comments, total=len(submission.comments), desc="Récupération des commentaires"):
+        remaining, reset_timestamp, used = check_limit(reddit_client)
+        record = (submission_id,) + parse_comments(reddit_client, comment)
+        all_records.append(record)
+
+    df = pd.DataFrame.from_records(all_records, columns=["submission_id", "comment_id", "comment_body", "comment_date", "comment_distinguished", "comment_is_submitter", "comment_link_id", "comment_parent_id", "comment_permalink", 
+                                                         "comment_controversiality", "comment_depth", "comment_score", "comment_total_awards_received", "comment_ups",
+                                                         "author_id", "author_name", "author_link_karma", "author_comment_karma", "author_created_utc", "author_icon_img", "author_is_employee", "author_is_mod", "author_is_gold"
+                                                         ])
+
+    return df
+
+def parse_author(reddit_client : praw.Reddit, author : praw.models.Redditor) -> tuple:
+    """
+    Parses a Reddit author object and extracts relevant information.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        author (praw.models.Redditor): The Reddit author object.
+
+    Returns:
+        tuple: A tuple containing the following information about the author:
+            - author_id: The ID of the author.
+            - author_name: The name of the author.
+            - author_link_karma: The link karma of the author.
+            - author_comment_karma: The comment karma of the author.    
+            - author_created_utc: The creation date of the author.
+            - author_icon_img: The icon image of the author.
+            - author_is_employee: Indicates if the author is an employee.
+            - author_is_mod: Indicates if the author is a moderator.
+            - author_is_gold: Indicates if the author has Reddit Gold.
+    """
+
+    if author:
+        if not vars(author).get('_fetched'):
+            remaining, reset_timestamp, used = check_limit(reddit_client)
+            author._fetch()
+        author_comment_karma= vars(author).get("comment_karma", None)
+
+        author_created_utc= vars(author).get("created_utc", None)
+        if author_created_utc:
+            author_created_utc = datetime.datetime.fromtimestamp(int(author_created_utc)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
+        else:
+            author_created_utc = datetime.datetime(1970,1,1,0,0,0)
+        
+        author_icon_img= vars(author).get("icon_img", None)
+        author_is_employee= vars(author).get("is_employee", None)
+        author_is_mod= vars(author).get("is_mod", None)
+        author_is_gold= vars(author).get("is_gold", None)
+        author_link_karma= vars(author).get("link_karma", None)
+        author_name= vars(author).get("name", None)
+        author_id= vars(author).get("id", None)
+
+        record = (author_id, author_name, author_link_karma, author_comment_karma, author_created_utc, author_icon_img, author_is_employee, author_is_mod, author_is_gold)
     else:
-        comment_ups = None
-    if comment.author:
-        author_record = parse_author(comment.author)
+        record = (None, None, None, None, None, None, None, None, None)
+    return record
+
+def parse_submission(reddit_client : praw.Reddit, submission : praw.models.Submission) -> tuple:
+    """
+    Parses a Reddit submission object and extracts relevant information.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        submission (praw.models.Submission): The Reddit submission object.
+
+    Returns:
+        tuple: A tuple containing information about the submission.
+    """
+
+    if submission :
+        if not vars(submission).get('_fetched'): 
+            remaining, reset_timestamp, used = check_limit(reddit_client)
+            submission._fetch()
+        submission_id= vars(submission).get("id", None)
+        submission_title= vars(submission).get("title", None)
+        submission_name= vars(submission).get("name", None)
+        submission_created_utc= vars(submission).get("created_utc", None)
+        if submission_created_utc:
+            submission_created_utc = datetime.datetime.fromtimestamp(int(submission_created_utc)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
+        else:
+            submission_created_utc = datetime.datetime(1970,1,1,0,0,0)
+        submission_distinguished= vars(submission).get("distinguished", None)
+        submission_edited= vars(submission).get("edited", None)
+        submission_is_self= vars(submission).get("is_self", None)
+        submission_link_flair_template_id= vars(submission).get("link_flair_template_id", None)
+        submission_link_flair_text= vars(submission).get("link_flair_text", None)
+        submission_locked= vars(submission).get("locked", None)
+        submission_num_comments= vars(submission).get("num_comments", None)
+        submission_over_18= vars(submission).get("over_18", None)
+        submission_permalink= vars(submission).get("permalink", None)
+        submission_selftext= vars(submission).get("selftext", None)
+        submission_spoiler= vars(submission).get("spoiler", None)
+        submission_stickied= vars(submission).get("stickied", None)
+        submission_upvote_ratio= vars(submission).get("upvote_ratio", None)
+        submission_url= vars(submission).get("url", None)
+        submission_downs= vars(submission).get("downs", None)
+        submission_num_crossposts= vars(submission).get("num_crossposts", None)
+        submission_num_reports= vars(submission).get("num_reports", None)
+        submission_score= vars(submission).get("score", None)
+        submission_total_awards_received= vars(submission).get("total_awards_received", None)
+        submission_view_count= vars(submission).get("view_count", None)
+        submission_ups= vars(submission).get("ups", None)
+        record = (submission_id, submission_title, submission_name, submission_created_utc, submission_distinguished, submission_edited, submission_is_self, submission_link_flair_template_id, 
+                  submission_link_flair_text, submission_locked, submission_num_comments, submission_over_18, submission_permalink, submission_selftext, submission_spoiler, 
+                  submission_stickied, submission_url, submission_upvote_ratio, submission_downs, submission_ups, submission_num_crossposts, submission_num_reports, submission_score, 
+                  submission_total_awards_received, submission_view_count)
+    else:
+        record = (None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None)
+    return record 
+
+
+def parse_subreddit(reddit_client : praw.Reddit, subreddit : praw.models.Subreddit) -> tuple:
+    """
+    Parses a Reddit subreddit object and extracts relevant information.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        subreddit (praw.models.Subreddit): The Reddit subreddit object.
+
+    Returns:
+        tuple: A tuple containing information about the subreddit.
+    """
+
+    if subreddit:
+        if not vars(subreddit).get('_fetched'):
+            remaining, reset_timestamp, used = check_limit(reddit_client)
+            subreddit._fetch()
+        subreddit_id= vars(subreddit).get("id", None)
+        name = vars(subreddit).get("name", None)
+        display_name = vars(subreddit).get("display_name", None)
+        subscribers = vars(subreddit).get("subscribers", None)
+        subscribers = vars(subreddit).get("subscribers", None)
+        date = vars(subreddit).get("created_utc", None)
+        if date:
+            date=datetime.datetime.fromtimestamp(int(date)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
+        else:
+            date = datetime.datetime(1970,1,1,0,0,0)
+        description = vars(subreddit).get("description", None)
+        public_description = vars(subreddit).get("public_description", None)
+        over18 = vars(subreddit).get("over18", None)
+        spoilers_enabled = vars(subreddit).get("spoilers_enabled", None)
+        can_assign_user_flair = vars(subreddit).get("can_assign_user_flair", None)
+        can_assign_link_flair = vars(subreddit).get("can_assign_link_flair", None)
+        lang = vars(subreddit).get("lang", None)
+        active_user_count = vars(subreddit).get("active_user_count", None)
+
+        record = (subreddit_id, name, display_name, subscribers, date, description, public_description, over18, spoilers_enabled, can_assign_user_flair, can_assign_link_flair, lang, active_user_count)
+
     else:
-        author_record = (None, None, None, None, None, None, None, None, None)
+        record = (None, None, None, None, None, None, None, None, None, None, None, None, None)
+    return record
 
-    record = (comment_id, comment_body, comment_date, comment_distinguished, comment_edited, comment_is_submitter, comment_link_id, comment_parent_id, comment_permalink, comment_controversiality, comment_depth, comment_downs, comment_likes, comment_num_reports, comment_score, comment_total_awards_received, comment_ups) + author_record
+def parse_comments(reddit_client : praw.Reddit, comment : praw.models.Comment) -> tuple:
+    """
+    Parses a Reddit comment object and extracts relevant information.
 
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        comment (praw.models.Comment): The Reddit comment object.
+
+    Returns:
+        tuple: A tuple containing information about the comment.
+    """
+
+    if comment:
+        if not vars(comment).get('_fetched'):
+            remaining, reset_timestamp, used = check_limit(reddit_client)
+            comment._fetch()
+        comment_id = vars(comment).get("id", None)
+        comment_body = vars(comment).get("body", None)
+        comment_date = vars(comment).get("created_utc", None)
+        if comment_date:
+            comment_date = datetime.datetime.fromtimestamp(int(comment_date)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
+        else:
+            comment_date = datetime.datetime(1970,1,1,0,0,0)
+        comment_distinguished = vars(comment).get("distinguished", None)
+        # comment_edited = vars(comment).get("edited", None)
+        comment_is_submitter = vars(comment).get("is_submitter", None)
+        comment_link_id = vars(comment).get("link_id", None)
+        comment_parent_id = vars(comment).get("parent_id", None)
+        comment_permalink = vars(comment).get("permalink", None)
+        comment_controversiality = vars(comment).get("controversiality", None)
+        comment_depth = vars(comment).get("depth", None)
+        # comment_downs = vars(comment).get("downs", None)
+        # comment_likes = vars(comment).get("likes", None)
+        # comment_num_reports = vars(comment).get("num_reports", None)
+        comment_score = vars(comment).get("score", None)
+        comment_total_awards_received = vars(comment).get("total_awards_received", None)
+        comment_ups = vars(comment).get("ups", None)
+        author = comment.author
+        author_record = parse_author(reddit_client, author)
+        record = (comment_id, comment_body, comment_date, comment_distinguished, comment_is_submitter, comment_link_id, comment_parent_id, comment_permalink, comment_controversiality, comment_depth, comment_score, comment_total_awards_received, comment_ups) + author_record    
+    else:
+        record = (None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None)  
     return record