opsci-toolbox 0.0.13__tar.gz → 0.0.14__tar.gz

{opsci_toolbox-0.0.13 → opsci_toolbox-0.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: opsci_toolbox
-Version: 0.0.13
+Version: 0.0.14
 Summary: a complete toolbox
 Home-page: UNKNOWN
 Author: Erwan Le Nagard

{opsci_toolbox-0.0.13 → opsci_toolbox-0.0.14}/opsci_toolbox/apis/rapidapi_helpers.py

@@ -1,5 +1,4 @@
 import requests
-from datetime import datetime
 import pandas as pd
 from tqdm import tqdm
 import re
@@ -495,7 +494,7 @@ def parse_list_entries(jsonl_data: list)-> pd.DataFrame:
         protected = data.get("protected", False)
         verified = data.get("verified", False)
         verified_type = data.get("verified_type", "")
-        entities = data.get("entities")
+        entities = data.get("entities", {})
         urls = [url.get("expanded_url","") for url in entities.get('url', {}).get("urls",[])]
         user_mentions = [um.get("screen_name","") for um in entities.get('description', {}).get('user_mentions', [])]
         user_mentions_indices = [um.get("indices",[]) for um in entities.get('description', {}).get('user_mentions', [])]

opsci_toolbox-0.0.14/opsci_toolbox/apis/reddit.py

@@ -0,0 +1,407 @@
+import praw
+import datetime 
+import pandas as pd
+from tqdm import tqdm
+import time
+
+def check_limit(reddit_client : praw.Reddit) -> tuple:
+    """
+    Check Reddit Client rate limit and wait if necessary.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+
+    Returns:
+        tuple containing the following information:
+            - remaining: remaining queries.
+            - reset_timestamp: time before reset.
+            - used: number of sent queries.
+
+    """
+    headers = reddit_client.auth.limits
+    remaining = headers.get('remaining')
+    reset_timestamp = headers.get('reset_timestamp')
+    used = headers.get('used')
+
+    if remaining and reset_timestamp :
+        if remaining <= 10:
+            # Calculate the time to wait until reset
+            current_time = time.time()
+            wait_time = reset_timestamp - current_time
+
+            if wait_time > 0:
+                # Convert wait_time to seconds and wait
+                print(f"Waiting for {wait_time:.2f} seconds until the next reset...")
+                time.sleep(wait_time)
+            else:
+                print("Reset time is in the past. No need to wait.")
+        # else:
+        #     print(f"{remaining} requests remaining. No need to wait.")
+    else : 
+        print("Missing required header information. Cannot determine wait time.")
+
+    return remaining, reset_timestamp, used
+
+
+def get_subreddit_info(reddit_client : praw.Reddit, lst_ids: list) -> pd.DataFrame:
+    """
+    Retrieves information about subreddits based on a list of subreddit IDs.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        lst_ids (list): A list of subreddit IDs.
+
+    Returns:
+        pd.DataFrame: A DataFrame containing the following information for each subreddit:
+            - subreddit_id: The ID of the subreddit.
+            - name: The name of the subreddit.
+            - display_name: The display name of the subreddit.
+            - subscribers: The number of subscribers to the subreddit.
+            - date: The creation date of the subreddit.
+            - description: The description of the subreddit.
+            - public_description: The public description of the subreddit.
+            - over18: Indicates if the subreddit is for users over 18 years old.
+            - spoilers_enabled: Indicates if spoilers are enabled in the subreddit.
+            - can_assign_user_flair: Indicates if users can assign their own flair in the subreddit.
+            - can_assign_link_flair: Indicates if users can assign flair to links in the subreddit.
+    """
+    all_records = []
+    for reddit_id in lst_ids:
+        remaining, reset_timestamp, used = check_limit(reddit_client)
+        subreddit = reddit_client.subreddit(str(reddit_id))
+        record = parse_subreddit(reddit_client, subreddit)
+  
+        all_records.append(record)
+
+    df = pd.DataFrame.from_records(all_records, columns=["subreddit_id", "subreddit_name", "subreddit_display_name", "subreddit_subscribers", "subreddit_date", "subreddit_description", "subreddit_public_description", "subreddit_over18", "subreddit_spoilers_enabled", "subreddit_can_assign_user_flair", "subreddit_can_assign_link_flair", "subreddit_lang", "subreddit_active_user_count"])
+    return df
+
+
+def getSubmissions(reddit_client : praw.Reddit, sub_id : str, subreddit_filter : str, subreddit_items : int, time_filter : str) -> pd.DataFrame:
+    """
+    Retrieves submission from a subreddit ID.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        sub_id (str): a subreddit ID.
+        subreddit_filter (str): the filter to apply to the subreddit (top, hot, new, controversial).
+        subreddit_items (int): the number of items to retrieve. None to retrieve all items.
+        time_filter (str): the time filter to apply to the subreddit (hour, day, week, month, year, all).
+
+    Returns:
+        pd.DataFrame: A DataFrame containing submissions metadata.
+    """
+    
+    all_records = []
+    # for sub_id in tqdm(lst_ids, total=len(lst_ids), desc="Récupération des soumissions"):
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+    subreddit = reddit_client.subreddit(str(sub_id))
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+    if not vars(subreddit).get('_fetched'):
+        subreddit._fetch()
+        remaining, reset_timestamp, used = check_limit(reddit_client)
+
+    sub_record = parse_subreddit(reddit_client, subreddit)
+
+    if subreddit_filter == "top":
+        subreddit_selection = subreddit.top(limit=subreddit_items, time_filter= time_filter)
+    elif subreddit_filter == "hot":
+        subreddit_selection = subreddit.hot(limit=subreddit_items)
+    elif subreddit_filter == "controversial":
+        subreddit_selection = subreddit.controversial(limit=subreddit_items, time_filter= time_filter)
+    elif subreddit_filter == "new":
+        subreddit_selection = subreddit.new(limit=subreddit_items)
+    elif subreddit_filter == "gilded":
+        subreddit_selection = subreddit.gilded(limit=subreddit_items)
+    elif subreddit_filter == "rising":
+        subreddit_selection = subreddit.rising(limit=subreddit_items)
+    else:
+        return pd.DataFrame()
+        
+    
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+    for i, submission in enumerate(subreddit_selection):
+        try:
+            remaining, reset_timestamp, used = check_limit(reddit_client)
+            if not vars(submission).get('_fetched'):
+                submission._fetch()
+                remaining, reset_timestamp, used = check_limit(reddit_client)
+
+            author = submission.author
+            author_record  = parse_author(reddit_client, author)
+            submission_record = parse_submission(reddit_client, submission)    
+
+            record = sub_record + author_record + submission_record
+            all_records.append(record)
+           
+        except Exception as e:
+            pass
+            print(e)
+
+        df = pd.DataFrame.from_records(all_records, 
+                                        columns = ["subreddit_id", "subreddit_name", "subreddit_display_name", "subreddit_subscribers", "subreddit_date", "subreddit_description", "subreddit_public_description", "subreddit_over18", 
+                                                    "subreddit_spoilers_enabled", "subreddit_can_assign_user_flair", "subreddit_can_assign_link_flair", "subreddit_lang", "subreddit_active_user_count",
+                                                    "author_id", "author_name", "author_link_karma", "author_comment_karma", "author_created_utc", "author_icon_img", "author_is_employee", "author_is_mod", "author_is_gold",
+                                                    "submission_id", "submission_title", "submission_name", "submission_created_utc", "submission_distinguished", "submission_edited", "submission_is_self", "submission_link_flair_template_id", 
+                                                    "submission_link_flair_text", "submission_locked", "submission_num_comments", "submission_over_18", "submission_permalink", "submission_selftext", "submission_spoiler", 
+                                                    "submission_stickied", "submission_url", "submission_upvote_ratio", "submission_downs", "submission_ups", "submission_num_crossposts", "submission_num_reports", "submission_score", 
+                                                    "submission_total_awards_received", "submission_view_count"]
+        )
+
+    return df
+
+
+
+def getComments(reddit_client : praw.Reddit, submission_id : str) -> pd.DataFrame:
+    """
+    Retrieves all comments from a submission ID.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        submission_id (str): a submission ID.
+
+    Returns:
+        pd.DataFrame: A DataFrame containing comments metadata.
+    
+    """
+    
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+    submission = reddit_client.submission(str(submission_id))
+    if not vars(submission).get('_fetched'):
+        submission._fetch()
+
+    submission.comments.replace_more(limit=None)
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+
+    all_records = []
+    for comment in tqdm(submission.comments.list(), total=len(submission.comments.list()), desc="Récupération des commentaires"):
+        remaining, reset_timestamp, used = check_limit(reddit_client)
+        record = (submission_id,) + parse_comments(reddit_client, comment)
+        all_records.append(record)
+
+    df = pd.DataFrame.from_records(all_records, columns=["submission_id", "comment_id", "comment_body", "comment_date", "comment_distinguished", "comment_is_submitter", "comment_link_id", "comment_parent_id", "comment_permalink", 
+                                                         "comment_controversiality", "comment_depth", "comment_score", "comment_total_awards_received", "comment_ups",
+                                                         "author_id", "author_name", "author_link_karma", "author_comment_karma", "author_created_utc", "author_icon_img", "author_is_employee", "author_is_mod", "author_is_gold"
+                                                         ])
+
+    return df
+
+def get_top_level_comments(reddit_client : praw.Reddit, submission_id : str) -> pd.DataFrame:
+    """
+    Retrieves top level comments from a submission ID.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        submission_id (str): a submission ID.
+
+    Returns:
+        pd.DataFrame: A DataFrame containing comments metadata.
+    
+    """
+
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+    submission = reddit_client.submission(str(submission_id))
+    if not vars(submission).get('_fetched'):
+        submission._fetch()
+
+    submission.comments.replace_more(limit=None)
+    remaining, reset_timestamp, used = check_limit(reddit_client)
+
+    all_records = []
+    for comment in tqdm(submission.comments, total=len(submission.comments), desc="Récupération des commentaires"):
+        remaining, reset_timestamp, used = check_limit(reddit_client)
+        record = (submission_id,) + parse_comments(reddit_client, comment)
+        all_records.append(record)
+
+    df = pd.DataFrame.from_records(all_records, columns=["submission_id", "comment_id", "comment_body", "comment_date", "comment_distinguished", "comment_is_submitter", "comment_link_id", "comment_parent_id", "comment_permalink", 
+                                                         "comment_controversiality", "comment_depth", "comment_score", "comment_total_awards_received", "comment_ups",
+                                                         "author_id", "author_name", "author_link_karma", "author_comment_karma", "author_created_utc", "author_icon_img", "author_is_employee", "author_is_mod", "author_is_gold"
+                                                         ])
+
+    return df
+
+def parse_author(reddit_client : praw.Reddit, author : praw.models.Redditor) -> tuple:
+    """
+    Parses a Reddit author object and extracts relevant information.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        author (praw.models.Redditor): The Reddit author object.
+
+    Returns:
+        tuple: A tuple containing the following information about the author:
+            - author_id: The ID of the author.
+            - author_name: The name of the author.
+            - author_link_karma: The link karma of the author.
+            - author_comment_karma: The comment karma of the author.    
+            - author_created_utc: The creation date of the author.
+            - author_icon_img: The icon image of the author.
+            - author_is_employee: Indicates if the author is an employee.
+            - author_is_mod: Indicates if the author is a moderator.
+            - author_is_gold: Indicates if the author has Reddit Gold.
+    """
+
+    if author:
+        if not vars(author).get('_fetched'):
+            remaining, reset_timestamp, used = check_limit(reddit_client)
+            author._fetch()
+        author_comment_karma= vars(author).get("comment_karma", None)
+
+        author_created_utc= vars(author).get("created_utc", None)
+        if author_created_utc:
+            author_created_utc = datetime.datetime.fromtimestamp(int(author_created_utc)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
+        else:
+            author_created_utc = datetime.datetime(1970,1,1,0,0,0)
+        
+        author_icon_img= vars(author).get("icon_img", None)
+        author_is_employee= vars(author).get("is_employee", None)
+        author_is_mod= vars(author).get("is_mod", None)
+        author_is_gold= vars(author).get("is_gold", None)
+        author_link_karma= vars(author).get("link_karma", None)
+        author_name= vars(author).get("name", None)
+        author_id= vars(author).get("id", None)
+
+        record = (author_id, author_name, author_link_karma, author_comment_karma, author_created_utc, author_icon_img, author_is_employee, author_is_mod, author_is_gold)
+    else:
+        record = (None, None, None, None, None, None, None, None, None)
+    return record
+
+def parse_submission(reddit_client : praw.Reddit, submission : praw.models.Submission) -> tuple:
+    """
+    Parses a Reddit submission object and extracts relevant information.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        submission (praw.models.Submission): The Reddit submission object.
+
+    Returns:
+        tuple: A tuple containing information about the submission.
+    """
+
+    if submission :
+        if not vars(submission).get('_fetched'): 
+            remaining, reset_timestamp, used = check_limit(reddit_client)
+            submission._fetch()
+        submission_id= vars(submission).get("id", None)
+        submission_title= vars(submission).get("title", None)
+        submission_name= vars(submission).get("name", None)
+        submission_created_utc= vars(submission).get("created_utc", None)
+        if submission_created_utc:
+            submission_created_utc = datetime.datetime.fromtimestamp(int(submission_created_utc)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
+        else:
+            submission_created_utc = datetime.datetime(1970,1,1,0,0,0)
+        submission_distinguished= vars(submission).get("distinguished", None)
+        submission_edited= vars(submission).get("edited", None)
+        submission_is_self= vars(submission).get("is_self", None)
+        submission_link_flair_template_id= vars(submission).get("link_flair_template_id", None)
+        submission_link_flair_text= vars(submission).get("link_flair_text", None)
+        submission_locked= vars(submission).get("locked", None)
+        submission_num_comments= vars(submission).get("num_comments", None)
+        submission_over_18= vars(submission).get("over_18", None)
+        submission_permalink= vars(submission).get("permalink", None)
+        submission_selftext= vars(submission).get("selftext", None)
+        submission_spoiler= vars(submission).get("spoiler", None)
+        submission_stickied= vars(submission).get("stickied", None)
+        submission_upvote_ratio= vars(submission).get("upvote_ratio", None)
+        submission_url= vars(submission).get("url", None)
+        submission_downs= vars(submission).get("downs", None)
+        submission_num_crossposts= vars(submission).get("num_crossposts", None)
+        submission_num_reports= vars(submission).get("num_reports", None)
+        submission_score= vars(submission).get("score", None)
+        submission_total_awards_received= vars(submission).get("total_awards_received", None)
+        submission_view_count= vars(submission).get("view_count", None)
+        submission_ups= vars(submission).get("ups", None)
+        record = (submission_id, submission_title, submission_name, submission_created_utc, submission_distinguished, submission_edited, submission_is_self, submission_link_flair_template_id, 
+                  submission_link_flair_text, submission_locked, submission_num_comments, submission_over_18, submission_permalink, submission_selftext, submission_spoiler, 
+                  submission_stickied, submission_url, submission_upvote_ratio, submission_downs, submission_ups, submission_num_crossposts, submission_num_reports, submission_score, 
+                  submission_total_awards_received, submission_view_count)
+    else:
+        record = (None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None)
+    return record 
+
+
+def parse_subreddit(reddit_client : praw.Reddit, subreddit : praw.models.Subreddit) -> tuple:
+    """
+    Parses a Reddit subreddit object and extracts relevant information.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        subreddit (praw.models.Subreddit): The Reddit subreddit object.
+
+    Returns:
+        tuple: A tuple containing information about the subreddit.
+    """
+
+    if subreddit:
+        if not vars(subreddit).get('_fetched'):
+            remaining, reset_timestamp, used = check_limit(reddit_client)
+            subreddit._fetch()
+        subreddit_id= vars(subreddit).get("id", None)
+        name = vars(subreddit).get("name", None)
+        display_name = vars(subreddit).get("display_name", None)
+        subscribers = vars(subreddit).get("subscribers", None)
+        subscribers = vars(subreddit).get("subscribers", None)
+        date = vars(subreddit).get("created_utc", None)
+        if date:
+            date=datetime.datetime.fromtimestamp(int(date)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
+        else:
+            date = datetime.datetime(1970,1,1,0,0,0)
+        description = vars(subreddit).get("description", None)
+        public_description = vars(subreddit).get("public_description", None)
+        over18 = vars(subreddit).get("over18", None)
+        spoilers_enabled = vars(subreddit).get("spoilers_enabled", None)
+        can_assign_user_flair = vars(subreddit).get("can_assign_user_flair", None)
+        can_assign_link_flair = vars(subreddit).get("can_assign_link_flair", None)
+        lang = vars(subreddit).get("lang", None)
+        active_user_count = vars(subreddit).get("active_user_count", None)
+
+        record = (subreddit_id, name, display_name, subscribers, date, description, public_description, over18, spoilers_enabled, can_assign_user_flair, can_assign_link_flair, lang, active_user_count)
+
+    else:
+        record = (None, None, None, None, None, None, None, None, None, None, None, None, None)
+    return record
+
+def parse_comments(reddit_client : praw.Reddit, comment : praw.models.Comment) -> tuple:
+    """
+    Parses a Reddit comment object and extracts relevant information.
+
+    Args:
+        reddit_client (praw.Reddit): current reddit client
+        comment (praw.models.Comment): The Reddit comment object.
+
+    Returns:
+        tuple: A tuple containing information about the comment.
+    """
+
+    if comment:
+        if not vars(comment).get('_fetched'):
+            remaining, reset_timestamp, used = check_limit(reddit_client)
+            comment._fetch()
+        comment_id = vars(comment).get("id", None)
+        comment_body = vars(comment).get("body", None)
+        comment_date = vars(comment).get("created_utc", None)
+        if comment_date:
+            comment_date = datetime.datetime.fromtimestamp(int(comment_date)).replace(tzinfo=datetime.timezone.utc).strftime("%d/%m/%Y %H:%M:%S")
+        else:
+            comment_date = datetime.datetime(1970,1,1,0,0,0)
+        comment_distinguished = vars(comment).get("distinguished", None)
+        # comment_edited = vars(comment).get("edited", None)
+        comment_is_submitter = vars(comment).get("is_submitter", None)
+        comment_link_id = vars(comment).get("link_id", None)
+        comment_parent_id = vars(comment).get("parent_id", None)
+        comment_permalink = vars(comment).get("permalink", None)
+        comment_controversiality = vars(comment).get("controversiality", None)
+        comment_depth = vars(comment).get("depth", None)
+        # comment_downs = vars(comment).get("downs", None)
+        # comment_likes = vars(comment).get("likes", None)
+        # comment_num_reports = vars(comment).get("num_reports", None)
+        comment_score = vars(comment).get("score", None)
+        comment_total_awards_received = vars(comment).get("total_awards_received", None)
+        comment_ups = vars(comment).get("ups", None)
+        author = comment.author
+        author_record = parse_author(reddit_client, author)
+        record = (comment_id, comment_body, comment_date, comment_distinguished, comment_is_submitter, comment_link_id, comment_parent_id, comment_permalink, comment_controversiality, comment_depth, comment_score, comment_total_awards_received, comment_ups) + author_record    
+    else:
+        record = (None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None)  
+    return record
+            

{opsci_toolbox-0.0.13 → opsci_toolbox-0.0.14}/opsci_toolbox/apis/telegram.py

@@ -44,6 +44,59 @@ def parse_mediafileid(message: Message) -> str:
         return None
 
 
+def parse_message_entities(messages : list) -> pd.DataFrame:
+    """
+    Parse Telegram messages entities.
+
+    Args:
+        messages : a list of Telegram messages.
+
+    Returns:
+        pd.DataFrame : a DataFrame containing the parsed entities.
+    """
+    all_records = []
+    for message in messages:
+        raw_text = message.raw_text
+
+        data = message.to_dict()
+
+        message_id = data.get("id")
+
+        peer_id = data.get("peer_id", {})
+        if peer_id is None:
+            peer_id = {}
+        channel_id = parse_from(peer_id)
+
+        from_id = data.get("from_id", {})
+        if from_id is None :
+            from_id = {}
+        from_id = parse_from(from_id)
+        if from_id is None:
+            from_id = channel_id
+
+        grouped_id = data.get("grouped_id")
+        if grouped_id:
+            grouped_id = grouped_id
+        else:
+            grouped_id = message_id
+
+        message = data.get("message")
+
+        entities = data.get("entities", [])
+        for entity in entities:
+            entity_type = entity.get("_")
+            offset = entity.get("offset")
+            length = entity.get("length")
+            url = entity.get("url")
+            document_id = entity.get("document_id")
+
+            entity_record = (message_id, channel_id, from_id, grouped_id, message, raw_text, entity_type, offset, length, url, document_id)
+            all_records.append(entity_record)
+    
+    df = pd.DataFrame.from_records(all_records, columns=["message_id", "channel_id", "from_id", "grouped_id", "message", "raw_text", "entity_type", "offset", "length", "url", "document_id"])
+    return df
+
+
 def parse_messages(messages : list) -> pd.DataFrame:
     """
     Parse Telegram messages.
@@ -57,6 +110,7 @@ def parse_messages(messages : list) -> pd.DataFrame:
     
     all_records = []
     for message in messages:
+        raw_text = message.raw_text
 
         data = message.to_dict()
 
@@ -112,10 +166,10 @@ def parse_messages(messages : list) -> pd.DataFrame:
         engagements = forwards + replies + total_reactions
 
 
-        post_record = (message_id, channel_id, from_id, grouped_id, date, message, views, forwards, replies, total_reactions, reactions_details, engagements, reply_to_message_id, reply_to_channel_id) + media_record + fwd_record
+        post_record = (message_id, channel_id, from_id, grouped_id, date, message, raw_text, views, forwards, replies, total_reactions, reactions_details, engagements, reply_to_message_id, reply_to_channel_id) + media_record + fwd_record
         all_records.append(post_record)
 
-    df = pd.DataFrame.from_records(all_records, columns=["message_id", "channel_id", "from_id", "grouped_id", "date", "message", "views", "forwards", "replies", "reactions", "details_reactions", "engagements", "reply_to_message_id", "reply_to_channel_id",
+    df = pd.DataFrame.from_records(all_records, columns=["message_id", "channel_id", "from_id", "grouped_id", "date", "message", "raw_text", "views", "forwards", "replies", "reactions", "details_reactions", "engagements", "reply_to_message_id", "reply_to_channel_id",
                                             "media_id", "media_date", "media_mime_type", "media_size", "media_filename", "duration", "width", "height", 
                                             "webpage_id", "webpage_url", "webpage_type", "webpage_site_name", "webpage_title", "webpage_description",
                                             "fwd_date", "fwd_from_id", "fwd_from_post_id", "fwd_from_from_name"])
@@ -338,7 +392,7 @@ def group_by_post(df : pd.DataFrame) -> pd.DataFrame:
     df = df.groupby(['channel_id',"grouped_id"]).agg(**aggregations).reset_index()
     return df
 
-async def get_messages_by_date(client: TelegramClient, phone_number: str, channel_username: int, dl_files: bool = False, reverse: bool = True, limit: int = None, offset_date: datetime = datetime(1970,1,1), ids: list = [], path_file: str = "files") -> list:
+async def get_messages_by_date(client: TelegramClient, phone_number: str, channel_username: int, dl_files: bool = False, reverse: bool = True, limit: int = None, offset_date: datetime = datetime(1970,1,1), ids: list = [], path_file: str = "files", dl_thumbs : bool = False) -> list:
     """
     Retrieves messages from a Telegram channel by date.
 
@@ -362,9 +416,14 @@ async def get_messages_by_date(client: TelegramClient, phone_number: str, channe
     try:
         await client.start(phone_number)
 
-        current_path_file = create_dir(os.path.join(path_file, "messages"))
+        # current_path_file = create_dir(os.path.join(path_file, "messages"))
+        path_messages = create_dir(os.path.join(path_file, "messages"))
+        path_entities = create_dir(os.path.join(path_file, "entities"))
+
         if dl_files:
             current_path_img = create_dir(os.path.join(path_file, "img", str(channel_username)))
+        if dl_thumbs:
+            current_path_thumbs = create_dir(os.path.join(path_file, "thumbs", str(channel_username)))
 
         # Get the message history
         messages = []
@@ -381,9 +440,21 @@ async def get_messages_by_date(client: TelegramClient, phone_number: str, channe
                 if media_fileid:
                     await message.download_media(file=os.path.join(current_path_img, media_fileid))
 
+            if dl_thumbs:                
+                media_fileid = parse_mediafileid(message)
+                if media_fileid:
+                    try:
+                        await message.download_media(file=os.path.join(current_path_thumbs, media_fileid), thumb=-1)
+                    except Exception as e:
+                        pass
+                        print(e)
+
         df_exploded = parse_messages(messages)
         df_exploded.loc[df_exploded['media_id'].notna(), "media_fileid"] = df_exploded['channel_id'].astype(str)+'_'+df_exploded['grouped_id'].astype(str)+'_'+df_exploded['media_id'].astype(str)
-        write_pickle(df_exploded, current_path_file, str(channel_username))
+        write_pickle(df_exploded, path_messages, str(channel_username))
+
+        df_entities = parse_message_entities(messages)
+        write_pickle(df_entities, path_entities, str(channel_username))
 
         # df_messages = group_by_post(df_exploded)
         # df_messages['uniq_id']=df_messages['channel_id'].astype(str)+'_'+df_messages['from_id'].astype(str)+'_'+df_messages['concatenated_message_id'].astype(str)
@@ -396,7 +467,7 @@ async def get_messages_by_date(client: TelegramClient, phone_number: str, channe
 
 async def get_messages_by_ids(client: TelegramClient, phone_number: str, channel_username:int, dl_files:bool=False, ids:list=[], path_file:str="files")-> list:
     """
-    Retrieves messages from a Telegram channel by date.
+    Retrieves messages from a Telegram channel by IDS.
 
     Args:
         client (TelegramClient): The Telegram client instance.
@@ -416,7 +487,8 @@ async def get_messages_by_ids(client: TelegramClient, phone_number: str, channel
     try:
         await client.start(phone_number)
 
-        current_path_file = create_dir(os.path.join(path_file, "messages"))
+        path_messages = create_dir(os.path.join(path_file, "messages"))
+        path_entities = create_dir(os.path.join(path_file, "entities"))
 
         # Get the message history
         messages = []
@@ -433,7 +505,12 @@ async def get_messages_by_ids(client: TelegramClient, phone_number: str, channel
 
         df_exploded = parse_messages(messages)
         df_exploded.loc[df_exploded['media_id'].notna(), "media_fileid"] = df_exploded['channel_id'].astype(str)+'_'+df_exploded['grouped_id'].astype(str)+'_'+df_exploded['media_id'].astype(str)
-        write_pickle(df_exploded, current_path_file, str(channel_username))
+        write_pickle(df_exploded, path_messages, str(channel_username))
+
+        df_entities = parse_message_entities(messages)
+        write_pickle(df_entities, path_entities, str(channel_username))
+
+
         # df_messages = group_by_post(df_exploded)
         # df_messages['uniq_id']=df_messages['channel_id'].astype(str)+'_'+df_messages['message_id'].astype(str)
         # write_pickle(df_messages, current_path_file, str(channel_username))
@@ -466,7 +543,10 @@ async def get_messages_by_search(client: TelegramClient, phone_number: str, sear
     try:
         await client.start(phone_number)
 
-        current_path_file = create_dir(os.path.join(path_file, "messages"))
+        # current_path_file = create_dir(os.path.join(path_file, "messages"))
+        path_messages = create_dir(os.path.join(path_file, "messages"))
+        path_entities = create_dir(os.path.join(path_file, "entities"))
+
         if dl_files:
             current_path_img = create_dir(os.path.join(path_file, "img", str(channel_username)))
 
@@ -489,10 +569,14 @@ async def get_messages_by_search(client: TelegramClient, phone_number: str, sear
         df_exploded.loc[df_exploded['media_id'].notna(), "media_fileid"] = df_exploded['channel_id'].astype(str)+'_'+df_exploded['grouped_id'].astype(str)+'_'+df_exploded['media_id'].astype(str)
         # df_messages = group_by_post(df_exploded)
         # df_messages['uniq_id']=df_messages['channel_id'].astype(str)+'_'+df_messages['concatenated_message_id'].astype(str)
+        df_entities = parse_message_entities(messages)
+
         if channel_username:
-            write_pickle(df_exploded, current_path_file, str(search)+'_'+str(channel_username))
+            write_pickle(df_exploded, path_messages, str(search)+'_'+str(channel_username))
+            write_pickle(df_entities, path_entities, str(search)+'_'+str(channel_username))
         else:
-            write_pickle(df_exploded, current_path_file, str(search))
+            write_pickle(df_exploded, path_messages, str(search))
+            write_pickle(df_entities, path_entities, str(search))
 
         return messages
     finally:
@@ -504,7 +588,9 @@ async def download_comments(client: TelegramClient, phone_number: str,channel_en
         # Connect the client
         await client.start(phone_number)
 
-        current_path_file = create_dir(os.path.join(path_file, "messages"))
+        # current_path_file = create_dir(os.path.join(path_file, "messages"))
+        path_messages = create_dir(os.path.join(path_file, "messages"))
+        path_entities = create_dir(os.path.join(path_file, "entities"))
 
         comments = []
         
@@ -519,7 +605,10 @@ async def download_comments(client: TelegramClient, phone_number: str,channel_en
 
         df_comments = parse_messages(comments)
         df_comments.loc[df_comments['media_id'].notna(), "media_fileid"] = df_comments['channel_id'].astype(str)+'_'+df_comments['grouped_id'].astype(str)+'_'+df_comments['media_id'].astype(str)
-        write_pickle(df_comments, current_path_file, str(channel_entity)+"_"+str(message_id))
+        write_pickle(df_comments, path_messages, str(channel_entity)+"_"+str(message_id))
+
+        df_entities = parse_message_entities(comments)
+        write_pickle(df_entities, path_entities, str(channel_entity)+"_"+str(message_id))
         
         return comments
     
@@ -954,7 +1043,7 @@ async def download_comments(client: TelegramClient, phone_number: str,channel_en
 #         # Disconnect the client
 #         await client.disconnect()
 
-async def get_channel_info(api_id : int, api_hash : str, phone_number : str, channel_username : str) -> dict:
+async def get_channel_info(api_id : int, api_hash : str, phone_number : str, channel_username : str, path_img :str) -> dict:
     """
     Retrieves information about a Telegram channel.
 
@@ -974,12 +1063,13 @@ async def get_channel_info(api_id : int, api_hash : str, phone_number : str, cha
     try:
         await client.start(phone_number)
         channel_full_info = await client(GetFullChannelRequest(channel=channel_username))
-        channel_full_info = channel_full_info.to_dict()
+        channel_full_info_json = channel_full_info.to_dict()
+        img_path = await client.download_profile_photo(channel_full_info.chats[0], download_big=False, file = path_img)
     finally:
         # Disconnect the client
         await client.disconnect()
 
-    return channel_full_info
+    return channel_full_info_json
 
 
 def parse_channel(channel : dict) -> pd.DataFrame: