facebook-graphql-scraper 0.1__py3-none-any.whl

facebook_graphql_scraper-0.1.dist-info/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) <2024> <FaustRen>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

facebook_graphql_scraper-0.1.dist-info/METADATA

@@ -0,0 +1,165 @@
+Metadata-Version: 2.1
+Name: facebook-graphql-scraper
+Version: 0.1
+Summary: Implement Facebook scraper for post data retrieval
+Home-page: https://github.com/FaustRen/facebook_graphql_scraper
+Author: FaustRen
+Author-email: faustren1z@gmail.com
+License: MIT
+Classifier: Programming Language :: Python :: 3.11
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# Facebook Graphql Scraper
+
+## Install
+
+To install the latest release from PyPI:
+
+```sh
+pip install facebook-graphql-scraper
+```
+
+Or, to install the latest master branch:
+
+```sh
+pip install git+https://github.com/faustren/facebook-graphql-scraper.git
+```
+
+### Usage
+
+You can choose between two methods to collect user posts data. 
+
+- **Log in with your account credentials**: login facebook account
+- **Without logging in**: Without logging in, click the X icon to 
+- **Difference**: The difference between these two methods is that for some personal accounts, you cannot browse the user's posts without logging into a Facebook account.
+
+```python
+from fb_user_spider import *
+from dotenv import load_dotenv
+import os
+
+## Log in account version
+load_dotenv()
+username_or_userid = os.getenv("FB_USERNAME_OR_USERID")
+pwd = os.getenv("FBPASSWORD")
+fb_user_id = "KaiCenatOfficial"
+res = get_user_posts(
+    fb_username_or_userid=fb_user_id, 
+    loop_times=50,
+    username_or_userid=username_or_userid, 
+    pwd=pwd
+)
+res
+
+## Without logging in account version
+# from fb_user_spider import *
+# fb_user_id = "KaiCenatOfficial"
+# res = get_user_posts(
+#     fb_username_or_userid=fb_user_id, 
+#     loop_times=150
+# )
+# res
+
+
+```
+### Optional parameters
+
+- **fb_user_id**: group id/fans page id/account id.
+- **timeout**: How many seconds to wait before timing out. Default is 600.
+- **looptimes**: The program scrolls down Facebook pages..
+
+
+## Post example
+
+```python
+[{'post_id': '385393337713956',
+  'post_url': 'https://www.facebook.com/385393337713956',
+  'username_or_userid': 'KaiCenatOfficial',
+  'published_date': Timestamp('2024-03-31 16:00:39'),
+  'published_date2': '2024-03-31',
+  'time': 1711900839,
+  'reaction_count.count': 1224,
+  'comment_rendering_instance.comments.total_count': 37,
+  'share_count.count': 32,
+  'sub_reactions': {'讚': 802,
+   '哈': 406,
+   '大心': 9,
+   '怒': 3,
+   '加油': 2,
+   '哇': 2,
+   '嗚': '0'},
+  'context': 'Druski Plays "What\'s In The Box?" 😱',
+  'video_view_count': 15870},
+ {'post_id': '387206327532657',
+  'post_url': 'https://www.facebook.com/387206327532657',
+  'username_or_userid': 'KaiCenatOfficial',
+  'published_date': Timestamp('2024-03-30 12:01:06'),
+  'published_date2': '2024-03-30',
+  'time': 1711800066,
+  'reaction_count.count': 7153,
+  'comment_rendering_instance.comments.total_count': 154,
+  'share_count.count': 67,
+  'sub_reactions': {'讚': 5674,
+   '哈': 1307,
+   '大心': 121,
+   '嗚': 22,
+   '加油': 21,
+   '哇': 8,
+   '怒': '0'},
+  'context': 'Kai Cenat Asks Tyla Out On A Date 😍 #viralreelsfb #comedy #kaicenat #reelsfb',
+  'video_view_count': 73898},
+ {'post_id': '385387707714519',
+  'post_url': 'https://www.facebook.com/385387707714519',
+  'username_or_userid': 'KaiCenatOfficial',
+  'published_date': Timestamp('2024-03-29 17:01:39'),
+  'published_date2': '2024-03-29',
+  'time': 1711731699,
+  'reaction_count.count': 1000,
+  'comment_rendering_instance.comments.total_count': 47,
+  'share_count.count': 15,
+  'sub_reactions': {'讚': 762,
+   '哈': 194,
+   '大心': 34,
+   '哇': 7,
+   '加油': 2,
+   '嗚': 1,
+   '怒': '0'},
+  'context': 'Chunkz Gets Hypnotised 😱',
+  'video_view_count': 13763},
+ {'post_id': '384078261178797',
+  'post_url': 'https://www.facebook.com/384078261178797',
+  'username_or_userid': 'KaiCenatOfficial',
+  'published_date': Timestamp('2024-03-28 17:00:21'),
+  'published_date2': '2024-03-28',
+  'time': 1711645221,
+  'reaction_count.count': 2587,
+  'comment_rendering_instance.comments.total_count': 77,
+  'share_count.count': 45,
+  'sub_reactions': {'讚': 1905,
+   '哈': 477,
+   '大心': 186,
+   '加油': 15,
+   '哇': 4,
+   '怒': '0',
+   '嗚': '0'},
+  'context': 'Trying The New Ice Spice Drink 😳',
+  'video_view_count': 125605}
+]
+```
+
+### Notes
+- If you choose to collect data by logging into your account, you may face the risk of your account being blocked, even if this program only scrolls through Facebook web pages.
+- Reaction Categories (EN): [`like`, `love`, `haha`, `sorry`, `wow`, `angry`, `care`]
+- Reaction Categories (TW): [`讚`, `哈`, `怒`, `大心`, `加油`, `哇`, `嗚`]
+
+
+```python
+
+## To-Do
+
+- Collect profile info
+- Collect image info

facebook_graphql_scraper-0.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+fb_graphql_scraper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fb_graphql_scraper/fb_user_spider.py,sha256=N3Uw2tgt0qBK9uz2ukWFPixNIYcm91nwJEhdOnw6v8U,4583
+fb_graphql_scraper/test_program.py,sha256=t1IYyWyvaqllTWPHpIoBZ5dfXrqMe70V0T_X19Ajies,584
+facebook_graphql_scraper-0.1.dist-info/LICENSE,sha256=kEBEyF0zymtwSR0YUx1nAjoC-Zq0aZYYezjbtcKtxbM,1055
+facebook_graphql_scraper-0.1.dist-info/METADATA,sha256=MGi7nJ2Jk1DnRBnHMIj0Qb2ffHUGP1pw7lhnj4JIv1I,4697
+facebook_graphql_scraper-0.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+facebook_graphql_scraper-0.1.dist-info/top_level.txt,sha256=IF6F1Bq6fXuzFxdDBe7rr-qagGzP_qiQD9qHCLm37-I,19
+facebook_graphql_scraper-0.1.dist-info/RECORD,,

facebook_graphql_scraper-0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.41.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

facebook_graphql_scraper-0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+fb_graphql_scraper

fb_graphql_scraper/fb_user_spider.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+import pandas as pd
+import time
+from IPython.display import display
+from seleniumwire.utils import decode
+from base.base_page import *
+from pages.page_optional import *
+from utils.locator import *
+from utils.utils import *
+from utils.parser import RequestsParser
+from tqdm import tqdm, trange
+
+
+class FacebookSettings:
+    def __init__(self,url,username_or_userid:str=None,pwd:str=None):
+        super().__init__()
+        self.username_or_userid = username_or_userid
+        self.pwd = pwd
+        self.url=url
+        self.set_spider()
+        self.set_container()
+    
+    def set_spider(self):
+        """>> Description: Auto login account or click "X" button to continue, 
+        but some account can't not be display info if you don't login account
+        >> Args: url (_str_): target user which you want to collect data."""
+        self.base_page = BasePage()
+        self.page_optional = PageOptional(
+            url_in=self.url, 
+            driver=self.base_page.driver,
+            username_or_userid=self.username_or_userid,
+            pwd=self.pwd
+        )
+        time.sleep(5)
+        self.requests_parser = RequestsParser(driver=self.page_optional.driver)
+        
+    def set_container(self):
+        self.post_id_list = []
+        self.reaction_count_list = []
+        self.res = {
+            "post_caption": [],
+            "post_date": [],
+            "post_likes": [],
+            "comment_share_type": [],
+            "comment_share_value": []
+        }
+
+
+class FacebookSpider(FacebookSettings):
+    def __init__(self,url,username_or_userid:str=None, pwd:str=None):
+        super().__init__(url=url, username_or_userid=username_or_userid, pwd=pwd)
+
+    def checking_page(self):
+        try:
+            self.page_optional.click_reject_login_button()
+            self.page_optional.scroll_window()
+        except Exception as e:
+            print(f"Get in to page unsucessfully, quit driver.\n error message: {e}")
+            self.page_optional.quit_driver()
+
+    def move_to_next_kol(self, url:str):
+        self.page_optional.driver.get(url=url)
+
+def pause():
+    time.sleep(1)
+
+# @timeout(600)
+def get_user_posts(fb_username_or_userid:str, loop_times:int=50, username_or_userid:str=None, pwd:str=None):
+    url = "https://www.facebook.com/"+fb_username_or_userid
+    spider = FacebookSpider(url=url,username_or_userid=username_or_userid,pwd=pwd)
+    # Scroll page
+    for round in tqdm(range(loop_times)): # 滾動次數
+        spider.page_optional.scroll_window()
+        pause()
+        # time.sleep(1)
+
+    # Collect data
+    driver_requests = spider.page_optional.driver.requests
+    for req in driver_requests:
+        req_response, req_url = req.response, req.url
+        body_out = spider.requests_parser.get_graphql_body_content(req_response=req_response, req_url=req_url)
+        if body_out:
+            spider.requests_parser.parse_body(body_content=body_out)
+
+    res_out = spider.requests_parser.collect_posts()
+    new_reactions = []
+
+    # Process data
+    for each_res in res_out:
+        each_reactions = each_res['top_reactions']['edges']
+        processed_reactions = spider.requests_parser.process_reactions(reactions_in=each_reactions)
+        new_reactions.append(processed_reactions)
+
+    final_res = pd.json_normalize(res_out)
+    final_res = final_res[[
+        'post_id', 
+        'reaction_count.count',
+        'comment_rendering_instance.comments.total_count',
+        'share_count.count',
+        'top_reactions.edges',
+        'video_view_count'
+    ]]
+    
+    final_res['context'] = spider.requests_parser.context_list
+    final_res['time'] = spider.requests_parser.creation_list
+    final_res['username_or_userid'] = fb_username_or_userid
+    final_res['published_date']=pd.to_datetime(final_res['time'],unit='s')
+    final_res['sub_reactions'] = new_reactions
+    final_res['post_url'] = "https://www.facebook.com/"+final_res['post_id']
+    final_res['published_date2'] = final_res['published_date'].dt.strftime('%Y-%m-%d')
+    final_res = final_res[[
+        'post_id', 
+        'post_url',
+        'username_or_userid',
+        'published_date',
+        'published_date2',
+        'time',
+        'reaction_count.count',
+        'comment_rendering_instance.comments.total_count',
+        'share_count.count',
+        'sub_reactions',
+        'context',
+        'video_view_count'
+    ]]
+    final_res.to_csv(f"./resources/data/res/{fb_username_or_userid}.csv", index=False)
+    final_res.to_excel(f"./resources/data/res/{fb_username_or_userid}_excel.xlsx", index=False)
+    return final_res

fb_graphql_scraper/test_program.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#%%
+from fb_user_spider import *
+from dotenv import load_dotenv
+import os
+
+## Log in account version
+# load_dotenv()
+# username_or_userid = os.getenv("FB_USERNAME_OR_USERID")
+# pwd = os.getenv("FBPASSWORD")
+# fb_user_id = "KaiCenatOfficial"
+# res = get_user_posts(
+#     fb_username_or_userid=fb_user_id, 
+#     loop_times=50,
+#     username_or_userid=username_or_userid, 
+#     pwd=pwd
+# )
+# res
+
+
+## Without logging in account version
+fb_user_id = "KaiCenatOfficial"
+res = get_user_posts(
+    fb_username_or_userid=fb_user_id, 
+    loop_times=50
+)
+print(res)