facebook-graphql-scraper 0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of facebook-graphql-scraper might be problematic. Click here for more details.
- facebook-graphql-scraper-0.1/LICENSE +19 -0
- facebook-graphql-scraper-0.1/PKG-INFO +165 -0
- facebook-graphql-scraper-0.1/README.md +150 -0
- facebook-graphql-scraper-0.1/facebook_graphql_scraper.egg-info/PKG-INFO +165 -0
- facebook-graphql-scraper-0.1/facebook_graphql_scraper.egg-info/SOURCES.txt +10 -0
- facebook-graphql-scraper-0.1/facebook_graphql_scraper.egg-info/dependency_links.txt +1 -0
- facebook-graphql-scraper-0.1/facebook_graphql_scraper.egg-info/top_level.txt +1 -0
- facebook-graphql-scraper-0.1/fb_graphql_scraper/__init__.py +0 -0
- facebook-graphql-scraper-0.1/fb_graphql_scraper/fb_user_spider.py +127 -0
- facebook-graphql-scraper-0.1/fb_graphql_scraper/test_program.py +27 -0
- facebook-graphql-scraper-0.1/setup.cfg +4 -0
- facebook-graphql-scraper-0.1/setup.py +22 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Copyright (c) <2024> <FaustRen>
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
4
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
5
|
+
in the Software without restriction, including without limitation the rights
|
|
6
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
7
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
8
|
+
furnished to do so, subject to the following conditions:
|
|
9
|
+
|
|
10
|
+
The above copyright notice and this permission notice shall be included in all
|
|
11
|
+
copies or substantial portions of the Software.
|
|
12
|
+
|
|
13
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
14
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
15
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
16
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
17
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
18
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
19
|
+
SOFTWARE.
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: facebook-graphql-scraper
|
|
3
|
+
Version: 0.1
|
|
4
|
+
Summary: Implement Facebook scraper for post data retrieval
|
|
5
|
+
Home-page: https://github.com/FaustRen/facebook_graphql_scraper
|
|
6
|
+
Author: FaustRen
|
|
7
|
+
Author-email: faustren1z@gmail.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
|
|
16
|
+
# Facebook Graphql Scraper
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
To install the latest release from PyPI:
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
pip install facebook-graphql-scraper
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Or, to install the latest master branch:
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
pip install git+https://github.com/faustren/facebook-graphql-scraper.git
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
### Usage
|
|
33
|
+
|
|
34
|
+
You can choose between two methods to collect user posts data.
|
|
35
|
+
|
|
36
|
+
- **Log in with your account credentials**: login facebook account
|
|
37
|
+
- **Without logging in**: Without logging in, click the X icon to
|
|
38
|
+
- **Difference**: The difference between these two methods is that for some personal accounts, you cannot browse the user's posts without logging into a Facebook account.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from fb_user_spider import *
|
|
42
|
+
from dotenv import load_dotenv
|
|
43
|
+
import os
|
|
44
|
+
|
|
45
|
+
## Log in account version
|
|
46
|
+
load_dotenv()
|
|
47
|
+
username_or_userid = os.getenv("FB_USERNAME_OR_USERID")
|
|
48
|
+
pwd = os.getenv("FBPASSWORD")
|
|
49
|
+
fb_user_id = "KaiCenatOfficial"
|
|
50
|
+
res = get_user_posts(
|
|
51
|
+
fb_username_or_userid=fb_user_id,
|
|
52
|
+
loop_times=50,
|
|
53
|
+
username_or_userid=username_or_userid,
|
|
54
|
+
pwd=pwd
|
|
55
|
+
)
|
|
56
|
+
res
|
|
57
|
+
|
|
58
|
+
## Without logging in account version
|
|
59
|
+
# from fb_user_spider import *
|
|
60
|
+
# fb_user_id = "KaiCenatOfficial"
|
|
61
|
+
# res = get_user_posts(
|
|
62
|
+
# fb_username_or_userid=fb_user_id,
|
|
63
|
+
# loop_times=150
|
|
64
|
+
# )
|
|
65
|
+
# res
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
### Optional parameters
|
|
70
|
+
|
|
71
|
+
- **fb_user_id**: group id/fans page id/account id.
|
|
72
|
+
- **timeout**: How many seconds to wait before timing out. Default is 600.
|
|
73
|
+
- **looptimes**: The program scrolls down Facebook pages..
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
## Post example
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
[{'post_id': '385393337713956',
|
|
80
|
+
'post_url': 'https://www.facebook.com/385393337713956',
|
|
81
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
82
|
+
'published_date': Timestamp('2024-03-31 16:00:39'),
|
|
83
|
+
'published_date2': '2024-03-31',
|
|
84
|
+
'time': 1711900839,
|
|
85
|
+
'reaction_count.count': 1224,
|
|
86
|
+
'comment_rendering_instance.comments.total_count': 37,
|
|
87
|
+
'share_count.count': 32,
|
|
88
|
+
'sub_reactions': {'讚': 802,
|
|
89
|
+
'哈': 406,
|
|
90
|
+
'大心': 9,
|
|
91
|
+
'怒': 3,
|
|
92
|
+
'加油': 2,
|
|
93
|
+
'哇': 2,
|
|
94
|
+
'嗚': '0'},
|
|
95
|
+
'context': 'Druski Plays "What\'s In The Box?" 😱',
|
|
96
|
+
'video_view_count': 15870},
|
|
97
|
+
{'post_id': '387206327532657',
|
|
98
|
+
'post_url': 'https://www.facebook.com/387206327532657',
|
|
99
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
100
|
+
'published_date': Timestamp('2024-03-30 12:01:06'),
|
|
101
|
+
'published_date2': '2024-03-30',
|
|
102
|
+
'time': 1711800066,
|
|
103
|
+
'reaction_count.count': 7153,
|
|
104
|
+
'comment_rendering_instance.comments.total_count': 154,
|
|
105
|
+
'share_count.count': 67,
|
|
106
|
+
'sub_reactions': {'讚': 5674,
|
|
107
|
+
'哈': 1307,
|
|
108
|
+
'大心': 121,
|
|
109
|
+
'嗚': 22,
|
|
110
|
+
'加油': 21,
|
|
111
|
+
'哇': 8,
|
|
112
|
+
'怒': '0'},
|
|
113
|
+
'context': 'Kai Cenat Asks Tyla Out On A Date 😍 #viralreelsfb #comedy #kaicenat #reelsfb',
|
|
114
|
+
'video_view_count': 73898},
|
|
115
|
+
{'post_id': '385387707714519',
|
|
116
|
+
'post_url': 'https://www.facebook.com/385387707714519',
|
|
117
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
118
|
+
'published_date': Timestamp('2024-03-29 17:01:39'),
|
|
119
|
+
'published_date2': '2024-03-29',
|
|
120
|
+
'time': 1711731699,
|
|
121
|
+
'reaction_count.count': 1000,
|
|
122
|
+
'comment_rendering_instance.comments.total_count': 47,
|
|
123
|
+
'share_count.count': 15,
|
|
124
|
+
'sub_reactions': {'讚': 762,
|
|
125
|
+
'哈': 194,
|
|
126
|
+
'大心': 34,
|
|
127
|
+
'哇': 7,
|
|
128
|
+
'加油': 2,
|
|
129
|
+
'嗚': 1,
|
|
130
|
+
'怒': '0'},
|
|
131
|
+
'context': 'Chunkz Gets Hypnotised 😱',
|
|
132
|
+
'video_view_count': 13763},
|
|
133
|
+
{'post_id': '384078261178797',
|
|
134
|
+
'post_url': 'https://www.facebook.com/384078261178797',
|
|
135
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
136
|
+
'published_date': Timestamp('2024-03-28 17:00:21'),
|
|
137
|
+
'published_date2': '2024-03-28',
|
|
138
|
+
'time': 1711645221,
|
|
139
|
+
'reaction_count.count': 2587,
|
|
140
|
+
'comment_rendering_instance.comments.total_count': 77,
|
|
141
|
+
'share_count.count': 45,
|
|
142
|
+
'sub_reactions': {'讚': 1905,
|
|
143
|
+
'哈': 477,
|
|
144
|
+
'大心': 186,
|
|
145
|
+
'加油': 15,
|
|
146
|
+
'哇': 4,
|
|
147
|
+
'怒': '0',
|
|
148
|
+
'嗚': '0'},
|
|
149
|
+
'context': 'Trying The New Ice Spice Drink 😳',
|
|
150
|
+
'video_view_count': 125605}
|
|
151
|
+
]
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
### Notes
|
|
155
|
+
- If you choose to collect data by logging into your account, you may face the risk of your account being blocked, even if this program only scrolls through Facebook web pages.
|
|
156
|
+
- Reaction Categories (EN): [`like`, `love`, `haha`, `sorry`, `wow`, `angry`, `care`]
|
|
157
|
+
- Reaction Categories (TW): [`讚`, `哈`, `怒`, `大心`, `加油`, `哇`, `嗚`]
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
|
|
162
|
+
## To-Do
|
|
163
|
+
|
|
164
|
+
- Collect profile info
|
|
165
|
+
- Collect image info
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# Facebook Graphql Scraper
|
|
2
|
+
|
|
3
|
+
## Install
|
|
4
|
+
|
|
5
|
+
To install the latest release from PyPI:
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
pip install facebook-graphql-scraper
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Or, to install the latest master branch:
|
|
12
|
+
|
|
13
|
+
```sh
|
|
14
|
+
pip install git+https://github.com/faustren/facebook-graphql-scraper.git
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
### Usage
|
|
18
|
+
|
|
19
|
+
You can choose between two methods to collect user posts data.
|
|
20
|
+
|
|
21
|
+
- **Log in with your account credentials**: login facebook account
|
|
22
|
+
- **Without logging in**: Without logging in, click the X icon to
|
|
23
|
+
- **Difference**: The difference between these two methods is that for some personal accounts, you cannot browse the user's posts without logging into a Facebook account.
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
from fb_user_spider import *
|
|
27
|
+
from dotenv import load_dotenv
|
|
28
|
+
import os
|
|
29
|
+
|
|
30
|
+
## Log in account version
|
|
31
|
+
load_dotenv()
|
|
32
|
+
username_or_userid = os.getenv("FB_USERNAME_OR_USERID")
|
|
33
|
+
pwd = os.getenv("FBPASSWORD")
|
|
34
|
+
fb_user_id = "KaiCenatOfficial"
|
|
35
|
+
res = get_user_posts(
|
|
36
|
+
fb_username_or_userid=fb_user_id,
|
|
37
|
+
loop_times=50,
|
|
38
|
+
username_or_userid=username_or_userid,
|
|
39
|
+
pwd=pwd
|
|
40
|
+
)
|
|
41
|
+
res
|
|
42
|
+
|
|
43
|
+
## Without logging in account version
|
|
44
|
+
# from fb_user_spider import *
|
|
45
|
+
# fb_user_id = "KaiCenatOfficial"
|
|
46
|
+
# res = get_user_posts(
|
|
47
|
+
# fb_username_or_userid=fb_user_id,
|
|
48
|
+
# loop_times=150
|
|
49
|
+
# )
|
|
50
|
+
# res
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
### Optional parameters
|
|
55
|
+
|
|
56
|
+
- **fb_user_id**: group id/fans page id/account id.
|
|
57
|
+
- **timeout**: How many seconds to wait before timing out. Default is 600.
|
|
58
|
+
- **looptimes**: The program scrolls down Facebook pages..
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
## Post example
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
[{'post_id': '385393337713956',
|
|
65
|
+
'post_url': 'https://www.facebook.com/385393337713956',
|
|
66
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
67
|
+
'published_date': Timestamp('2024-03-31 16:00:39'),
|
|
68
|
+
'published_date2': '2024-03-31',
|
|
69
|
+
'time': 1711900839,
|
|
70
|
+
'reaction_count.count': 1224,
|
|
71
|
+
'comment_rendering_instance.comments.total_count': 37,
|
|
72
|
+
'share_count.count': 32,
|
|
73
|
+
'sub_reactions': {'讚': 802,
|
|
74
|
+
'哈': 406,
|
|
75
|
+
'大心': 9,
|
|
76
|
+
'怒': 3,
|
|
77
|
+
'加油': 2,
|
|
78
|
+
'哇': 2,
|
|
79
|
+
'嗚': '0'},
|
|
80
|
+
'context': 'Druski Plays "What\'s In The Box?" 😱',
|
|
81
|
+
'video_view_count': 15870},
|
|
82
|
+
{'post_id': '387206327532657',
|
|
83
|
+
'post_url': 'https://www.facebook.com/387206327532657',
|
|
84
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
85
|
+
'published_date': Timestamp('2024-03-30 12:01:06'),
|
|
86
|
+
'published_date2': '2024-03-30',
|
|
87
|
+
'time': 1711800066,
|
|
88
|
+
'reaction_count.count': 7153,
|
|
89
|
+
'comment_rendering_instance.comments.total_count': 154,
|
|
90
|
+
'share_count.count': 67,
|
|
91
|
+
'sub_reactions': {'讚': 5674,
|
|
92
|
+
'哈': 1307,
|
|
93
|
+
'大心': 121,
|
|
94
|
+
'嗚': 22,
|
|
95
|
+
'加油': 21,
|
|
96
|
+
'哇': 8,
|
|
97
|
+
'怒': '0'},
|
|
98
|
+
'context': 'Kai Cenat Asks Tyla Out On A Date 😍 #viralreelsfb #comedy #kaicenat #reelsfb',
|
|
99
|
+
'video_view_count': 73898},
|
|
100
|
+
{'post_id': '385387707714519',
|
|
101
|
+
'post_url': 'https://www.facebook.com/385387707714519',
|
|
102
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
103
|
+
'published_date': Timestamp('2024-03-29 17:01:39'),
|
|
104
|
+
'published_date2': '2024-03-29',
|
|
105
|
+
'time': 1711731699,
|
|
106
|
+
'reaction_count.count': 1000,
|
|
107
|
+
'comment_rendering_instance.comments.total_count': 47,
|
|
108
|
+
'share_count.count': 15,
|
|
109
|
+
'sub_reactions': {'讚': 762,
|
|
110
|
+
'哈': 194,
|
|
111
|
+
'大心': 34,
|
|
112
|
+
'哇': 7,
|
|
113
|
+
'加油': 2,
|
|
114
|
+
'嗚': 1,
|
|
115
|
+
'怒': '0'},
|
|
116
|
+
'context': 'Chunkz Gets Hypnotised 😱',
|
|
117
|
+
'video_view_count': 13763},
|
|
118
|
+
{'post_id': '384078261178797',
|
|
119
|
+
'post_url': 'https://www.facebook.com/384078261178797',
|
|
120
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
121
|
+
'published_date': Timestamp('2024-03-28 17:00:21'),
|
|
122
|
+
'published_date2': '2024-03-28',
|
|
123
|
+
'time': 1711645221,
|
|
124
|
+
'reaction_count.count': 2587,
|
|
125
|
+
'comment_rendering_instance.comments.total_count': 77,
|
|
126
|
+
'share_count.count': 45,
|
|
127
|
+
'sub_reactions': {'讚': 1905,
|
|
128
|
+
'哈': 477,
|
|
129
|
+
'大心': 186,
|
|
130
|
+
'加油': 15,
|
|
131
|
+
'哇': 4,
|
|
132
|
+
'怒': '0',
|
|
133
|
+
'嗚': '0'},
|
|
134
|
+
'context': 'Trying The New Ice Spice Drink 😳',
|
|
135
|
+
'video_view_count': 125605}
|
|
136
|
+
]
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
### Notes
|
|
140
|
+
- If you choose to collect data by logging into your account, you may face the risk of your account being blocked, even if this program only scrolls through Facebook web pages.
|
|
141
|
+
- Reaction Categories (EN): [`like`, `love`, `haha`, `sorry`, `wow`, `angry`, `care`]
|
|
142
|
+
- Reaction Categories (TW): [`讚`, `哈`, `怒`, `大心`, `加油`, `哇`, `嗚`]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
```python
|
|
146
|
+
|
|
147
|
+
## To-Do
|
|
148
|
+
|
|
149
|
+
- Collect profile info
|
|
150
|
+
- Collect image info
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: facebook-graphql-scraper
|
|
3
|
+
Version: 0.1
|
|
4
|
+
Summary: Implement Facebook scraper for post data retrieval
|
|
5
|
+
Home-page: https://github.com/FaustRen/facebook_graphql_scraper
|
|
6
|
+
Author: FaustRen
|
|
7
|
+
Author-email: faustren1z@gmail.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
|
|
16
|
+
# Facebook Graphql Scraper
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
To install the latest release from PyPI:
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
pip install facebook-graphql-scraper
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Or, to install the latest master branch:
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
pip install git+https://github.com/faustren/facebook-graphql-scraper.git
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
### Usage
|
|
33
|
+
|
|
34
|
+
You can choose between two methods to collect user posts data.
|
|
35
|
+
|
|
36
|
+
- **Log in with your account credentials**: login facebook account
|
|
37
|
+
- **Without logging in**: Without logging in, click the X icon to
|
|
38
|
+
- **Difference**: The difference between these two methods is that for some personal accounts, you cannot browse the user's posts without logging into a Facebook account.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from fb_user_spider import *
|
|
42
|
+
from dotenv import load_dotenv
|
|
43
|
+
import os
|
|
44
|
+
|
|
45
|
+
## Log in account version
|
|
46
|
+
load_dotenv()
|
|
47
|
+
username_or_userid = os.getenv("FB_USERNAME_OR_USERID")
|
|
48
|
+
pwd = os.getenv("FBPASSWORD")
|
|
49
|
+
fb_user_id = "KaiCenatOfficial"
|
|
50
|
+
res = get_user_posts(
|
|
51
|
+
fb_username_or_userid=fb_user_id,
|
|
52
|
+
loop_times=50,
|
|
53
|
+
username_or_userid=username_or_userid,
|
|
54
|
+
pwd=pwd
|
|
55
|
+
)
|
|
56
|
+
res
|
|
57
|
+
|
|
58
|
+
## Without logging in account version
|
|
59
|
+
# from fb_user_spider import *
|
|
60
|
+
# fb_user_id = "KaiCenatOfficial"
|
|
61
|
+
# res = get_user_posts(
|
|
62
|
+
# fb_username_or_userid=fb_user_id,
|
|
63
|
+
# loop_times=150
|
|
64
|
+
# )
|
|
65
|
+
# res
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
### Optional parameters
|
|
70
|
+
|
|
71
|
+
- **fb_user_id**: group id/fans page id/account id.
|
|
72
|
+
- **timeout**: How many seconds to wait before timing out. Default is 600.
|
|
73
|
+
- **looptimes**: The program scrolls down Facebook pages..
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
## Post example
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
[{'post_id': '385393337713956',
|
|
80
|
+
'post_url': 'https://www.facebook.com/385393337713956',
|
|
81
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
82
|
+
'published_date': Timestamp('2024-03-31 16:00:39'),
|
|
83
|
+
'published_date2': '2024-03-31',
|
|
84
|
+
'time': 1711900839,
|
|
85
|
+
'reaction_count.count': 1224,
|
|
86
|
+
'comment_rendering_instance.comments.total_count': 37,
|
|
87
|
+
'share_count.count': 32,
|
|
88
|
+
'sub_reactions': {'讚': 802,
|
|
89
|
+
'哈': 406,
|
|
90
|
+
'大心': 9,
|
|
91
|
+
'怒': 3,
|
|
92
|
+
'加油': 2,
|
|
93
|
+
'哇': 2,
|
|
94
|
+
'嗚': '0'},
|
|
95
|
+
'context': 'Druski Plays "What\'s In The Box?" 😱',
|
|
96
|
+
'video_view_count': 15870},
|
|
97
|
+
{'post_id': '387206327532657',
|
|
98
|
+
'post_url': 'https://www.facebook.com/387206327532657',
|
|
99
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
100
|
+
'published_date': Timestamp('2024-03-30 12:01:06'),
|
|
101
|
+
'published_date2': '2024-03-30',
|
|
102
|
+
'time': 1711800066,
|
|
103
|
+
'reaction_count.count': 7153,
|
|
104
|
+
'comment_rendering_instance.comments.total_count': 154,
|
|
105
|
+
'share_count.count': 67,
|
|
106
|
+
'sub_reactions': {'讚': 5674,
|
|
107
|
+
'哈': 1307,
|
|
108
|
+
'大心': 121,
|
|
109
|
+
'嗚': 22,
|
|
110
|
+
'加油': 21,
|
|
111
|
+
'哇': 8,
|
|
112
|
+
'怒': '0'},
|
|
113
|
+
'context': 'Kai Cenat Asks Tyla Out On A Date 😍 #viralreelsfb #comedy #kaicenat #reelsfb',
|
|
114
|
+
'video_view_count': 73898},
|
|
115
|
+
{'post_id': '385387707714519',
|
|
116
|
+
'post_url': 'https://www.facebook.com/385387707714519',
|
|
117
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
118
|
+
'published_date': Timestamp('2024-03-29 17:01:39'),
|
|
119
|
+
'published_date2': '2024-03-29',
|
|
120
|
+
'time': 1711731699,
|
|
121
|
+
'reaction_count.count': 1000,
|
|
122
|
+
'comment_rendering_instance.comments.total_count': 47,
|
|
123
|
+
'share_count.count': 15,
|
|
124
|
+
'sub_reactions': {'讚': 762,
|
|
125
|
+
'哈': 194,
|
|
126
|
+
'大心': 34,
|
|
127
|
+
'哇': 7,
|
|
128
|
+
'加油': 2,
|
|
129
|
+
'嗚': 1,
|
|
130
|
+
'怒': '0'},
|
|
131
|
+
'context': 'Chunkz Gets Hypnotised 😱',
|
|
132
|
+
'video_view_count': 13763},
|
|
133
|
+
{'post_id': '384078261178797',
|
|
134
|
+
'post_url': 'https://www.facebook.com/384078261178797',
|
|
135
|
+
'username_or_userid': 'KaiCenatOfficial',
|
|
136
|
+
'published_date': Timestamp('2024-03-28 17:00:21'),
|
|
137
|
+
'published_date2': '2024-03-28',
|
|
138
|
+
'time': 1711645221,
|
|
139
|
+
'reaction_count.count': 2587,
|
|
140
|
+
'comment_rendering_instance.comments.total_count': 77,
|
|
141
|
+
'share_count.count': 45,
|
|
142
|
+
'sub_reactions': {'讚': 1905,
|
|
143
|
+
'哈': 477,
|
|
144
|
+
'大心': 186,
|
|
145
|
+
'加油': 15,
|
|
146
|
+
'哇': 4,
|
|
147
|
+
'怒': '0',
|
|
148
|
+
'嗚': '0'},
|
|
149
|
+
'context': 'Trying The New Ice Spice Drink 😳',
|
|
150
|
+
'video_view_count': 125605}
|
|
151
|
+
]
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
### Notes
|
|
155
|
+
- If you choose to collect data by logging into your account, you may face the risk of your account being blocked, even if this program only scrolls through Facebook web pages.
|
|
156
|
+
- Reaction Categories (EN): [`like`, `love`, `haha`, `sorry`, `wow`, `angry`, `care`]
|
|
157
|
+
- Reaction Categories (TW): [`讚`, `哈`, `怒`, `大心`, `加油`, `哇`, `嗚`]
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
|
|
162
|
+
## To-Do
|
|
163
|
+
|
|
164
|
+
- Collect profile info
|
|
165
|
+
- Collect image info
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
setup.py
|
|
4
|
+
facebook_graphql_scraper.egg-info/PKG-INFO
|
|
5
|
+
facebook_graphql_scraper.egg-info/SOURCES.txt
|
|
6
|
+
facebook_graphql_scraper.egg-info/dependency_links.txt
|
|
7
|
+
facebook_graphql_scraper.egg-info/top_level.txt
|
|
8
|
+
fb_graphql_scraper/__init__.py
|
|
9
|
+
fb_graphql_scraper/fb_user_spider.py
|
|
10
|
+
fb_graphql_scraper/test_program.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fb_graphql_scraper
|
|
File without changes
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import time
|
|
4
|
+
from IPython.display import display
|
|
5
|
+
from seleniumwire.utils import decode
|
|
6
|
+
from base.base_page import *
|
|
7
|
+
from pages.page_optional import *
|
|
8
|
+
from utils.locator import *
|
|
9
|
+
from utils.utils import *
|
|
10
|
+
from utils.parser import RequestsParser
|
|
11
|
+
from tqdm import tqdm, trange
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class FacebookSettings:
|
|
15
|
+
def __init__(self,url,username_or_userid:str=None,pwd:str=None):
|
|
16
|
+
super().__init__()
|
|
17
|
+
self.username_or_userid = username_or_userid
|
|
18
|
+
self.pwd = pwd
|
|
19
|
+
self.url=url
|
|
20
|
+
self.set_spider()
|
|
21
|
+
self.set_container()
|
|
22
|
+
|
|
23
|
+
def set_spider(self):
|
|
24
|
+
""">> Description: Auto login account or click "X" button to continue,
|
|
25
|
+
but some account can't not be display info if you don't login account
|
|
26
|
+
>> Args: url (_str_): target user which you want to collect data."""
|
|
27
|
+
self.base_page = BasePage()
|
|
28
|
+
self.page_optional = PageOptional(
|
|
29
|
+
url_in=self.url,
|
|
30
|
+
driver=self.base_page.driver,
|
|
31
|
+
username_or_userid=self.username_or_userid,
|
|
32
|
+
pwd=self.pwd
|
|
33
|
+
)
|
|
34
|
+
time.sleep(5)
|
|
35
|
+
self.requests_parser = RequestsParser(driver=self.page_optional.driver)
|
|
36
|
+
|
|
37
|
+
def set_container(self):
|
|
38
|
+
self.post_id_list = []
|
|
39
|
+
self.reaction_count_list = []
|
|
40
|
+
self.res = {
|
|
41
|
+
"post_caption": [],
|
|
42
|
+
"post_date": [],
|
|
43
|
+
"post_likes": [],
|
|
44
|
+
"comment_share_type": [],
|
|
45
|
+
"comment_share_value": []
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class FacebookSpider(FacebookSettings):
|
|
50
|
+
def __init__(self,url,username_or_userid:str=None, pwd:str=None):
|
|
51
|
+
super().__init__(url=url, username_or_userid=username_or_userid, pwd=pwd)
|
|
52
|
+
|
|
53
|
+
def checking_page(self):
|
|
54
|
+
try:
|
|
55
|
+
self.page_optional.click_reject_login_button()
|
|
56
|
+
self.page_optional.scroll_window()
|
|
57
|
+
except Exception as e:
|
|
58
|
+
print(f"Get in to page unsucessfully, quit driver.\n error message: {e}")
|
|
59
|
+
self.page_optional.quit_driver()
|
|
60
|
+
|
|
61
|
+
def move_to_next_kol(self, url:str):
|
|
62
|
+
self.page_optional.driver.get(url=url)
|
|
63
|
+
|
|
64
|
+
def pause():
|
|
65
|
+
time.sleep(1)
|
|
66
|
+
|
|
67
|
+
# @timeout(600)
|
|
68
|
+
def get_user_posts(fb_username_or_userid:str, loop_times:int=50, username_or_userid:str=None, pwd:str=None):
|
|
69
|
+
url = "https://www.facebook.com/"+fb_username_or_userid
|
|
70
|
+
spider = FacebookSpider(url=url,username_or_userid=username_or_userid,pwd=pwd)
|
|
71
|
+
# Scroll page
|
|
72
|
+
for round in tqdm(range(loop_times)): # 滾動次數
|
|
73
|
+
spider.page_optional.scroll_window()
|
|
74
|
+
pause()
|
|
75
|
+
# time.sleep(1)
|
|
76
|
+
|
|
77
|
+
# Collect data
|
|
78
|
+
driver_requests = spider.page_optional.driver.requests
|
|
79
|
+
for req in driver_requests:
|
|
80
|
+
req_response, req_url = req.response, req.url
|
|
81
|
+
body_out = spider.requests_parser.get_graphql_body_content(req_response=req_response, req_url=req_url)
|
|
82
|
+
if body_out:
|
|
83
|
+
spider.requests_parser.parse_body(body_content=body_out)
|
|
84
|
+
|
|
85
|
+
res_out = spider.requests_parser.collect_posts()
|
|
86
|
+
new_reactions = []
|
|
87
|
+
|
|
88
|
+
# Process data
|
|
89
|
+
for each_res in res_out:
|
|
90
|
+
each_reactions = each_res['top_reactions']['edges']
|
|
91
|
+
processed_reactions = spider.requests_parser.process_reactions(reactions_in=each_reactions)
|
|
92
|
+
new_reactions.append(processed_reactions)
|
|
93
|
+
|
|
94
|
+
final_res = pd.json_normalize(res_out)
|
|
95
|
+
final_res = final_res[[
|
|
96
|
+
'post_id',
|
|
97
|
+
'reaction_count.count',
|
|
98
|
+
'comment_rendering_instance.comments.total_count',
|
|
99
|
+
'share_count.count',
|
|
100
|
+
'top_reactions.edges',
|
|
101
|
+
'video_view_count'
|
|
102
|
+
]]
|
|
103
|
+
|
|
104
|
+
final_res['context'] = spider.requests_parser.context_list
|
|
105
|
+
final_res['time'] = spider.requests_parser.creation_list
|
|
106
|
+
final_res['username_or_userid'] = fb_username_or_userid
|
|
107
|
+
final_res['published_date']=pd.to_datetime(final_res['time'],unit='s')
|
|
108
|
+
final_res['sub_reactions'] = new_reactions
|
|
109
|
+
final_res['post_url'] = "https://www.facebook.com/"+final_res['post_id']
|
|
110
|
+
final_res['published_date2'] = final_res['published_date'].dt.strftime('%Y-%m-%d')
|
|
111
|
+
final_res = final_res[[
|
|
112
|
+
'post_id',
|
|
113
|
+
'post_url',
|
|
114
|
+
'username_or_userid',
|
|
115
|
+
'published_date',
|
|
116
|
+
'published_date2',
|
|
117
|
+
'time',
|
|
118
|
+
'reaction_count.count',
|
|
119
|
+
'comment_rendering_instance.comments.total_count',
|
|
120
|
+
'share_count.count',
|
|
121
|
+
'sub_reactions',
|
|
122
|
+
'context',
|
|
123
|
+
'video_view_count'
|
|
124
|
+
]]
|
|
125
|
+
final_res.to_csv(f"./resources/data/res/{fb_username_or_userid}.csv", index=False)
|
|
126
|
+
final_res.to_excel(f"./resources/data/res/{fb_username_or_userid}_excel.xlsx", index=False)
|
|
127
|
+
return final_res
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
#%%
|
|
3
|
+
from fb_user_spider import *
|
|
4
|
+
from dotenv import load_dotenv
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
## Log in account version
|
|
8
|
+
# load_dotenv()
|
|
9
|
+
# username_or_userid = os.getenv("FB_USERNAME_OR_USERID")
|
|
10
|
+
# pwd = os.getenv("FBPASSWORD")
|
|
11
|
+
# fb_user_id = "KaiCenatOfficial"
|
|
12
|
+
# res = get_user_posts(
|
|
13
|
+
# fb_username_or_userid=fb_user_id,
|
|
14
|
+
# loop_times=50,
|
|
15
|
+
# username_or_userid=username_or_userid,
|
|
16
|
+
# pwd=pwd
|
|
17
|
+
# )
|
|
18
|
+
# res
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
## Without logging in account version
|
|
22
|
+
fb_user_id = "KaiCenatOfficial"
|
|
23
|
+
res = get_user_posts(
|
|
24
|
+
fb_username_or_userid=fb_user_id,
|
|
25
|
+
loop_times=50
|
|
26
|
+
)
|
|
27
|
+
print(res)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
import setuptools
|
|
3
|
+
from setuptools import setup, find_packages
|
|
4
|
+
|
|
5
|
+
setup(
|
|
6
|
+
name='facebook-graphql-scraper',
|
|
7
|
+
version='0.1',
|
|
8
|
+
packages=find_packages(),
|
|
9
|
+
license='MIT',
|
|
10
|
+
description='Implement Facebook scraper for post data retrieval',
|
|
11
|
+
long_description=open('README.md').read(),
|
|
12
|
+
long_description_content_type='text/markdown',
|
|
13
|
+
author='FaustRen',
|
|
14
|
+
author_email='faustren1z@gmail.com',
|
|
15
|
+
url='https://github.com/FaustRen/facebook_graphql_scraper',
|
|
16
|
+
classifiers=[
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
],
|
|
21
|
+
python_requires='>=3.11',
|
|
22
|
+
)
|