instagram-posts-scraper 0.0.1__tar.gz → 0.0.2__tar.gz

instagram_posts_scraper-0.0.2/PKG-INFO

@@ -0,0 +1,81 @@
+Metadata-Version: 2.2
+Name: instagram-posts-scraper
+Version: 0.0.2
+Summary: Implement Instagram Posts Scraper for post data retrieval
+Home-page: https://github.com/FaustRen/instagram-posts-scraper
+Author: FaustRen
+Author-email: faustren1z@gmail.com
+License: MIT
+Classifier: Programming Language :: Python :: 3.11
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-python
+Dynamic: summary
+
+# Instagram Posts Scraper
+
+InstagramPostsScraper is a Python library for collect instagram users' data.
+
+The data obtained by web crawlers is not real-time data, but rather data from a specific point in time on the same day.
+
+I’d really appreciate your support! You can star ⭐ or fork this repository to help me keep sharing more interesting web scrapers.
+
+# Support Me
+
+If you enjoy this project and would like to support me, please consider donating 🙌  
+Your support will help me continue developing this project and working on other exciting ideas!
+
+## 💖 Ways to Support:
+
+- **PayPal**: [https://www.paypal.me/faustren1z](https://www.paypal.me/faustren1z)
+- **Buy Me a Coffee**: [https://buymeacoffee.com/faustren1z](https://buymeacoffee.com/faustren1z)
+
+Thank you for your support!! 🎉
+
+
+## Requirements
+```bash
+beautifulsoup4==4.13.3
+cloudscraper==1.2.71
+pandas==2.2.3
+pytz==2025.1
+```
+
+## Installation
+
+To install the latest release from PyPI:
+
+```sh
+pip install instagram-posts-scraper
+```
+
+## Usage - Sample
+
+```python
+from instagram_posts_scraper.instagram_posts_scraper import InstaPeriodScraper
+
+
+target_info = {"username": "kaicenat", "days_limit":60}
+ig_posts_scraper = InstaPeriodScraper()
+res = ig_posts_scraper.get_posts(target_info=target_info)
+```
+
+### Optional parameters
+
+- **username**: target instagram user 
+- **days_limit**: Number of days within which to scrape posts..
+
+## Sample(Usage & Result) - KaiCenat
+![image](https://github.com/FaustRen/instagram-posts-scraper/blob/main/usage.png)
+
+![image](https://github.com/FaustRen/instagram-posts-scraper/blob/main/scraped_posts.png)

{instagram_posts_scraper-0.0.1 → instagram_posts_scraper-0.0.2}/instagram_posts_scraper/example.py

@@ -3,7 +3,7 @@
 from instagram_posts_scraper.instagram_posts_scraper import InstaPeriodScraper
 
 
-target_info = {"username": "joemanweng", "days_limit":5}
+target_info = {"username": "kaicenat", "days_limit":5}
 ig_posts_scraper = InstaPeriodScraper()
 res = ig_posts_scraper.get_posts(target_info=target_info)
 res

{instagram_posts_scraper-0.0.1 → instagram_posts_scraper-0.0.2}/instagram_posts_scraper/file_operation.py

@@ -17,12 +17,3 @@ class JsonOperation:
         with open(current_directory, 'r') as file:
             result = json.load(file)
         return result
-
-    
-
-class CsvOperation:
-    @staticmethod
-    def read_kol_csv():
-        current_directory = os.path.dirname(__file__) + "/resources/new_input_scraper.csv"
-        result = pd.read_csv(current_directory)
-        return result

instagram_posts_scraper-0.0.2/instagram_posts_scraper/instagram_posts_scraper.py

@@ -0,0 +1,263 @@
+# -*- coding: utf-8 -*-
+from instagram_posts_scraper.request import *
+from instagram_posts_scraper.parse import *
+from instagram_posts_scraper.utils import *
+from instagram_posts_scraper.scraper import *
+from instagram_posts_scraper.utils.utils import *
+from instagram_posts_scraper.file_operation import *
+from datetime import datetime
+
+
+class ScrapedDataManager(object):
+    def __init__(self):
+        pass
+
+
+class InstaPeriodScraper(object):
+    def __init__(self) -> None:
+        self.pixwox_request = PixwoxRequest()
+        self.parser=Parser()
+        self.api_parser=ApiParser()
+        self.scraper=Scraper( 
+            pixwox_request=self.pixwox_request, 
+            parser=self.parser, 
+            api_parser=self.api_parser
+        )
+
+    def check_account_is_public(self):
+        init_response = self.pixwox_request.send_requests(url=self.scraper.init_url)
+        self.profile_soup = self.parser.get_soup(response=init_response)
+        self.userid = self.parser.get_userid(profile_soup=self.profile_soup)
+        self.account_status = get_account_status(userid=self.userid, profile_soup=self.profile_soup)
+        return self.account_status == "public"
+    
+    def get_profile(self):
+        self.followings = self.parser.get_followings(self.profile_soup)
+        self.followers = self.parser.get_followers(self.profile_soup)
+        self.counts_of_posts = self.parser.get_counts_of_posts(self.profile_soup)
+        try:
+            self.introduction = self.parser.get_introduction(self.profile_soup)
+        except:
+            self.introduction = None
+        
+        self.profile_info = {
+            "introduction": self.introduction,
+            "counts_of_posts": self.counts_of_posts,
+            "followers": self.followers,
+            "followings": self.followings,
+        }
+
+    def get_init_api_data(self):
+        init_api_data = self.scraper.get_init_api_data(userid=self.userid)
+        return init_api_data
+    
+    def get_next_api_data(self, next_maxid:str, next_:str,username:str):
+        next_api_data = self.scraper.get_next_api_data(userid=self.userid, next_maxid=next_maxid, next_=next_, username=username)
+        return next_api_data
+
+    def get_private_account_res(self):
+        res = {
+            "profile":{
+                "userid":self.userid,
+                "username":self.target_info["username"],
+                "followers":self.followers,
+                "followings":self.followings,
+                "counts_of_posts":self.counts_of_posts,
+                "introduction":self.introduction
+                },
+            "account_status":self.account_status,
+            "updated_at": get_current_time(timezone="Asia/Taipei"),
+            "data":[]
+        }
+        return res
+
+    def get_missing_account_res(self):
+        res = {
+            "profile":{
+                "userid":None,
+                "username":self.target_info["username"],
+                "followers":None,
+                "followings":None,
+                "counts_of_posts":None,
+                "introduction":None
+                },
+            "account_status":self.account_status,
+            "updated_at": get_current_time(timezone="Asia/Taipei"),
+            "data":[]
+        }
+        return res
+
+    def get_public_account_res(self, scraped_posts, init_api_data):
+        res = {
+            "profile": self.profile_info,
+            "account_status":self.account_status,
+            "updated_at": get_current_time(timezone="Asia/Taipei"),
+            "data":scraped_posts}
+        return res
+
+    # @timeout(300)
+    def get_period_data(self, days_limit: int, init_maxid: str, init_api_data: dict, username: str) -> list:
+        """
+        Fetches social media posts within specified days with retry mechanism and deduplication.
+        
+        Implements pagination handling with error resilience and detailed logging for Apify platform.
+
+        Args:
+            days_limit: Maximum number of pagination rounds to attempt
+            init_maxid: Initial pagination identifier
+            init_api_data: Initial API response data
+            username: Target account username
+
+        Returns:
+            list: Deduplicated list of post items
+
+        Raises:
+            ValueError: If initial API data structure is invalid
+        """
+        # Validate initial API structure
+        if 'posts' not in init_api_data or 'items' not in init_api_data['posts']:
+            raise ValueError("Invalid initial API data structure")
+
+        # Initialize data containers
+        scraped_posts_res = init_api_data.get('posts', {}).get('items', [])[:]  # Create list copy
+        next_metadata = {
+            'maxid': init_api_data.get('posts', {}).get('maxid', init_maxid),
+            'next': init_api_data.get('posts', {}).get('next', 0),
+            'has_next': init_api_data.get('posts', {}).get('has_next', False)
+        }
+        
+        # Deduplication setup
+        processed_shortcodes = {item['shortcode'] for item in scraped_posts_res if 'shortcode' in item}
+        
+        rounds = 0
+        MAX_RETRIES = 3
+        RETRY_DELAY = 3
+
+        while rounds < days_limit:
+            retry_count = 0
+            success = False
+
+            while retry_count < MAX_RETRIES and not success:
+                try:
+                    # Fetch next page data
+                    next_api_data = self.get_next_api_data(
+                        next_maxid=next_metadata['maxid'],
+                        next_=next_metadata['next'],
+                        username=username
+                    )
+
+                    # Process API response
+                    posts_data = next_api_data.get('posts', {})
+                    new_items = posts_data.get('items', [])
+                    
+                    # Validate data format
+                    if not isinstance(new_items, list):
+                        new_items = []
+                        current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                        print(f"{current_time} | WARNING | {self.__class__.__name__}(get_period_data) - "
+                            f"Invalid data format at round {rounds}, resetting to empty list")
+
+                    # Deduplication processing
+                    new_items_deduplicated = [
+                        item for item in new_items 
+                        if 'shortcode' in item and item['shortcode'] not in processed_shortcodes
+                    ]
+                    scraped_posts_res.extend(new_items_deduplicated)
+                    processed_shortcodes.update(item['shortcode'] for item in new_items_deduplicated)
+
+                    # Termination conditions
+                    termination_conditions = [
+                        has_all_data_been_collected(scraped_posts_res, self.counts_of_posts),
+                        is_date_exceed_half_year(new_items_deduplicated, days_limit),
+                        len(new_items_deduplicated) == 0  # No new data
+                    ]
+                    if any(termination_conditions):
+                        return scraped_posts_res
+
+                    # Update pagination parameters
+                    next_metadata = {
+                        'maxid': posts_data.get('maxid', ''),
+                        'next': posts_data.get('next', 0),
+                        'has_next': posts_data.get('has_next', False)
+                    }
+
+                    if not next_metadata['has_next']:
+                        return scraped_posts_res
+
+                    success = True
+                    rounds += 1
+
+                except Exception as e:
+                    retry_count += 1
+                    current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                    
+                    ## display the entire failed details
+                    # error_type = type(e).__name__                    
+                    # # Structured error logging
+                    # print(f"\n{current_time} | ERROR | {self.__class__.__name__}(get_period_data) - "
+                    #     f"Round {rounds} Attempt {retry_count}/{MAX_RETRIES} failed\n"
+                    #     f"Error Type: {error_type}\n"
+                    #     f"Message: {str(e)}\n"
+                    #     "Traceback Details:")
+                    
+                    # # Detailed traceback printing
+                    # traceback_str = traceback.format_exc()
+                    # print(f"{'-'*100}\n{traceback_str}\n{'-'*100}\n")
+                    
+                    ## display failed resone
+                    # print(f"[第 {rounds} 輪] 第 {retry_count} 次重試失敗，原因: {str(e)}")
+                    
+                    ## display retry rounds
+                    print(f"Instagram-posts-scraper Round {retry_count} retry.")
+                    
+                    time.sleep(RETRY_DELAY * retry_count)  # Progressive backoff
+
+            if not success:
+                current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                # print(f"\n{current_time} | CRITICAL | {self.__class__.__name__}(get_period_data) - "
+                #     f"Round {rounds} failed after {MAX_RETRIES} retries\n"
+                #     f"{'='*100}\n")
+                rounds += 1
+
+        return scraped_posts_res
+    
+    def get_posts(self, target_info:dict):
+        self.target_info = target_info
+        self.username = self.target_info["username"]
+        username = self.target_info["username"]
+        self.scraper.set_username(username)
+        days_limit = target_info["days_limit"]
+        if not self.check_account_is_public():
+            print("This is private account")
+            if self.account_status == "private":
+                self.get_profile()
+                res = self.get_private_account_res()
+                return res
+            elif self.account_status == "missing":
+                res = self.get_missing_account_res()
+                return res
+        
+        if self.check_account_is_public():
+            init_api_data = self.get_init_api_data() # 帳號資訊 & 上方頁面內容
+            self.get_profile()
+            print(f"This is public account")
+            # can scrape next round's posts
+            if init_api_data["posts"]["has_next"] != False:
+                maxid = init_api_data["posts"]["maxid"]
+                period_posts = self.get_period_data(
+                    init_maxid=maxid,
+                    days_limit=days_limit,
+                    init_api_data=init_api_data,
+                    username=username
+                    )
+
+                # return period_posts
+                res = self.get_public_account_res(
+                    scraped_posts=period_posts, 
+                    init_api_data=init_api_data
+                    )
+                return res
+            # # no more posts
+            elif init_api_data["posts"]["has_next"] == False: # (表示該帳號貼文數<=12, 無法繼續往下找)
+                res = self.get_public_account_res(scraped_posts=init_api_data["posts"]["items"], init_api_data=init_api_data)
+                return res

instagram_posts_scraper-0.0.2/instagram_posts_scraper.egg-info/PKG-INFO

@@ -0,0 +1,81 @@
+Metadata-Version: 2.2
+Name: instagram-posts-scraper
+Version: 0.0.2
+Summary: Implement Instagram Posts Scraper for post data retrieval
+Home-page: https://github.com/FaustRen/instagram-posts-scraper
+Author: FaustRen
+Author-email: faustren1z@gmail.com
+License: MIT
+Classifier: Programming Language :: Python :: 3.11
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-python
+Dynamic: summary
+
+# Instagram Posts Scraper
+
+InstagramPostsScraper is a Python library for collect instagram users' data.
+
+The data obtained by web crawlers is not real-time data, but rather data from a specific point in time on the same day.
+
+I’d really appreciate your support! You can star ⭐ or fork this repository to help me keep sharing more interesting web scrapers.
+
+# Support Me
+
+If you enjoy this project and would like to support me, please consider donating 🙌  
+Your support will help me continue developing this project and working on other exciting ideas!
+
+## 💖 Ways to Support:
+
+- **PayPal**: [https://www.paypal.me/faustren1z](https://www.paypal.me/faustren1z)
+- **Buy Me a Coffee**: [https://buymeacoffee.com/faustren1z](https://buymeacoffee.com/faustren1z)
+
+Thank you for your support!! 🎉
+
+
+## Requirements
+```bash
+beautifulsoup4==4.13.3
+cloudscraper==1.2.71
+pandas==2.2.3
+pytz==2025.1
+```
+
+## Installation
+
+To install the latest release from PyPI:
+
+```sh
+pip install instagram-posts-scraper
+```
+
+## Usage - Sample
+
+```python
+from instagram_posts_scraper.instagram_posts_scraper import InstaPeriodScraper
+
+
+target_info = {"username": "kaicenat", "days_limit":60}
+ig_posts_scraper = InstaPeriodScraper()
+res = ig_posts_scraper.get_posts(target_info=target_info)
+```
+
+### Optional parameters
+
+- **username**: target instagram user 
+- **days_limit**: Number of days within which to scrape posts..
+
+## Sample(Usage & Result) - KaiCenat
+![image](https://github.com/FaustRen/instagram-posts-scraper/blob/main/usage.png)
+
+![image](https://github.com/FaustRen/instagram-posts-scraper/blob/main/scraped_posts.png)

{instagram_posts_scraper-0.0.1 → instagram_posts_scraper-0.0.2}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='instagram-posts-scraper',
-    version='0.0.1',
+    version='0.0.2',
     packages=[
         "instagram_posts_scraper",
         "instagram_posts_scraper.utils"
@@ -15,7 +15,7 @@ setup(
     long_description_content_type='text/markdown',
     author='FaustRen',
     author_email='faustren1z@gmail.com',
-    url='https://github.com/FaustRen/instagram_posts_scraper',
+    url='https://github.com/FaustRen/instagram-posts-scraper',
     classifiers=[
         "Programming Language :: Python :: 3.11",
         "License :: OSI Approved :: MIT License",

instagram_posts_scraper-0.0.1/PKG-INFO

@@ -1,104 +0,0 @@
-Metadata-Version: 2.2
-Name: instagram-posts-scraper
-Version: 0.0.1
-Summary: Implement Instagram Posts Scraper for post data retrieval
-Home-page: https://github.com/FaustRen/instagram_posts_scraper
-Author: FaustRen
-Author-email: faustren1z@gmail.com
-License: MIT
-Classifier: Programming Language :: Python :: 3.11
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.11
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license
-Dynamic: requires-python
-Dynamic: summary
-
-# Instagram Posts Scraper
-
-InstagramPostsScraper is a Python library for collect instagram users' data.
-
-The data obtained by web crawlers is not real-time data, but rather data from a specific point in time on the same day.
-
-## Installation
-
-Use the package manager [pip](https://pip.pypa.io/en/stable/) to install instagram_posts_scraper.
-
-```bash
-beautifulsoup4==4.13.3
-cloudscraper==1.2.71
-pandas==2.2.3
-pytz==2025.1
-```
-
-
-## Usage - Sample
-
-```python
-from instagram_posts_scraper import InstaPeriodScraper
-
-
-target_info = {"username": "joemanweng", "days_limit":5}
-ig_posts_scraper = InstaPeriodScraper()
-res = ig_posts_scraper.get_posts(target_info=target_info)
-
-```
-
-## Result Sample
-
-```
-{
-    'profile': {
-        'introduction': ['台灣Youtuber\n⬇️專屬團購連結⬇️'],
-        'counts_of_posts': '1392',
-        'followers': '581198',
-        'followings': '859'
-    },
-    'account_status': 'public',
-    'updated_at': datetime.datetime(2025, 2, 9, 1, 28, 8, 793770, tzinfo=<DstTzInfo 'Asia/Taipei' CST+8:00:00 STD>),
-    'data': [
-        {
-            'type': 'igtv',
-            'sum': '《靈能的挑戰》參賽者後台花絮露出👀\n老師們比賽中較勁，但私下娛樂竟是互相算命！\n不知道有沒有先算出冠軍了（吃瓜）\n預祝大家新年快樂~~\n*節目中個人言論不代表本節目立場*\n﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏\n#靈能的挑戰\xa0#Joeman\xa0#東森超視33頻道\n全台第一檔靈能競賽節目\n📺️ 全季觀看平台｜\n🎥 Joeman YT 頻道觀看全季\n🎥 東森超視 33 頻道已播出完畢\n﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋\n#玄學 #易經 #塔羅 #通靈 #占星',
-            'sum_pure': '《靈能的挑戰》參賽者後台花絮露出 老師們比賽中較勁，但私下娛樂竟是互相算命！不知道有沒有先算出冠軍了（吃瓜）預祝大家新年快樂~~*節目中個人言論不代表本節目立場*﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏#靈能的挑戰\xa0#Joeman\xa0#東森超視33頻道全台第一檔靈能競賽節目 ️ 全季觀看平台｜ Joeman YT 頻道觀看全季 東森超視 33 頻道已播出完畢﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋#玄學 #易經 #塔羅 #通靈 #占星',
-            'shortcode': '6741582137415658464423',
-            'time': 1737885600,
-            'ftime': '13 days ago',
-            'count_like': 2442,
-            'count_comment': 6,
-            'count_like_pure': '2,442',
-            'count_comment_pure': '6',
-            'thum': 'https://scontent-fra3-1.cdninstagram.com/v/t51.2885-15/474907887_1129543381888335_8766424988966118915_n.jpg?stp=c0.248.640.640a_dst-jpg_e15_tt6&_nc_ht=scontent-fra3-1.cdninstagram.com&_nc_cat=103&_nc_ohc=YPiycXP_teMQ7kNvgF-DLaF&_nc_gid=3e3699c22b4246b98fb450d120d9527f&edm=APU89FABAAAA&ccb=7-5&oh=00_AYAb75YGceozAPWzbq4CQQGd_q_XyfKdjdwcKYak3J-pHA&oe=67AD07FF&_nc_sid=bc0c2c',
-            'pic': 'https://scontent-fra3-1.cdninstagram.com/v/t51.2885-15/474907887_1129543381888335_8766424988966118915_n.jpg?stp=dst-jpg_e15_tt6&_nc_ht=scontent-fra3-1.cdninstagram.com&_nc_cat=103&_nc_ohc=YPiycXP_teMQ7kNvgF-DLaF&_nc_gid=3e3699c22b4246b98fb450d120d9527f&edm=APU89FABAAAA&ccb=7-5&oh=00_AYA0qczIKFhJEbukWZRRSUvJJ28iqBuhIXZqRB3EEA450Q&oe=67AD07FF&_nc_sid=bc0c2c',
-            'pic_p': 'https://sp1.piokok.com/p/pt_6741582137415658464423_0_233bd378e8b9a4f088f6c850099951c7.jpg?u=https%3A%2F%2Fscontent-fra3-1.cdninstagram.com%2Fv%2Ft51.2885-15%2F474907887_1129543381888335_8766424988966118915_n.jpg%3Fstp%3Dc0.248.640.640a_dst-jpg_e15_tt6%26_nc_ht%3Dscontent-fra3-1.cdninstagram.com%26_nc_cat%3D103%26_nc_ohc%3DYPiycXP_teMQ7kNvgF-DLaF%26_nc_gid%3D3e3699c22b4246b98fb450d120d9527f%26edm%3DAPU89FABAAAA%26ccb%3D7-5%26oh%3D00_AYAb75YGceozAPWzbq4CQQGd_q_XyfKdjdwcKYak3J-pHA%26oe%3D67AD07FF%26_nc_sid%3Dbc0c2c',
-            'down_pic': 'https://scontent-fra3-1.cdninstagram.com/v/t51.2885-15/474907887_1129543381888335_8766424988966118915_n.jpg?stp=dst-jpg_e15_tt6&_nc_ht=scontent-fra3-1.cdninstagram.com&_nc_cat=103&_nc_ohc=YPiycXP_teMQ7kNvgF-DLaF&_nc_gid=3e3699c22b4246b98fb450d120d9527f&edm=APU89FABAAAA&ccb=7-5&oh=00_AYA0qczIKFhJEbukWZRRSUvJJ28iqBuhIXZqRB3EEA450Q&oe=67AD07FF&_nc_sid=bc0c2c&dl=1',
-            'is_video': True,
-            'video': 'https://scontent-fra3-1.cdninstagram.com/o1/v/t16/f2/m86/AQNbkAdWefxU8OJ015A7RX0oUxLJS-03KKFEbd2ueAbuWXS5jla4AsgRPhGeuuD9HsvVUz0mG5uKwOTWpYhvCVbbIPtxarY3vAIwG7A.mp4?stp=dst-mp4&efg=eyJxZV9ncm91cHMiOiJbXCJpZ193ZWJfZGVsaXZlcnlfdnRzX290ZlwiXSIsInZlbmNvZGVfdGFnIjoidnRzX3ZvZF91cmxnZW4uY2xpcHMuYzIuNzIwLmJhc2VsaW5lIn0&_nc_cat=103&vs=427033163733715_1223826287&_nc_vs=HBksFQIYUmlnX3hwdl9yZWVsc19wZXJtYW5lbnRfc3JfcHJvZC9ERDQ1NERCQUU1RkU4RDA5MzA5NzA2Q0IzODgxRUVBM192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dPcGZUeHgyY1EtV3l3c0VBQjRRdldDeTRZdHFicV9FQUFBRhUCAsgBACgAGAAbABUAACamtPfZ0u%2BTQBUCKAJDMywXQF2QEGJN0vIYEmRhc2hfYmFzZWxpbmVfMV92MREAdf4HAA%3D%3D&_nc_rid=3e36937551&ccb=9-4&oh=00_AYANPnQ7Zlk5Z_6wS0OF1Hn3ttS8vsk-0K6OyM9y8xzM6A&oe=67A925DC&_nc_sid=bc0c2c',
-            'down_video': 'same_as_above'
-        }
-    ]
-}
-
-
-
-
-## Contributing - Sample
-
-comming soon..
-
-## License - Sample
-
-comming soon..
-
-## Tests - cd to tests folder
-coverage run test_crawler.py
-
-coverage html

instagram_posts_scraper-0.0.1/instagram_posts_scraper/instagram_posts_scraper.py

@@ -1,160 +0,0 @@
-# -*- coding: utf-8 -*-
-from instagram_posts_scraper.request import *
-from instagram_posts_scraper.parse import *
-from instagram_posts_scraper.utils import *
-from instagram_posts_scraper.scraper import *
-from instagram_posts_scraper.utils.utils import *
-from instagram_posts_scraper.file_operation import *
-
-
-class ScrapedDataManager(object):
-    def __init__(self):
-        pass
-
-
-class InstaPeriodScraper(object):
-    def __init__(self) -> None:
-        self.pixwox_request = PixwoxRequest()
-        self.parser=Parser()
-        self.api_parser=ApiParser()
-        self.scraper=Scraper( 
-            pixwox_request=self.pixwox_request, 
-            parser=self.parser, 
-            api_parser=self.api_parser
-        )
-
-    def check_account_is_public(self):
-        init_response = self.pixwox_request.send_requests(url=self.scraper.init_url)
-        self.profile_soup = self.parser.get_soup(response=init_response)
-        self.userid = self.parser.get_userid(profile_soup=self.profile_soup)
-        self.account_status = get_account_status(userid=self.userid, profile_soup=self.profile_soup)
-        return self.account_status == "public"
-    
-    def get_profile(self):
-        self.followings = self.parser.get_followings(self.profile_soup)
-        self.followers = self.parser.get_followers(self.profile_soup)
-        self.counts_of_posts = self.parser.get_counts_of_posts(self.profile_soup)
-        try:
-            self.introduction = self.parser.get_introduction(self.profile_soup)
-        except:
-            self.introduction = None
-        
-        self.profile_info = {
-            "introduction": self.introduction,
-            "counts_of_posts": self.counts_of_posts,
-            "followers": self.followers,
-            "followings": self.followings,
-        }
-
-    def get_init_api_data(self):
-        init_api_data = self.scraper.get_init_api_data(userid=self.userid)
-        return init_api_data
-    
-    def get_next_api_data(self, next_maxid:str, next_:str,username:str):
-        next_api_data = self.scraper.get_next_api_data(userid=self.userid, next_maxid=next_maxid, next_=next_, username=username)
-        return next_api_data
-
-    def get_private_account_res(self):
-        res = {
-            "profile":{
-                "userid":self.userid,
-                "username":self.target_info["username"],
-                "followers":self.followers,
-                "followings":self.followings,
-                "counts_of_posts":self.counts_of_posts,
-                "introduction":self.introduction
-                },
-            "account_status":self.account_status,
-            "updated_at": get_current_time(timezone="Asia/Taipei"),
-            "data":[]
-        }
-        return res
-
-    def get_missing_account_res(self):
-        res = {
-            "profile":{
-                "userid":None,
-                "username":self.target_info["username"],
-                "followers":None,
-                "followings":None,
-                "counts_of_posts":None,
-                "introduction":None
-                },
-            "account_status":self.account_status,
-            "updated_at": get_current_time(timezone="Asia/Taipei"),
-            "data":[]
-        }
-        return res
-
-    def get_public_account_res(self, scraped_posts, init_api_data):
-        res = {
-            "profile": self.profile_info,
-            "account_status":self.account_status,
-            "updated_at": get_current_time(timezone="Asia/Taipei"),
-            "data":scraped_posts}
-        return res
-
-    # @timeout(300)
-    def get_period_data(self, days_limit:int, init_maxid:str, init_api_data, username):
-        scraped_posts_res = init_api_data["posts"]["items"]
-        next_ = init_api_data["posts"]["next"]
-        next_maxid = init_maxid
-        for rounds in range(days_limit):
-            # Scraped next rounds data
-            next_api_data = self.get_next_api_data(next_maxid=next_maxid, next_=next_, username=username)
-            scraped_posts = next_api_data["posts"]
-            scraped_items = scraped_posts["items"]
-            scraped_posts_res += scraped_items
-            
-            # if get all posts or get target period posts
-            if has_all_data_been_collected(scraped_items=scraped_posts_res, counts_of_posts=self.counts_of_posts) or is_date_exceed_half_year(scraped_items=scraped_items,days_limit=days_limit):
-                return scraped_posts_res
-            
-            elif next_api_data["posts"]["has_next"]: # if there are posts can scrape
-                next_maxid = scraped_posts["maxid"]
-                next_ = scraped_posts["next"]
-                continue
-            else:
-                return scraped_posts_res
-        return scraped_posts_res
-    
-    def get_posts(self, target_info:dict):
-        self.target_info = target_info
-        self.username = self.target_info["username"]
-        username = self.target_info["username"]
-        self.scraper.set_username(username)
-        days_limit = target_info["days_limit"]
-        if not self.check_account_is_public():
-            print("This is private account")
-            if self.account_status == "private":
-                self.get_profile()
-                res = self.get_private_account_res()
-                return res
-            elif self.account_status == "missing":
-                res = self.get_missing_account_res()
-                return res
-        
-        if self.check_account_is_public():
-            init_api_data = self.get_init_api_data() # 帳號資訊 & 上方頁面內容
-            self.get_profile()
-            print(f"This is public account")
-            # can scrape next round's posts
-            if init_api_data["posts"]["has_next"] != False:
-                maxid = init_api_data["posts"]["maxid"]
-                period_posts = self.get_period_data(
-                    init_maxid=maxid,
-                    days_limit=days_limit,
-                    init_api_data=init_api_data,
-                    username=username
-                    )
-
-                # return period_posts
-                res = self.get_public_account_res(
-                    scraped_posts=period_posts, 
-                    init_api_data=init_api_data
-                    )
-                return res
-            # # no more posts
-            elif init_api_data["posts"]["has_next"] == False: # (表示該帳號貼文數<=12, 無法繼續往下找)
-                res = self.get_public_account_res(scraped_posts=init_api_data["posts"]["items"], init_api_data=init_api_data)
-                return res

instagram_posts_scraper-0.0.1/instagram_posts_scraper.egg-info/PKG-INFO

@@ -1,104 +0,0 @@
-Metadata-Version: 2.2
-Name: instagram-posts-scraper
-Version: 0.0.1
-Summary: Implement Instagram Posts Scraper for post data retrieval
-Home-page: https://github.com/FaustRen/instagram_posts_scraper
-Author: FaustRen
-Author-email: faustren1z@gmail.com
-License: MIT
-Classifier: Programming Language :: Python :: 3.11
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.11
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license
-Dynamic: requires-python
-Dynamic: summary
-
-# Instagram Posts Scraper
-
-InstagramPostsScraper is a Python library for collect instagram users' data.
-
-The data obtained by web crawlers is not real-time data, but rather data from a specific point in time on the same day.
-
-## Installation
-
-Use the package manager [pip](https://pip.pypa.io/en/stable/) to install instagram_posts_scraper.
-
-```bash
-beautifulsoup4==4.13.3
-cloudscraper==1.2.71
-pandas==2.2.3
-pytz==2025.1
-```
-
-
-## Usage - Sample
-
-```python
-from instagram_posts_scraper import InstaPeriodScraper
-
-
-target_info = {"username": "joemanweng", "days_limit":5}
-ig_posts_scraper = InstaPeriodScraper()
-res = ig_posts_scraper.get_posts(target_info=target_info)
-
-```
-
-## Result Sample
-
-```
-{
-    'profile': {
-        'introduction': ['台灣Youtuber\n⬇️專屬團購連結⬇️'],
-        'counts_of_posts': '1392',
-        'followers': '581198',
-        'followings': '859'
-    },
-    'account_status': 'public',
-    'updated_at': datetime.datetime(2025, 2, 9, 1, 28, 8, 793770, tzinfo=<DstTzInfo 'Asia/Taipei' CST+8:00:00 STD>),
-    'data': [
-        {
-            'type': 'igtv',
-            'sum': '《靈能的挑戰》參賽者後台花絮露出👀\n老師們比賽中較勁，但私下娛樂竟是互相算命！\n不知道有沒有先算出冠軍了（吃瓜）\n預祝大家新年快樂~~\n*節目中個人言論不代表本節目立場*\n﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏\n#靈能的挑戰\xa0#Joeman\xa0#東森超視33頻道\n全台第一檔靈能競賽節目\n📺️ 全季觀看平台｜\n🎥 Joeman YT 頻道觀看全季\n🎥 東森超視 33 頻道已播出完畢\n﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋\n#玄學 #易經 #塔羅 #通靈 #占星',
-            'sum_pure': '《靈能的挑戰》參賽者後台花絮露出 老師們比賽中較勁，但私下娛樂竟是互相算命！不知道有沒有先算出冠軍了（吃瓜）預祝大家新年快樂~~*節目中個人言論不代表本節目立場*﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏﹏#靈能的挑戰\xa0#Joeman\xa0#東森超視33頻道全台第一檔靈能競賽節目 ️ 全季觀看平台｜ Joeman YT 頻道觀看全季 東森超視 33 頻道已播出完畢﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋﹋#玄學 #易經 #塔羅 #通靈 #占星',
-            'shortcode': '6741582137415658464423',
-            'time': 1737885600,
-            'ftime': '13 days ago',
-            'count_like': 2442,
-            'count_comment': 6,
-            'count_like_pure': '2,442',
-            'count_comment_pure': '6',
-            'thum': 'https://scontent-fra3-1.cdninstagram.com/v/t51.2885-15/474907887_1129543381888335_8766424988966118915_n.jpg?stp=c0.248.640.640a_dst-jpg_e15_tt6&_nc_ht=scontent-fra3-1.cdninstagram.com&_nc_cat=103&_nc_ohc=YPiycXP_teMQ7kNvgF-DLaF&_nc_gid=3e3699c22b4246b98fb450d120d9527f&edm=APU89FABAAAA&ccb=7-5&oh=00_AYAb75YGceozAPWzbq4CQQGd_q_XyfKdjdwcKYak3J-pHA&oe=67AD07FF&_nc_sid=bc0c2c',
-            'pic': 'https://scontent-fra3-1.cdninstagram.com/v/t51.2885-15/474907887_1129543381888335_8766424988966118915_n.jpg?stp=dst-jpg_e15_tt6&_nc_ht=scontent-fra3-1.cdninstagram.com&_nc_cat=103&_nc_ohc=YPiycXP_teMQ7kNvgF-DLaF&_nc_gid=3e3699c22b4246b98fb450d120d9527f&edm=APU89FABAAAA&ccb=7-5&oh=00_AYA0qczIKFhJEbukWZRRSUvJJ28iqBuhIXZqRB3EEA450Q&oe=67AD07FF&_nc_sid=bc0c2c',
-            'pic_p': 'https://sp1.piokok.com/p/pt_6741582137415658464423_0_233bd378e8b9a4f088f6c850099951c7.jpg?u=https%3A%2F%2Fscontent-fra3-1.cdninstagram.com%2Fv%2Ft51.2885-15%2F474907887_1129543381888335_8766424988966118915_n.jpg%3Fstp%3Dc0.248.640.640a_dst-jpg_e15_tt6%26_nc_ht%3Dscontent-fra3-1.cdninstagram.com%26_nc_cat%3D103%26_nc_ohc%3DYPiycXP_teMQ7kNvgF-DLaF%26_nc_gid%3D3e3699c22b4246b98fb450d120d9527f%26edm%3DAPU89FABAAAA%26ccb%3D7-5%26oh%3D00_AYAb75YGceozAPWzbq4CQQGd_q_XyfKdjdwcKYak3J-pHA%26oe%3D67AD07FF%26_nc_sid%3Dbc0c2c',
-            'down_pic': 'https://scontent-fra3-1.cdninstagram.com/v/t51.2885-15/474907887_1129543381888335_8766424988966118915_n.jpg?stp=dst-jpg_e15_tt6&_nc_ht=scontent-fra3-1.cdninstagram.com&_nc_cat=103&_nc_ohc=YPiycXP_teMQ7kNvgF-DLaF&_nc_gid=3e3699c22b4246b98fb450d120d9527f&edm=APU89FABAAAA&ccb=7-5&oh=00_AYA0qczIKFhJEbukWZRRSUvJJ28iqBuhIXZqRB3EEA450Q&oe=67AD07FF&_nc_sid=bc0c2c&dl=1',
-            'is_video': True,
-            'video': 'https://scontent-fra3-1.cdninstagram.com/o1/v/t16/f2/m86/AQNbkAdWefxU8OJ015A7RX0oUxLJS-03KKFEbd2ueAbuWXS5jla4AsgRPhGeuuD9HsvVUz0mG5uKwOTWpYhvCVbbIPtxarY3vAIwG7A.mp4?stp=dst-mp4&efg=eyJxZV9ncm91cHMiOiJbXCJpZ193ZWJfZGVsaXZlcnlfdnRzX290ZlwiXSIsInZlbmNvZGVfdGFnIjoidnRzX3ZvZF91cmxnZW4uY2xpcHMuYzIuNzIwLmJhc2VsaW5lIn0&_nc_cat=103&vs=427033163733715_1223826287&_nc_vs=HBksFQIYUmlnX3hwdl9yZWVsc19wZXJtYW5lbnRfc3JfcHJvZC9ERDQ1NERCQUU1RkU4RDA5MzA5NzA2Q0IzODgxRUVBM192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dPcGZUeHgyY1EtV3l3c0VBQjRRdldDeTRZdHFicV9FQUFBRhUCAsgBACgAGAAbABUAACamtPfZ0u%2BTQBUCKAJDMywXQF2QEGJN0vIYEmRhc2hfYmFzZWxpbmVfMV92MREAdf4HAA%3D%3D&_nc_rid=3e36937551&ccb=9-4&oh=00_AYANPnQ7Zlk5Z_6wS0OF1Hn3ttS8vsk-0K6OyM9y8xzM6A&oe=67A925DC&_nc_sid=bc0c2c',
-            'down_video': 'same_as_above'
-        }
-    ]
-}
-
-
-
-
-## Contributing - Sample
-
-comming soon..
-
-## License - Sample
-
-comming soon..
-
-## Tests - cd to tests folder
-coverage run test_crawler.py
-
-coverage html