instagram-archiver 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

instagram_archiver/__init__.py

@@ -6,4 +6,4 @@ from .profile_scraper import ProfileScraper
 from .saved_scraper import SavedScraper
 
 __all__ = ('InstagramClient', 'ProfileScraper', 'SavedScraper')
-__version__ = 'v0.3.0'
+__version__ = 'v0.3.1'

instagram_archiver/client.py

@@ -8,12 +8,11 @@ from typing import TYPE_CHECKING, Any, Self, TypeVar, cast
 import json
 import logging
 
-from bs4 import BeautifulSoup as Soup
 from requests import HTTPError
 from yt_dlp_utils import setup_session
 import requests
 
-from .constants import API_HEADERS, PAGE_FETCH_HEADERS, SHARED_HEADERS
+from .constants import API_HEADERS, SHARED_HEADERS
 from .typing import (
     CarouselMedia,
     Comments,
@@ -31,7 +30,7 @@ if TYPE_CHECKING:
 
     from .typing import BrowserName
 
-__all__ = ('CSRFTokenNotFound', 'InstagramClient')
+__all__ = ('CSRFTokenNotFound', 'InstagramClient', 'UnexpectedRedirect')
 
 T = TypeVar('T')
 log = logging.getLogger(__name__)
@@ -41,6 +40,10 @@ class CSRFTokenNotFound(RuntimeError):
     """CSRF token not found in cookies."""
 
 
+class UnexpectedRedirect(RuntimeError):
+    """Unexpected redirect in a request."""
+
+
 class InstagramClient:
     """Generic client for Instagram."""
     def __init__(self, browser: BrowserName = 'chrome', browser_profile: str = 'Default') -> None:
@@ -59,7 +62,6 @@ class InstagramClient:
                                      browser_profile,
                                      SHARED_HEADERS,
                                      domains={'instagram.com'},
-                                     setup_retry=True,
                                      status_forcelist=(413, 429, 500, 502, 503, 504))
         self.failed_urls: set[str] = set()
         """Set of failed URLs."""
@@ -193,27 +195,29 @@ class InstagramClient:
             json.dump(top_comment_data, f, sort_keys=True, indent=2)
 
     def save_media(self, edge: Edge) -> None:
-        """Save media for an edge node."""
-        log.info('Saving media at URL: https://www.instagram.com/p/%s', edge['node']['code'])
-        media_info_url = f'https://www.instagram.com/p/{edge["node"]["code"]}/'
+        """
+        Save media for an edge node.
+
+        Raises
+        ------
+        UnexpectedRedirect
+            If a redirect occurs unexpectedly.
+        """
+        media_info_url = f'https://www.instagram.com/api/v1/media/{edge["node"]["pk"]}/info/'
+        log.info('Saving media at URL: %s', media_info_url)
         if self.is_saved(media_info_url):
             return
-        r = self.session.get(media_info_url, headers=PAGE_FETCH_HEADERS)
+        r = self.session.get(media_info_url, headers=API_HEADERS, allow_redirects=False)
         if r.status_code != HTTPStatus.OK:
+            if r.status_code in {HTTPStatus.MOVED_PERMANENTLY, HTTPStatus.FOUND}:
+                raise UnexpectedRedirect
             log.warning('GET request failed with status code %s.', r.status_code)
+            log.debug('Content: %s', r.text)
             return
         if 'image_versions2' not in r.text or 'taken_at' not in r.text:
             log.warning('Invalid response. image_versions2 dict not found.')
             return
-        soup = Soup(r.text, 'html5lib')
-        media_info_embedded = next(
-            json.loads(s) for s in (''.join(
-                getattr(c, 'text', '') for c in getattr(script, 'contents', ''))
-                                    for script in soup.select('script[type="application/json"]'))
-            if 'image_versions2' in s and 'taken_at' in s)
-        media_info: MediaInfo = (
-            media_info_embedded['require'][0][3][0]['__bbox']['require'][0][3][1]['__bbox']
-            ['result']['data']['xdt_api__v1__media__shortcode__web_info'])
+        media_info: MediaInfo = r.json()
         timestamp = media_info['items'][0]['taken_at']
         id_json_file = f'{edge["node"]["id"]}.json'
         media_info_json_file = f'{edge["node"]["id"]}-media-info-0000.json'
@@ -246,7 +250,7 @@ class InstagramClient:
                     else:
                         log.exception('Unknown shortcode.')
                 if edge['node'].get('video_dash_manifest'):
-                    self.add_video_url(f'https://www.instagram.com/p/{shortcode}')
+                    self.add_video_url(f'https://www.instagram.com/p/{shortcode}/')
                 else:
                     try:
                         self.save_comments(edge)
@@ -259,7 +263,7 @@ class InstagramClient:
                     'Unknown type: `%s`. Item %s will not be processed.',
                     edge['node']['__typename'], edge['node']['id'])
                 shortcode = edge['node']['code']
-                self.failed_urls.add(f'https://www.instagram.com/p/{shortcode}')
+                self.failed_urls.add(f'https://www.instagram.com/p/{shortcode}/')
 
     def get_json(self, url: str, *, cast_to: type[T], params: Mapping[str, str] | None = None) -> T:
         """Get JSON data from a URL."""

instagram_archiver/main.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING
 
 import click
 
+from .client import UnexpectedRedirect
 from .constants import BROWSER_CHOICES
 from .profile_scraper import ProfileScraper
 from .saved_scraper import SavedScraper
@@ -55,6 +56,9 @@ def main(output_dir: str,
                                         if '%(username)s' in output_dir else Path(output_dir)),
                             username=username) as client:
             client.process()
+    except UnexpectedRedirect as e:
+        click.echo('Unexpected redirect. Assuming request limit has been reached.', err=True)
+        raise click.Abort from e
     except Exception as e:
         if isinstance(e, KeyboardInterrupt) or debug:
             raise
@@ -91,6 +95,9 @@ def save_saved_main(output_dir: str,
     setup_logging(debug=debug)
     try:
         SavedScraper(browser, profile, output_dir, comments=include_comments).process(unsave=unsave)
+    except UnexpectedRedirect as e:
+        click.echo('Unexpected redirect. Assuming request limit has been reached.', err=True)
+        raise click.Abort from e
     except Exception as e:
         if isinstance(e, KeyboardInterrupt) or debug:
             raise

instagram_archiver/saved_scraper.py

@@ -20,7 +20,7 @@ __all__ = ('SavedScraper',)
 log = logging.getLogger(__name__)
 
 
-class SavedScraper(InstagramClient, SaveCommentsCheckDisabledMixin):
+class SavedScraper(SaveCommentsCheckDisabledMixin, InstagramClient):
     """Scrape saved posts."""
     def __init__(
         self,
@@ -69,6 +69,7 @@ class SavedScraper(InstagramClient, SaveCommentsCheckDisabledMixin):
                     'id': item['media']['id'],
                     'code': item['media']['code'],
                     'owner': item['media']['owner'],
+                    'pk': item['media']['pk'],
                     'video_dash_manifest': item['media'].get('video_dash_manifest')
                 }
             } for item in feed['items'])

instagram_archiver/typing.py

@@ -131,6 +131,8 @@ class XDTMediaDict(TypedDict):
     """Media ID."""
     owner: Owner
     """Owner information."""
+    pk: str
+    """Primary key. Also carousel ID."""
     video_dash_manifest: NotRequired[str | None]
     """Video dash manifest URL, if available."""
 

{instagram_archiver-0.3.0.dist-info → instagram_archiver-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: instagram-archiver
-Version: 0.3.0
+Version: 0.3.1
 Summary: Save Instagram content you have access to.
 License: MIT
 Keywords: command line,instagram
@@ -33,7 +33,7 @@ Description-Content-Type: text/markdown
 [![PyPI - Version](https://img.shields.io/pypi/v/instagram-archiver)](https://pypi.org/project/instagram-archiver/)
 [![GitHub tag (with filter)](https://img.shields.io/github/v/tag/Tatsh/instagram-archiver)](https://github.com/Tatsh/instagram-archiver/tags)
 [![License](https://img.shields.io/github/license/Tatsh/instagram-archiver)](https://github.com/Tatsh/instagram-archiver/blob/master/LICENSE.txt)
-[![GitHub commits since latest release (by SemVer including pre-releases)](https://img.shields.io/github/commits-since/Tatsh/instagram-archiver/v0.3.0/master)](https://github.com/Tatsh/instagram-archiver/compare/v0.3.0...master)
+[![GitHub commits since latest release (by SemVer including pre-releases)](https://img.shields.io/github/commits-since/Tatsh/instagram-archiver/v0.3.1/master)](https://github.com/Tatsh/instagram-archiver/compare/v0.3.1...master)
 [![QA](https://github.com/Tatsh/instagram-archiver/actions/workflows/qa.yml/badge.svg)](https://github.com/Tatsh/instagram-archiver/actions/workflows/qa.yml)
 [![Tests](https://github.com/Tatsh/instagram-archiver/actions/workflows/tests.yml/badge.svg)](https://github.com/Tatsh/instagram-archiver/actions/workflows/tests.yml)
 [![Coverage Status](https://coveralls.io/repos/github/Tatsh/instagram-archiver/badge.svg?branch=master)](https://coveralls.io/github/Tatsh/instagram-archiver?branch=master)

instagram_archiver-0.3.1.dist-info/RECORD

@@ -0,0 +1,15 @@
+instagram_archiver/__init__.py,sha256=FtYFsiDxC03kcF2BoC_fCDdSIV_Q81cMBQIeYhA9hwk,270
+instagram_archiver/__main__.py,sha256=oQ6s6zvZTBiEOgt-qep3bDY9ayxSanQr7KHzr6ENK0o,115
+instagram_archiver/client.py,sha256=xde2O-ADvyFw5q7bZxbX9R2ae02ToJUElw-IKSbQhss,11091
+instagram_archiver/constants.py,sha256=NJ8QlQZviY3dwwrIONThK_G9VcvAzOQM6Yg-hSyaj9A,1459
+instagram_archiver/main.py,sha256=5blHCN2PSNGdTlIvXPyWr5vbgM6fW1Wn9nbzJUtBS2Y,4173
+instagram_archiver/profile_scraper.py,sha256=BGnJZD3rF2e4aHpqSvG7UBoAiDjTyrpcYKSzsDgHO3M,7988
+instagram_archiver/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+instagram_archiver/saved_scraper.py,sha256=PyrRCQ00x65nyiIUvmMw5-paJ6_aVrl5xmcTrHi5tHo,2769
+instagram_archiver/typing.py,sha256=svEWoYwGXhkbdISw1r6tuOzo_5TwrhQTn3q0VxzBIlY,4359
+instagram_archiver/utils.py,sha256=l6f0W_brZhVPOjlKwoFYYum7ICyHJXpboTU7ANIQSPI,3842
+instagram_archiver-0.3.1.dist-info/LICENSE.txt,sha256=cDLmbhzFwEUz5FL_OnA6Jp9zdz80330J6YyEq-00yNQ,1093
+instagram_archiver-0.3.1.dist-info/METADATA,sha256=qoIzF1FkbuqwVB9npS6JE7xS5o-fVdVSf9k4XqvDISs,5925
+instagram_archiver-0.3.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+instagram_archiver-0.3.1.dist-info/entry_points.txt,sha256=kNXd0Sy6896DEBRcx2mVYiaE-OR9-XR56MpWuaNa49g,128
+instagram_archiver-0.3.1.dist-info/RECORD,,

instagram_archiver-0.3.0.dist-info/RECORD

@@ -1,15 +0,0 @@
-instagram_archiver/__init__.py,sha256=406v5g-tu6QhUfdMTNgI9rYbPYm7WfjiIL68HOXLXQY,270
-instagram_archiver/__main__.py,sha256=oQ6s6zvZTBiEOgt-qep3bDY9ayxSanQr7KHzr6ENK0o,115
-instagram_archiver/client.py,sha256=uMYembbrYWKJdXlBpuYsSK-p7tuj0qG3WMVV8M785NE,11326
-instagram_archiver/constants.py,sha256=NJ8QlQZviY3dwwrIONThK_G9VcvAzOQM6Yg-hSyaj9A,1459
-instagram_archiver/main.py,sha256=lW8rHPjQpqNH8TqjitW3DMILazn09sSWxJhVlkUC5Ck,3808
-instagram_archiver/profile_scraper.py,sha256=BGnJZD3rF2e4aHpqSvG7UBoAiDjTyrpcYKSzsDgHO3M,7988
-instagram_archiver/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-instagram_archiver/saved_scraper.py,sha256=PHSXkmK-MC-42Z4GZzNCXa8dSD682NEav1rms7mL-yk,2722
-instagram_archiver/typing.py,sha256=J8TcQftpIY-IVsbx007e1WbA-XdLUhBuEWWAisFlpHA,4306
-instagram_archiver/utils.py,sha256=l6f0W_brZhVPOjlKwoFYYum7ICyHJXpboTU7ANIQSPI,3842
-instagram_archiver-0.3.0.dist-info/LICENSE.txt,sha256=cDLmbhzFwEUz5FL_OnA6Jp9zdz80330J6YyEq-00yNQ,1093
-instagram_archiver-0.3.0.dist-info/METADATA,sha256=Vq8yWfCHOzOJduL9E0PEcIGz9qu900uiI96c1g0q3-I,5925
-instagram_archiver-0.3.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-instagram_archiver-0.3.0.dist-info/entry_points.txt,sha256=kNXd0Sy6896DEBRcx2mVYiaE-OR9-XR56MpWuaNa49g,128
-instagram_archiver-0.3.0.dist-info/RECORD,,