twitwi 0.19.0__tar.gz → 0.20.0__tar.gz

{twitwi-0.19.0/twitwi.egg-info → twitwi-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: twitwi
-Version: 0.19.0
+Version: 0.20.0
 Summary: A collection of Twitter-related helper functions for python.
 Home-page: http://github.com/medialab/twitwi
 Author: Béatrice Mazoyer, Guillaume Plique, Benjamin Ooghe-Tabanou
@@ -11,8 +11,18 @@ Requires-Python: >=3.4
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: pytz>=2019.3
-Requires-Dist: twitter==2.0a2
 Requires-Dist: ural>=0.31.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 [![Build Status](https://github.com/medialab/twitwi/workflows/Tests/badge.svg)](https://github.com/medialab/twitwi/actions)
 

{twitwi-0.19.0 → twitwi-0.20.0}/setup.py

@@ -4,7 +4,7 @@ with open('./README.md', 'r') as f:
     long_description = f.read()
 
 setup(name='twitwi',
-      version='0.19.0',
+      version='0.20.0',
       description='A collection of Twitter-related helper functions for python.',
       long_description=long_description,
       long_description_content_type='text/markdown',
@@ -18,7 +18,6 @@ setup(name='twitwi',
       package_data={'docs': ['README.md']},
       install_requires=[
         'pytz>=2019.3',
-        'twitter==2.0a2',
         'ural>=0.31.1'
       ],
       zip_safe=True)

{twitwi-0.19.0 → twitwi-0.20.0}/twitwi/__init__.py

@@ -3,7 +3,6 @@
 # =============================================================================
 #
 from twitwi.anonymizers import anonymize_normalized_tweet
-from twitwi.client_wrapper import TwitterWrapper
 from twitwi.formatters import (
     transform_tweet_into_csv_dict,
     format_tweet_as_csv_row,

{twitwi-0.19.0 → twitwi-0.20.0}/twitwi/constants.py

@@ -426,11 +426,5 @@ LIST_TWEETS_OR_MEMBERS_PARAMS = {
     'user.fields': ','.join(USER_FIELDS_V2)
 }
 
-APP_ONLY_ROUTES = {
-    'tweets/counts/recent',
-    'tweets/counts/all',
-    'tweets/search/all'
-}
-
 PRE_SNOWFLAKE_LAST_TWEET_ID = 29700859247
 OFFSET_TIMESTAMP = 1288834974657

{twitwi-0.19.0 → twitwi-0.20.0}/twitwi/exceptions.py

@@ -10,10 +10,6 @@ class TwitwiError(Exception):
     pass
 
 
-class TwitterWrapperMaxAttemptsExceeded(TwitwiError):
-    pass
-
-
 class TwitterPayloadV2IncompleteIncludesError(TwitwiError):
     def __init__(self, kind, key):
         self.kind = kind

{twitwi-0.19.0 → twitwi-0.20.0}/twitwi/normalizers.py

@@ -330,7 +330,11 @@ def normalize_tweet(tweet, locale=None, extract_referenced_tweets=False,
                     media_urls.append(med_url.split('?tag=')[0])
                     media_files.append('%s_%s' % (source_id, med_name))
                     media_alt_texts.append(entity.get("ext_alt_text") or '')
-            else:
+
+            # NOTE: fun fact, Twitter is starting to break down and we cannot guarantee
+            # expanded_url exists anymore. It even crashes the website itself lol:
+            # https://x.com/lmerzeau/status/426318495450943488
+            elif "expanded_url" in entity:
                 normalized = custom_normalize_url(entity['expanded_url'])
                 links.add(normalized)
 

{twitwi-0.19.0 → twitwi-0.20.0/twitwi.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: twitwi
-Version: 0.19.0
+Version: 0.20.0
 Summary: A collection of Twitter-related helper functions for python.
 Home-page: http://github.com/medialab/twitwi
 Author: Béatrice Mazoyer, Guillaume Plique, Benjamin Ooghe-Tabanou
@@ -11,8 +11,18 @@ Requires-Python: >=3.4
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: pytz>=2019.3
-Requires-Dist: twitter==2.0a2
 Requires-Dist: ural>=0.31.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 [![Build Status](https://github.com/medialab/twitwi/workflows/Tests/badge.svg)](https://github.com/medialab/twitwi/actions)
 

{twitwi-0.19.0 → twitwi-0.20.0}/twitwi.egg-info/SOURCES.txt

@@ -3,7 +3,6 @@ README.md
 setup.py
 twitwi/__init__.py
 twitwi/anonymizers.py
-twitwi/client_wrapper.py
 twitwi/constants.py
 twitwi/exceptions.py
 twitwi/formatters.py

{twitwi-0.19.0 → twitwi-0.20.0}/twitwi.egg-info/requires.txt

@@ -1,3 +1,2 @@
 pytz>=2019.3
-twitter==2.0a2
 ural>=0.31.1

twitwi-0.19.0/twitwi/client_wrapper.py

@@ -1,166 +0,0 @@
-# =============================================================================
-# Twitwi Client Wrapper
-# =============================================================================
-#
-# Wrapper for the `twitter` library API client able to rotate the two possible
-# endpoints to maximize throughput.
-#
-import json
-from time import sleep, time
-from operator import itemgetter
-from twitter import Twitter, OAuth, OAuth2, TwitterHTTPError, Twitter2
-
-from twitwi.exceptions import TwitterWrapperMaxAttemptsExceeded
-from twitwi.constants import APP_ONLY_ROUTES
-
-DEFAULT_MAX_ATTEMPTS = 5
-
-# Established from: https://developer.twitter.com/en/support/twitter-api/error-troubleshooting
-NO_RETRY_STATUSES = set([
-    400,
-    401,
-    403,
-    404,
-    406,
-    410,
-    422
-])
-
-
-class TwitterWrapper(object):
-
-    def __init__(self, token, token_secret, consumer_key, consumer_secret,
-                 listener=None, api_version='1.1'):
-
-        api_version = str(api_version)
-
-        if api_version not in ['1.1', '2']:
-            raise TypeError('API version can only be \'1.1\' or \'2\'.')
-
-        self.oauth = OAuth(
-            token,
-            token_secret,
-            consumer_key,
-            consumer_secret
-        )
-
-        bearer_token_client = Twitter(
-            api_version=None,
-            format='',
-            secure=True,
-            auth=OAuth2(consumer_key, consumer_secret)
-        )
-
-        bearer_token = json.loads(
-            bearer_token_client.oauth2.token(grant_type='client_credentials')
-        )['access_token']
-
-        self.oauth2 = OAuth2(bearer_token=bearer_token)
-
-        self.auth = {}
-        self.waits = {}
-
-        TwitterClass = Twitter
-
-        if api_version == '2':
-            TwitterClass = Twitter2
-
-            for route in APP_ONLY_ROUTES:
-                self.auth[route] = 'app'
-                self.waits[route] = {'app': 0}
-
-        self.endpoints = {
-            'user': TwitterClass(auth=self.oauth),
-            'app': TwitterClass(auth=self.oauth2)
-        }
-
-        self.listener = listener
-
-    def call(self, route, max_attempts=DEFAULT_MAX_ATTEMPTS, **kwargs):
-        attempts = 0
-
-        if not isinstance(route, list):
-            raise TypeError('twitwi.TwitterWrapper.call: expecting route as a list, such as ["friends", "ids"].')
-
-        route = '/'.join(route)
-
-        while attempts < max_attempts:
-
-            if route not in self.auth:
-                self.auth[route] = 'user'
-
-            auth = self.auth[route]
-
-            try:
-                return self.endpoints[auth].__getattr__(route)(**kwargs)
-
-            except TwitterHTTPError as e:
-
-                # Rate limited
-                if e.e.code == 429:
-                    now = int(time())
-
-                    # If there are still API calls available, we obviously
-                    # queried Twitter too fast and should just let it breathe
-                    remaining = int(e.e.headers['X-Rate-Limit-Remaining'])
-                    if remaining > 0:
-                        attempts += 1
-
-                        if callable(self.listener):
-                            self.listener('excessive-rate', {
-                                'error': e,
-                                'route': route,
-                                'attempts': attempts,
-                                'auth': auth
-                            })
-
-                        sleep(1)
-                        continue
-
-                    reset = int(e.e.headers['X-Rate-Limit-Reset'])
-
-                    if route not in self.waits:
-                        self.waits[route] = {'user': now, 'app': now}
-
-                    self.waits[route][auth] = reset
-
-                    if callable(self.listener):
-                        self.listener('rate-limited', {
-                            'route': route,
-                            'kwargs': kwargs,
-                            'reset': reset,
-                            'auth': auth
-                        })
-
-                    min_wait = min(self.waits[route].items(), key=itemgetter(1))
-
-                    if min_wait[1] > now:
-                        sleeptime = 5 + max(0, int(min_wait[1] - now))
-
-                        if callable(self.listener):
-                            self.listener('waiting', {
-                                'auth': min_wait[0],
-                                'reset': min_wait[1],
-                                'sleep': sleeptime
-                            })
-
-                        sleep(sleeptime)
-
-                    self.auth[route] = min_wait[0]
-
-                    continue
-
-                # Errors that should terminate immediately
-                elif e.e.code in NO_RETRY_STATUSES:
-                    raise e
-
-                # Different error
-                else:
-                    attempts += 1
-
-                    if callable(self.listener):
-                        self.listener('error', {
-                            'error': e
-                        })
-
-        raise TwitterWrapperMaxAttemptsExceeded