twitwi 0.19.0__py3-none-any.whl → 0.20.0__py3-none-any.whl

twitwi/__init__.py

@@ -3,7 +3,6 @@
 # =============================================================================
 #
 from twitwi.anonymizers import anonymize_normalized_tweet
-from twitwi.client_wrapper import TwitterWrapper
 from twitwi.formatters import (
     transform_tweet_into_csv_dict,
     format_tweet_as_csv_row,

twitwi/constants.py

@@ -426,11 +426,5 @@ LIST_TWEETS_OR_MEMBERS_PARAMS = {
     'user.fields': ','.join(USER_FIELDS_V2)
 }
 
-APP_ONLY_ROUTES = {
-    'tweets/counts/recent',
-    'tweets/counts/all',
-    'tweets/search/all'
-}
-
 PRE_SNOWFLAKE_LAST_TWEET_ID = 29700859247
 OFFSET_TIMESTAMP = 1288834974657

twitwi/exceptions.py

@@ -10,10 +10,6 @@ class TwitwiError(Exception):
     pass
 
 
-class TwitterWrapperMaxAttemptsExceeded(TwitwiError):
-    pass
-
-
 class TwitterPayloadV2IncompleteIncludesError(TwitwiError):
     def __init__(self, kind, key):
         self.kind = kind

twitwi/normalizers.py

@@ -330,7 +330,11 @@ def normalize_tweet(tweet, locale=None, extract_referenced_tweets=False,
                     media_urls.append(med_url.split('?tag=')[0])
                     media_files.append('%s_%s' % (source_id, med_name))
                     media_alt_texts.append(entity.get("ext_alt_text") or '')
-            else:
+
+            # NOTE: fun fact, Twitter is starting to break down and we cannot guarantee
+            # expanded_url exists anymore. It even crashes the website itself lol:
+            # https://x.com/lmerzeau/status/426318495450943488
+            elif "expanded_url" in entity:
                 normalized = custom_normalize_url(entity['expanded_url'])
                 links.add(normalized)
 

{twitwi-0.19.0.dist-info → twitwi-0.20.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: twitwi
-Version: 0.19.0
+Version: 0.20.0
 Summary: A collection of Twitter-related helper functions for python.
 Home-page: http://github.com/medialab/twitwi
 Author: Béatrice Mazoyer, Guillaume Plique, Benjamin Ooghe-Tabanou
@@ -10,9 +10,19 @@ Keywords: twitter
 Requires-Python: >=3.4
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: pytz >=2019.3
-Requires-Dist: twitter ==2.0a2
-Requires-Dist: ural >=0.31.1
+Requires-Dist: pytz>=2019.3
+Requires-Dist: ural>=0.31.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 [![Build Status](https://github.com/medialab/twitwi/workflows/Tests/badge.svg)](https://github.com/medialab/twitwi/actions)
 

twitwi-0.20.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+twitwi/__init__.py,sha256=sAiVmg5YzZXFHUlCjm7oAccGoUZP0j19oyO37YGol-o,705
+twitwi/anonymizers.py,sha256=Dt72fdAJ4RzAxj-ZRmCVUNNWstlgvlJUNmUk520COOI,1594
+twitwi/constants.py,sha256=3kuQk_dijOJUKkSqBp7N4sHHNoDeAJWOQSDcnh7wSrg,16409
+twitwi/exceptions.py,sha256=kMHoWs57mpOzQGcGw4B3izCFVFzW5WIN1wgTbrP0Z2w,467
+twitwi/formatters.py,sha256=rK4IBBH-0PMRxQdDZ5szHKBk-J-AzefN8Wdj79L7YiY,3111
+twitwi/normalizers.py,sha256=L4YcQZudizP7gdHQmhwafT6o39uGNk3cxtOcA9U7OKE,27694
+twitwi/utils.py,sha256=iBxlCKjwPOz3RKIa8I_vs7d1miRi-eMXMpobKC5apZI,2349
+twitwi-0.20.0.dist-info/licenses/LICENSE.txt,sha256=Ddg_PcGnl0qd2167o2dheCjE_rCZJOoBxjJnJhhOpX4,1099
+twitwi-0.20.0.dist-info/METADATA,sha256=dVu4IykGtQyVd8-DCFBe3cXzOyd3jKcprEiy-38Jsvo,4360
+twitwi-0.20.0.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+twitwi-0.20.0.dist-info/top_level.txt,sha256=kellKVCFa_xo69jlXUtZXY2RYDd9ywdBe_-_iYK4W74,7
+twitwi-0.20.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+twitwi-0.20.0.dist-info/RECORD,,

{twitwi-0.19.0.dist-info → twitwi-0.20.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: setuptools (77.0.3)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

twitwi/client_wrapper.py

@@ -1,166 +0,0 @@
-# =============================================================================
-# Twitwi Client Wrapper
-# =============================================================================
-#
-# Wrapper for the `twitter` library API client able to rotate the two possible
-# endpoints to maximize throughput.
-#
-import json
-from time import sleep, time
-from operator import itemgetter
-from twitter import Twitter, OAuth, OAuth2, TwitterHTTPError, Twitter2
-
-from twitwi.exceptions import TwitterWrapperMaxAttemptsExceeded
-from twitwi.constants import APP_ONLY_ROUTES
-
-DEFAULT_MAX_ATTEMPTS = 5
-
-# Established from: https://developer.twitter.com/en/support/twitter-api/error-troubleshooting
-NO_RETRY_STATUSES = set([
-    400,
-    401,
-    403,
-    404,
-    406,
-    410,
-    422
-])
-
-
-class TwitterWrapper(object):
-
-    def __init__(self, token, token_secret, consumer_key, consumer_secret,
-                 listener=None, api_version='1.1'):
-
-        api_version = str(api_version)
-
-        if api_version not in ['1.1', '2']:
-            raise TypeError('API version can only be \'1.1\' or \'2\'.')
-
-        self.oauth = OAuth(
-            token,
-            token_secret,
-            consumer_key,
-            consumer_secret
-        )
-
-        bearer_token_client = Twitter(
-            api_version=None,
-            format='',
-            secure=True,
-            auth=OAuth2(consumer_key, consumer_secret)
-        )
-
-        bearer_token = json.loads(
-            bearer_token_client.oauth2.token(grant_type='client_credentials')
-        )['access_token']
-
-        self.oauth2 = OAuth2(bearer_token=bearer_token)
-
-        self.auth = {}
-        self.waits = {}
-
-        TwitterClass = Twitter
-
-        if api_version == '2':
-            TwitterClass = Twitter2
-
-            for route in APP_ONLY_ROUTES:
-                self.auth[route] = 'app'
-                self.waits[route] = {'app': 0}
-
-        self.endpoints = {
-            'user': TwitterClass(auth=self.oauth),
-            'app': TwitterClass(auth=self.oauth2)
-        }
-
-        self.listener = listener
-
-    def call(self, route, max_attempts=DEFAULT_MAX_ATTEMPTS, **kwargs):
-        attempts = 0
-
-        if not isinstance(route, list):
-            raise TypeError('twitwi.TwitterWrapper.call: expecting route as a list, such as ["friends", "ids"].')
-
-        route = '/'.join(route)
-
-        while attempts < max_attempts:
-
-            if route not in self.auth:
-                self.auth[route] = 'user'
-
-            auth = self.auth[route]
-
-            try:
-                return self.endpoints[auth].__getattr__(route)(**kwargs)
-
-            except TwitterHTTPError as e:
-
-                # Rate limited
-                if e.e.code == 429:
-                    now = int(time())
-
-                    # If there are still API calls available, we obviously
-                    # queried Twitter too fast and should just let it breathe
-                    remaining = int(e.e.headers['X-Rate-Limit-Remaining'])
-                    if remaining > 0:
-                        attempts += 1
-
-                        if callable(self.listener):
-                            self.listener('excessive-rate', {
-                                'error': e,
-                                'route': route,
-                                'attempts': attempts,
-                                'auth': auth
-                            })
-
-                        sleep(1)
-                        continue
-
-                    reset = int(e.e.headers['X-Rate-Limit-Reset'])
-
-                    if route not in self.waits:
-                        self.waits[route] = {'user': now, 'app': now}
-
-                    self.waits[route][auth] = reset
-
-                    if callable(self.listener):
-                        self.listener('rate-limited', {
-                            'route': route,
-                            'kwargs': kwargs,
-                            'reset': reset,
-                            'auth': auth
-                        })
-
-                    min_wait = min(self.waits[route].items(), key=itemgetter(1))
-
-                    if min_wait[1] > now:
-                        sleeptime = 5 + max(0, int(min_wait[1] - now))
-
-                        if callable(self.listener):
-                            self.listener('waiting', {
-                                'auth': min_wait[0],
-                                'reset': min_wait[1],
-                                'sleep': sleeptime
-                            })
-
-                        sleep(sleeptime)
-
-                    self.auth[route] = min_wait[0]
-
-                    continue
-
-                # Errors that should terminate immediately
-                elif e.e.code in NO_RETRY_STATUSES:
-                    raise e
-
-                # Different error
-                else:
-                    attempts += 1
-
-                    if callable(self.listener):
-                        self.listener('error', {
-                            'error': e
-                        })
-
-        raise TwitterWrapperMaxAttemptsExceeded

twitwi-0.19.0.dist-info/RECORD

@@ -1,14 +0,0 @@
-twitwi/__init__.py,sha256=_QyXJBafpPuU_Q6iu6u7m-c7ud7JLFRnJE2jDUe1isM,754
-twitwi/anonymizers.py,sha256=Dt72fdAJ4RzAxj-ZRmCVUNNWstlgvlJUNmUk520COOI,1594
-twitwi/client_wrapper.py,sha256=vX3viQ97xTQ9JJfztnNRCGrfJm-XYpdswo6NDItXufQ,5056
-twitwi/constants.py,sha256=KG0yIyWaLlwpqxxud75fsEY3n3CPio6uFWqZL9I401Y,16509
-twitwi/exceptions.py,sha256=3AlspgS4u6Y2skbCKluVJnCQcnY-y7AI9UbMNMTn7Ak,532
-twitwi/formatters.py,sha256=rK4IBBH-0PMRxQdDZ5szHKBk-J-AzefN8Wdj79L7YiY,3111
-twitwi/normalizers.py,sha256=S6_ImrIZoIdZlw7j1D0w-Mh7tKGw1_FRKeuon5pJ5zw,27434
-twitwi/utils.py,sha256=iBxlCKjwPOz3RKIa8I_vs7d1miRi-eMXMpobKC5apZI,2349
-twitwi-0.19.0.dist-info/LICENSE.txt,sha256=Ddg_PcGnl0qd2167o2dheCjE_rCZJOoBxjJnJhhOpX4,1099
-twitwi-0.19.0.dist-info/METADATA,sha256=DX-z5q1wW7tN_9Tm2f4p-IfN3RAccxGTdr4W2C_Zx4A,4159
-twitwi-0.19.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-twitwi-0.19.0.dist-info/top_level.txt,sha256=kellKVCFa_xo69jlXUtZXY2RYDd9ywdBe_-_iYK4W74,7
-twitwi-0.19.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-twitwi-0.19.0.dist-info/RECORD,,