twitwi 0.19.2__py3-none-any.whl → 0.21.0__py3-none-any.whl

twitwi/client_wrapper.py

@@ -1,166 +0,0 @@
-# =============================================================================
-# Twitwi Client Wrapper
-# =============================================================================
-#
-# Wrapper for the `twitter` library API client able to rotate the two possible
-# endpoints to maximize throughput.
-#
-import json
-from time import sleep, time
-from operator import itemgetter
-from twitter import Twitter, OAuth, OAuth2, TwitterHTTPError, Twitter2
-
-from twitwi.exceptions import TwitterWrapperMaxAttemptsExceeded
-from twitwi.constants import APP_ONLY_ROUTES
-
-DEFAULT_MAX_ATTEMPTS = 5
-
-# Established from: https://developer.twitter.com/en/support/twitter-api/error-troubleshooting
-NO_RETRY_STATUSES = set([
-    400,
-    401,
-    403,
-    404,
-    406,
-    410,
-    422
-])
-
-
-class TwitterWrapper(object):
-
-    def __init__(self, token, token_secret, consumer_key, consumer_secret,
-                 listener=None, api_version='1.1'):
-
-        api_version = str(api_version)
-
-        if api_version not in ['1.1', '2']:
-            raise TypeError('API version can only be \'1.1\' or \'2\'.')
-
-        self.oauth = OAuth(
-            token,
-            token_secret,
-            consumer_key,
-            consumer_secret
-        )
-
-        bearer_token_client = Twitter(
-            api_version=None,
-            format='',
-            secure=True,
-            auth=OAuth2(consumer_key, consumer_secret)
-        )
-
-        bearer_token = json.loads(
-            bearer_token_client.oauth2.token(grant_type='client_credentials')
-        )['access_token']
-
-        self.oauth2 = OAuth2(bearer_token=bearer_token)
-
-        self.auth = {}
-        self.waits = {}
-
-        TwitterClass = Twitter
-
-        if api_version == '2':
-            TwitterClass = Twitter2
-
-            for route in APP_ONLY_ROUTES:
-                self.auth[route] = 'app'
-                self.waits[route] = {'app': 0}
-
-        self.endpoints = {
-            'user': TwitterClass(auth=self.oauth),
-            'app': TwitterClass(auth=self.oauth2)
-        }
-
-        self.listener = listener
-
-    def call(self, route, max_attempts=DEFAULT_MAX_ATTEMPTS, **kwargs):
-        attempts = 0
-
-        if not isinstance(route, list):
-            raise TypeError('twitwi.TwitterWrapper.call: expecting route as a list, such as ["friends", "ids"].')
-
-        route = '/'.join(route)
-
-        while attempts < max_attempts:
-
-            if route not in self.auth:
-                self.auth[route] = 'user'
-
-            auth = self.auth[route]
-
-            try:
-                return self.endpoints[auth].__getattr__(route)(**kwargs)
-
-            except TwitterHTTPError as e:
-
-                # Rate limited
-                if e.e.code == 429:
-                    now = int(time())
-
-                    # If there are still API calls available, we obviously
-                    # queried Twitter too fast and should just let it breathe
-                    remaining = int(e.e.headers['X-Rate-Limit-Remaining'])
-                    if remaining > 0:
-                        attempts += 1
-
-                        if callable(self.listener):
-                            self.listener('excessive-rate', {
-                                'error': e,
-                                'route': route,
-                                'attempts': attempts,
-                                'auth': auth
-                            })
-
-                        sleep(1)
-                        continue
-
-                    reset = int(e.e.headers['X-Rate-Limit-Reset'])
-
-                    if route not in self.waits:
-                        self.waits[route] = {'user': now, 'app': now}
-
-                    self.waits[route][auth] = reset
-
-                    if callable(self.listener):
-                        self.listener('rate-limited', {
-                            'route': route,
-                            'kwargs': kwargs,
-                            'reset': reset,
-                            'auth': auth
-                        })
-
-                    min_wait = min(self.waits[route].items(), key=itemgetter(1))
-
-                    if min_wait[1] > now:
-                        sleeptime = 5 + max(0, int(min_wait[1] - now))
-
-                        if callable(self.listener):
-                            self.listener('waiting', {
-                                'auth': min_wait[0],
-                                'reset': min_wait[1],
-                                'sleep': sleeptime
-                            })
-
-                        sleep(sleeptime)
-
-                    self.auth[route] = min_wait[0]
-
-                    continue
-
-                # Errors that should terminate immediately
-                elif e.e.code in NO_RETRY_STATUSES:
-                    raise e
-
-                # Different error
-                else:
-                    attempts += 1
-
-                    if callable(self.listener):
-                        self.listener('error', {
-                            'error': e
-                        })
-
-        raise TwitterWrapperMaxAttemptsExceeded

twitwi-0.19.2.dist-info/METADATA

@@ -1,146 +0,0 @@
-Metadata-Version: 2.1
-Name: twitwi
-Version: 0.19.2
-Summary: A collection of Twitter-related helper functions for python.
-Home-page: http://github.com/medialab/twitwi
-Author: Béatrice Mazoyer, Guillaume Plique, Benjamin Ooghe-Tabanou
-Author-email: guillaume.plique@sciencespo.fr
-License: MIT
-Keywords: twitter
-Requires-Python: >=3.4
-Description-Content-Type: text/markdown
-License-File: LICENSE.txt
-Requires-Dist: pytz >=2019.3
-Requires-Dist: twitter ==2.0a2
-Requires-Dist: ural >=0.31.1
-
-[![Build Status](https://github.com/medialab/twitwi/workflows/Tests/badge.svg)](https://github.com/medialab/twitwi/actions)
-
-# Twitwi
-
-A collection of Twitter-related helper functions for python.
-
-## Installation
-
-You can install `twitwi` with pip with the following command:
-
-```
-pip install twitwi
-```
-
-## Usage
-
-*Normalization functions*
-
-* [normalize_tweets_payload_v2](#normalize_tweets_payload_v2)
-
-*Formatting functions*
-
-* [transform_tweet_into_csv_dict](#transform_tweet_into_csv_dict)
-* [transform_user_into_csv_dict](#transform_user_into_csv_dict)
-* [format_tweet_as_csv_row](#format_tweet_as_csv_row)
-
-*Useful constants (under twitwi.constants)*
-
-* [TWEET_FIELDS](#tweet_fields)
-* [USER_FIELDS](#user_fields)
-
-### normalize_tweets_payload_v2
-
-Function taking an entire tweets payload from the v2 API and returning a list of the contained tweets normalized and structured in a way that makes further analysis of the data convenient.
-
-```python
-from twitwi import normalize_tweets_payload_v2
-
-# Normalizing an entire tweets payload to extract a list of tweets
-normalize_tweets_payload_v2(payload)
-
-# Normalizing an entire tweets payload to extract a list of tweets
-# as well as the referenced tweets (quoted, retweeted, etc.)
-normalize_tweets_payload_v2(payload, extract_referenced_tweets=True)
-
-# Converting found dates to a chosen timezone
-from pytz import timezone
-paris_tz = timezone('Europe/Paris')
-
-normalize_tweets_payload_v2(payload, locale=paris_tz)
-```
-
-*Arguments*
-
-* **payload** *(dict)*: tweets payload coming from Twitter API v2.
-* **locale** *(pytz.timezone, optional)*: timezone used to convert dates. If not given, will default to UTC.
-* **extract_referenced_tweets** *(bool, optional)*: whether to keep referenced tweets (retweeted, quoted etc.) in the output. Defaults to `False`.
-* **collection_source** *(string, optional): An optional information to add to the tweets to indicate whence you collected them.
-
-### transform_tweet_into_csv_dict
-
-Function transforming (i.e. mutating, so beware) a given normalized tweet into a suitable dict able to be written by a `csv.DictWriter` as a row.
-
-```python
-from twitwi import transform_tweet_into_csv_dict
-
-# The function returns nothing, `normalized_tweet` has been mutated
-transform_tweet_into_csv_dict(normalized_tweet)
-```
-
-### transform_user_into_csv_dict
-
-Function transforming (i.e. mutating, so beware) a given normalized Twitter user into a suitable dict able to be written by a `csv.DictWriter` as a row.
-
-```python
-from twitwi import transform_user_into_csv_dict
-
-# The function returns nothing, `normalized_user` has been mutated
-transform_user_into_csv_dict(normalized_user)
-```
-
-### format_tweet_as_csv_row
-
-Function formatting the given normalized tweet as a list able to be written by a `csv.writer` as a row.
-
-```python
-from twitwi import format_tweet_as_csv_row
-
-row = format_tweet_as_csv_row(normalized_tweet)
-```
-
-### format_user_as_csv_row
-
-Function formatting the given normalized Twitter user as a list able to be written by a `csv.writer` as a row.
-
-```python
-from twitwi import format_user_as_csv_row
-
-row = format_user_as_csv_row(normalized_user)
-```
-
-### TWEET_FIELDS
-
-List of tweet field names. Useful to declare headers with csv writers:
-
-```python
-from twitwi.constants import TWEET_FIELDS
-
-# Using csv.writer
-w = csv.writer(f)
-w.writerow(TWEET_FIELDS)
-
-# Using csv.DictWriter
-w = csv.DictWriter(f, fieldnames=TWEET_FIELDS)
-w.writeheader()
-```
-
-### USER_FIELDS
-
-```python
-from twitwi.constants import USER_FIELDS
-
-# Using csv.writer
-w = csv.writer(f)
-w.writerow(USER_FIELDS)
-
-# Using csv.DictWriter
-w = csv.DictWriter(f, fieldnames=USER_FIELDS)
-w.writeheader()
-```

twitwi-0.19.2.dist-info/RECORD

@@ -1,14 +0,0 @@
-twitwi/__init__.py,sha256=_QyXJBafpPuU_Q6iu6u7m-c7ud7JLFRnJE2jDUe1isM,754
-twitwi/anonymizers.py,sha256=Dt72fdAJ4RzAxj-ZRmCVUNNWstlgvlJUNmUk520COOI,1594
-twitwi/client_wrapper.py,sha256=vX3viQ97xTQ9JJfztnNRCGrfJm-XYpdswo6NDItXufQ,5056
-twitwi/constants.py,sha256=KG0yIyWaLlwpqxxud75fsEY3n3CPio6uFWqZL9I401Y,16509
-twitwi/exceptions.py,sha256=3AlspgS4u6Y2skbCKluVJnCQcnY-y7AI9UbMNMTn7Ak,532
-twitwi/formatters.py,sha256=rK4IBBH-0PMRxQdDZ5szHKBk-J-AzefN8Wdj79L7YiY,3111
-twitwi/normalizers.py,sha256=L4YcQZudizP7gdHQmhwafT6o39uGNk3cxtOcA9U7OKE,27694
-twitwi/utils.py,sha256=iBxlCKjwPOz3RKIa8I_vs7d1miRi-eMXMpobKC5apZI,2349
-twitwi-0.19.2.dist-info/LICENSE.txt,sha256=Ddg_PcGnl0qd2167o2dheCjE_rCZJOoBxjJnJhhOpX4,1099
-twitwi-0.19.2.dist-info/METADATA,sha256=wigdEBMaf3jq-z89cWt2F3lhuQkWL9-lb6qeu0WbWuI,4159
-twitwi-0.19.2.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
-twitwi-0.19.2.dist-info/top_level.txt,sha256=kellKVCFa_xo69jlXUtZXY2RYDd9ywdBe_-_iYK4W74,7
-twitwi-0.19.2.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-twitwi-0.19.2.dist-info/RECORD,,