twitwi 0.18.2__tar.gz → 0.19.2__tar.gz

{twitwi-0.18.2/twitwi.egg-info → twitwi-0.19.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: twitwi
-Version: 0.18.2
+Version: 0.19.2
 Summary: A collection of Twitter-related helper functions for python.
 Home-page: http://github.com/medialab/twitwi
 Author: Béatrice Mazoyer, Guillaume Plique, Benjamin Ooghe-Tabanou

{twitwi-0.18.2 → twitwi-0.19.2}/setup.py

@@ -4,7 +4,7 @@ with open('./README.md', 'r') as f:
     long_description = f.read()
 
 setup(name='twitwi',
-      version='0.18.2',
+      version='0.19.2',
       description='A collection of Twitter-related helper functions for python.',
       long_description=long_description,
       long_description_content_type='text/markdown',

{twitwi-0.18.2 → twitwi-0.19.2}/twitwi/__init__.py

@@ -2,6 +2,7 @@
 # Twitwi Library Endpoint
 # =============================================================================
 #
+from twitwi.anonymizers import anonymize_normalized_tweet
 from twitwi.client_wrapper import TwitterWrapper
 from twitwi.formatters import (
     transform_tweet_into_csv_dict,

twitwi-0.19.2/twitwi/anonymizers.py

@@ -0,0 +1,71 @@
+import re
+
+QUOTED_REDACT_RE = re.compile(r"«\s+[^»]+:\s+([^»]+)\s+»")
+
+
+def redact_quoted_text(text: str) -> str:
+    return QUOTED_REDACT_RE.sub(r"« \1 »", text)
+
+
+def redact_rt_text(text: str) -> str:
+    return 'RT: ' + text.split(': ', 1)[1]
+
+
+FIELDS_TO_DELETE = [
+    # The tweet's url leaks the user
+    "url",
+
+    # User's place
+    "lat",
+    "lng",
+    "place_coordinates",
+    "place_country_code",
+    "place_name",
+    "place_type",
+    "user_location",
+
+    # User info
+    "user_created_at",
+    "user_description",
+    "user_id",
+    "user_image",
+    "user_name",
+    "user_screen_name",
+    "user_timestamp_utc",
+    "user_url",
+    "user_verified",
+
+    # Retweeted user info
+    "retweeted_timestamp_utc",
+    "retweeted_user",
+    "retweeted_user_id",
+
+    # Replied user info
+    "to_userid",
+    "to_username",
+
+    # Quoted user info
+    "quoted_user",
+    "quoted_user_id",
+    "quoted_timestamp_utc",
+]
+
+
+# NOTE: currently we still keep the id, but we should drop it
+# to really call this an anonymized tweet.
+# NOTE: we do not redact mentions either.
+# NOTE: we also don't redact replies.
+def anonymize_normalized_tweet(normalized_tweet) -> None:
+
+    # Text mangling
+    text = normalized_tweet["text"]
+
+    if normalized_tweet.get('retweeted_id', None) is not None:
+        normalized_tweet["text"] = redact_rt_text(text)
+
+    elif normalized_tweet.get('quoted_id', None) is not None:
+        normalized_tweet["text"] = redact_quoted_text(text)
+
+    for field in FIELDS_TO_DELETE:
+        if field in normalized_tweet:
+            del normalized_tweet[field]

{twitwi-0.18.2 → twitwi-0.19.2}/twitwi/normalizers.py

@@ -330,7 +330,11 @@ def normalize_tweet(tweet, locale=None, extract_referenced_tweets=False,
                     media_urls.append(med_url.split('?tag=')[0])
                     media_files.append('%s_%s' % (source_id, med_name))
                     media_alt_texts.append(entity.get("ext_alt_text") or '')
-            else:
+
+            # NOTE: fun fact, Twitter is starting to break down and we cannot guarantee
+            # expanded_url exists anymore. It even crashes the website itself lol:
+            # https://x.com/lmerzeau/status/426318495450943488
+            elif "expanded_url" in entity:
                 normalized = custom_normalize_url(entity['expanded_url'])
                 links.add(normalized)
 
@@ -459,8 +463,8 @@ def normalize_user(user, locale=None, pure=True, v2=False):
         'timestamp_utc': timestamp_utc,
         'local_time': local_time,
         'location': user.get('location'),
-        'verified': user.get('verified'),
-        'protected': user.get('protected'),
+        'verified': user.get('verified', False),
+        'protected': user.get('protected', False),
         'tweets': user['statuses_count'] if not v2 else user['public_metrics']['tweet_count'],
         'followers': user['followers_count'] if not v2 else user['public_metrics']['followers_count'],
         'friends': user['friends_count'] if not v2 else user['public_metrics']['following_count'],

{twitwi-0.18.2 → twitwi-0.19.2/twitwi.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: twitwi
-Version: 0.18.2
+Version: 0.19.2
 Summary: A collection of Twitter-related helper functions for python.
 Home-page: http://github.com/medialab/twitwi
 Author: Béatrice Mazoyer, Guillaume Plique, Benjamin Ooghe-Tabanou

{twitwi-0.18.2 → twitwi-0.19.2}/twitwi.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ LICENSE.txt
 README.md
 setup.py
 twitwi/__init__.py
+twitwi/anonymizers.py
 twitwi/client_wrapper.py
 twitwi/constants.py
 twitwi/exceptions.py