twitwi 0.21.0__py3-none-any.whl → 0.21.1__py3-none-any.whl

twitwi/bluesky/normalizers.py

@@ -15,6 +15,7 @@ from twitwi.bluesky.utils import (
     format_post_url,
     parse_post_url,
     parse_post_uri,
+    format_starterpack_url,
     format_media_url,
 )
 from twitwi.bluesky.types import BlueskyProfile, BlueskyPost
@@ -37,11 +38,11 @@ def normalize_profile(data: Dict, locale: Optional[str] = None) -> BlueskyProfil
         "did": data["did"],
         "url": format_profile_url(data["handle"]),
         "handle": data["handle"],
-        "display_name": data["displayName"],
+        "display_name": data.get("displayName", ""),
         "created_at": created_at,
         "timestamp_utc": timestamp_utc,
         "description": data["description"],
-        "avatar": data["avatar"],
+        "avatar": data.get("avatar", ""),
         "posts": data["postsCount"],
         "followers": data["followersCount"],
         "follows": data["followsCount"],
@@ -55,8 +56,13 @@ def normalize_profile(data: Dict, locale: Optional[str] = None) -> BlueskyProfil
 
 
 def prepare_native_gif_as_media(gif_data, user_did, source):
-    media_cid = gif_data["thumb"]["ref"]["$link"]
-    _, thumbnail = format_media_url(user_did, media_cid, "image/jpeg", source)
+    if "thumb" in gif_data:
+        media_cid = gif_data["thumb"]["ref"]["$link"]
+        _, thumbnail = format_media_url(user_did, media_cid, "image/jpeg", source)
+    else:
+        media_cid = ""
+        thumbnail = ""
+
     return {
         "id": media_cid,
         "type": "video/gif",
@@ -81,6 +87,18 @@ def prepare_video_as_media(video_data):
     }
 
 
+def process_starterpack_card(embed_data, post):
+    # Warning: mutates post
+
+    card = embed_data.get("record", {})
+    creator_did, pack_did = parse_post_uri(embed_data["uri"])
+    post["card_link"] = format_starterpack_url(embed_data.get("creator", {}).get("handle") or creator_did, pack_did)
+    post["card_title"] = card.get("name", "")
+    post["card_description"] = card.get("description", "")
+    post["card_thumbnail"] = card.get("thumb", "")
+    return post
+
+
 def process_card_data(embed_data, post):
     # Warning: mutates post
 
@@ -122,10 +140,12 @@ def prepare_quote_data(embed_quote, card_data, post, links):
                 break
 
         # Remove quoted link from post links
-        links.remove(post["quoted_url"])
+        if post["quoted_url"] in links:
+            links.remove(post["quoted_url"])
 
         # Extract user handle from url
-        post["quoted_user_handle"], _ = parse_post_url(post["quoted_url"], post["url"])
+        if "did:plc:" not in post["quoted_url"]:
+            post["quoted_user_handle"], _ = parse_post_url(post["quoted_url"], post["url"])
 
     return (post, quoted_data, links)
 
@@ -255,8 +275,8 @@ def normalize_post(
         )
 
     # Handle user metadata
-    post["user_diplay_name"] = data["author"]["displayName"]
-    post["user_avatar"] = data["author"]["avatar"]
+    post["user_diplay_name"] = data["author"].get("displayName", "")
+    post["user_avatar"] = data["author"].get("avatar", "")
     post["user_timestamp_utc"], post["user_created_at"] = get_dates(
         data["author"]["createdAt"], locale=locale, source="bluesky"
     )
@@ -312,15 +332,32 @@ def normalize_post(
 
         # Links
         elif feat["$type"].endswith("#link"):
+            # Handle native polls
+            if "https://poll.blue/" in feat["uri"]:
+                if feat["uri"].endswith("/0"):
+                    links.add(custom_normalize_url(feat["uri"]))
+                    text += b" %s" % feat["uri"].encode("utf-8")
+                continue
+
             links.add(custom_normalize_url(feat["uri"]))
+            # Check & fix occasional errored link positioning
+            # example: https://bsky.app/profile/ecrime.ch/post/3lqotmopayr23
+            byteStart = facet["index"]["byteStart"]
+            if b" " in text[byteStart : facet["index"]["byteEnd"]]:
+                byteStart = text.find(b"http", byteStart)
+
             links_to_replace.append(
                 {
                     "uri": feat["uri"].encode("utf-8"),
-                    "start": facet["index"]["byteStart"],
-                    "end": facet["index"]["byteEnd"],
+                    "start": byteStart,
+                    "end": byteStart - facet["index"]["byteStart"] + facet["index"]["byteEnd"],
                 }
             )
 
+        elif feat["$type"].endswith("#bold"):
+            pass
+        elif feat["$type"].endswith("#option"):
+            pass
         else:
             raise BlueskyPayloadError(
                 post["url"], "unusual record facet feature $type: %s" % feat
@@ -329,7 +366,10 @@ def normalize_post(
 
     # Rewrite full links within post's text
     for link in sorted(links_to_replace, key=lambda x: x["start"], reverse=True):
-        text = text[: link["start"]] + link["uri"] + text[link["end"] :]
+        if link["start"] < 0:
+            text = text + b" " + link["uri"]
+        else:
+            text = text[: link["start"]] + link["uri"] + text[link["end"] :]
 
     # Handle thread info when applicable
     # Unfortunately posts' payload only provide at uris for these so we do not have the handles
@@ -399,11 +439,16 @@ def normalize_post(
         if embed["$type"].endswith(".video"):
             media_data.append(prepare_video_as_media(embed["video"]))
 
-        # Quote
+        # Quote & Starter-packs
         if embed["$type"].endswith(".record"):
-            post, quoted_data, links = prepare_quote_data(
-                embed["record"], data.get("embed", {}).get("record"), post, links
-            )
+            if "app.bsky.graph.starterpack" in embed["record"]["uri"]:
+                post = process_starterpack_card(data.get("embed", {}).get("record"), post)
+                if post["card_link"]:
+                    extra_links.append(post["card_link"])
+            else:
+                post, quoted_data, links = prepare_quote_data(
+                    embed["record"], data.get("embed", {}).get("record"), post, links
+                )
 
         # Quote with medias
         if embed["$type"].endswith(".recordWithMedia"):
@@ -482,7 +527,7 @@ def normalize_post(
                 ).encode("utf-8")
 
         # Process quotes
-        if quoted_data:
+        if quoted_data and "value" in quoted_data:
             if quoted_data["cid"] != post["quoted_cid"]:
                 raise BlueskyPayloadError(
                     post["url"],

twitwi/bluesky/types.py

@@ -9,7 +9,7 @@ class BlueskyProfile(TypedDict):
     did: str                            # persistent long-term identifier of the account
     url: str                            # URL of the profile accessible on the web
     handle: str                         # updatable human-readable username of the account (usually like username.bsky.social or username.com)
-    display_name: str                   # updatable human-readable name of the account
+    display_name: Optional[str]         # updatable human-readable name of the account
     description: str                    # profile short description written by the user
     posts: int                          # total number of posts submitted by the user (at collection time)
     followers: int                      # total number of followers of the user (at collection time)
@@ -17,7 +17,7 @@ class BlueskyProfile(TypedDict):
     lists: int                          # total number of lists created by the user (at collection time)
     feedgens: int                       # total number of custom feeds created by the user (at collection time)
     starter_packs: int                  # total number of starter packs created by the user (at collection time)
-    avatar: str                         # URL to the image serving as avatar to the user
+    avatar: Optional[str]               # URL to the image serving as avatar to the user
     banner: str                         # URL to the image serving as profile banner to the user
     pinned_post_uri: Optional[str]      # ATProto's internal URI to the post potentially pinned by the user to appear at the top of his posts on his profile
     created_at: str                     # datetime (potentially timezoned) of when the user created the account
@@ -63,7 +63,7 @@ class BlueskyPost(TypedDict):
     # user_follows: int                 # not available from posts payloads
     # user_lists: int                   # not available from posts payloads
     user_langs: List[str]               # languages in which the author of the posts usually writes posts (declarative)
-    user_avatar: str                    # URL to the image serving as avatar to the user who authored the post
+    user_avatar: Optional[str]          # URL to the image serving as avatar to the user who authored the post
     user_created_at: str                # datetime (potentially timezoned) ofwhen the user who authored the post created the account
     user_timestamp_utc: int             # Unix UTC timestamp of when the user who authored the post created the account
 

twitwi/bluesky/utils.py

@@ -17,7 +17,7 @@ valid_post_keys = [
 valid_record_keys = ["$type", "createdAt", "text"]
 
 
-valid_author_keys = ["did", "handle", "displayName", "avatar", "createdAt"]
+valid_author_keys = ["did", "handle", "createdAt"]
 
 
 def validate_post_payload(data):
@@ -81,6 +81,9 @@ def parse_post_url(url, source):
 def parse_post_uri(uri, source=None):
     """Returns a tuple of (author_did, post_did) from an at:// post URI"""
 
+    if uri.startswith("at://") and "/app.bsky.graph.starterpack/" in uri:
+        return uri[5:].split("/app.bsky.graph.starterpack/")
+
     if not uri.startswith("at://") and "/app.bsky.feed.post/" not in uri:
         raise BlueskyPayloadError(
             source or uri, f"{uri} is not a usual Bluesky post uri"
@@ -88,6 +91,10 @@ def parse_post_uri(uri, source=None):
     return uri[5:].split("/app.bsky.feed.post/")
 
 
+def format_starterpack_url(user_handle_or_did, record_did):
+    return f"https://bsky.app/starter-pack/{user_handle_or_did}/{record_did}"
+
+
 def format_media_url(user_did, media_cid, mime_type, source):
     media_type = mime_type.split("/")[1]
     if mime_type.startswith("image"):

twitwi/constants.py

@@ -6,7 +6,6 @@
 #
 SOURCE_DATETIME_FORMAT = "%a %b %d %H:%M:%S +0000 %Y"
 SOURCE_DATETIME_FORMAT_V2 = "%Y-%m-%dT%H:%M:%S.%fZ"
-SOURCE_DATETIME_FORMAT_V3 = "%Y-%m-%dT%H:%M:%SZ"
 FORMATTED_TWEET_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
 
 FORMATTED_FULL_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"

twitwi/utils.py

@@ -5,6 +5,7 @@
 # Miscellaneous utility functions.
 #
 from pytz import timezone
+from dateutil.parser import parse as parse_date
 from ural import normalize_url, get_normalized_hostname
 from functools import partial
 from datetime import datetime
@@ -12,7 +13,6 @@ from datetime import datetime
 from twitwi.constants import (
     SOURCE_DATETIME_FORMAT,
     SOURCE_DATETIME_FORMAT_V2,
-    SOURCE_DATETIME_FORMAT_V3,
     FORMATTED_TWEET_DATETIME_FORMAT,
     FORMATTED_FULL_DATETIME_FORMAT,
     CANONICAL_URL_KWARGS,
@@ -47,12 +47,13 @@ def get_dates(date_str, locale=None, source="v1"):
             SOURCE_DATETIME_FORMAT if source == "v1" else SOURCE_DATETIME_FORMAT_V2,
         )
     except ValueError as e:
-        if source == "bluesky":
-            parsed_datetime = datetime.strptime(date_str, SOURCE_DATETIME_FORMAT_V3)
-        else:
+        if source != "bluesky":
             raise e
+        parsed_datetime = parse_date(date_str)
 
-    utc_datetime = UTC_TIMEZONE.localize(parsed_datetime)
+    utc_datetime = parsed_datetime
+    if not parsed_datetime.tzinfo:
+        utc_datetime = UTC_TIMEZONE.localize(parsed_datetime)
     locale_datetime = utc_datetime.astimezone(locale)
 
     return (

{twitwi-0.21.0.dist-info → twitwi-0.21.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: twitwi
-Version: 0.21.0
+Version: 0.21.1
 Summary: A collection of Twitter-related helper functions for python.
 Home-page: http://github.com/medialab/twitwi
 Author: Béatrice Mazoyer, Guillaume Plique, Benjamin Ooghe-Tabanou
@@ -12,6 +12,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: pytz>=2019.3
 Requires-Dist: ural>=0.31.1
+Requires-Dist: python-dateutil>=2.9.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{twitwi-0.21.0.dist-info → twitwi-0.21.1.dist-info}/RECORD

@@ -3,20 +3,20 @@ test/bluesky/formatters_test.py,sha256=PUEFGdp6fUpNhiWaoEC1-CXfX9L3jPDYcAjoOZ_Di
 test/bluesky/normalizers_test.py,sha256=hYIAM53RYHxxw1YsJ9kJMn8TljnttpczqRpYHyZezAo,4281
 twitwi/__init__.py,sha256=y0bAx9gE3THtlWE1YpXDIhGwqJ5_I8DCStWyyiiXJkw,1095
 twitwi/anonymizers.py,sha256=nkl6HL1BWLz00wJ060XSbqjN5JF8pvcpEPnRXt70TUY,1588
-twitwi/constants.py,sha256=yQFgqe6yFyKcFTGKB2jBja5khGvPR0QZed_2AAl-MmM,16050
+twitwi/constants.py,sha256=fvqCngJIGyz5CpdVWbcAfjmE3_kvcx9giN0rEljL7OU,16001
 twitwi/exceptions.py,sha256=OCIDagu2ErDyOGWunRBCK3O62TnzFpIMQ9gS8l9EALQ,696
 twitwi/formatters.py,sha256=yn14AsrGAUw8rShOnYJvoMbzdWpfTeSs0P0ZPNTwhLU,3142
 twitwi/normalizers.py,sha256=CWUK-XwhcEjLDjWH_qb6E03WZKsbIcwiRAVUjwXKQho,28438
-twitwi/utils.py,sha256=HutQFBA_FInOfodQCrCAppFYHsUMy5RihPMJTAZxpH4,3051
+twitwi/utils.py,sha256=8BiUrkzeydTh4a-rcFW0IHhSNdxlvCXaEpHZGVROl3A,3090
 twitwi/bluesky/__init__.py,sha256=7ITVm1bWE9p0HCWcUs2iOxwKQXK8NEzrvp9hXaicrp0,445
 twitwi/bluesky/constants.py,sha256=CSxNZGYlZvlZ0IMYpP8tVTO6AnzoTmGDFh185hs8n88,473
 twitwi/bluesky/formatters.py,sha256=r8MKOpU8z024z9DQ8-tlKjzaB92T1Su2IjcFV5hjfXg,731
-twitwi/bluesky/normalizers.py,sha256=FslQs73pOhfVYsGVZdUc4548htMAcduX8JfGyX36lrY,24415
-twitwi/bluesky/types.py,sha256=PqWTDIy47I4OjapAiYBeNQBI3cuXgFg0eREXJ0VcUTI,12288
-twitwi/bluesky/utils.py,sha256=1beoDdt4dtIJ6OPxNLSkyt4D3H5ZszKvZWsv1EYEzvs,3235
-twitwi-0.21.0.dist-info/licenses/LICENSE.txt,sha256=Ddg_PcGnl0qd2167o2dheCjE_rCZJOoBxjJnJhhOpX4,1099
-twitwi-0.21.0.dist-info/METADATA,sha256=AhyvMJK02sZDLnnJR8sXeiDI2ncynVwNqXKmKw1t57s,17979
-twitwi-0.21.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-twitwi-0.21.0.dist-info/top_level.txt,sha256=TaKyGU7j_EVbP5KI0UD6qjbaKv2Qn0OrkfUQ29a04kg,12
-twitwi-0.21.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-twitwi-0.21.0.dist-info/RECORD,,
+twitwi/bluesky/normalizers.py,sha256=LpxWgwrIG1TsjL-6nEG8U_FsMKh_BQHuXRrAJdu9lEA,26318
+twitwi/bluesky/types.py,sha256=TaOnAbLPKJqWumLybKmDDh46FLb9WXzqOPyDQU8RHrI,12288
+twitwi/bluesky/utils.py,sha256=KQd0YVWZHcCRcqnX5A1aeKdavwocN6Bsi8pCnHqRgXc,3486
+twitwi-0.21.1.dist-info/licenses/LICENSE.txt,sha256=Ddg_PcGnl0qd2167o2dheCjE_rCZJOoBxjJnJhhOpX4,1099
+twitwi-0.21.1.dist-info/METADATA,sha256=ruJ7MYnoR-xAoMFKf8RWYSri-jqvf8ulW-2fquLjcys,18017
+twitwi-0.21.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+twitwi-0.21.1.dist-info/top_level.txt,sha256=TaKyGU7j_EVbP5KI0UD6qjbaKv2Qn0OrkfUQ29a04kg,12
+twitwi-0.21.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+twitwi-0.21.1.dist-info/RECORD,,