twitwi 0.20.0__py3-none-any.whl → 0.21.0__py3-none-any.whl

test/bluesky/formatters_test.py

@@ -0,0 +1,101 @@
+import csv
+from io import StringIO
+from twitwi.bluesky import (
+    format_profile_as_csv_row,
+    format_post_as_csv_row,
+    transform_profile_into_csv_dict,
+    transform_post_into_csv_dict,
+)
+from twitwi.bluesky.constants import PROFILE_FIELDS, POST_FIELDS
+from test.utils import get_json_resource, open_resource
+
+
+# Set to True to regenerate test results
+OVERWRITE_TESTS = False
+
+
+class TestFormatters:
+    def test_format_profile_as_csv_row(self):
+        normalized_profiles = get_json_resource("bluesky-normalized-profiles.json")
+
+        buffer = StringIO(newline=None)
+        writer = csv.writer(buffer, quoting=csv.QUOTE_MINIMAL)
+        writer.writerow(PROFILE_FIELDS)
+
+        for profile in normalized_profiles:
+            writer.writerow(format_profile_as_csv_row(profile))
+
+        if OVERWRITE_TESTS:
+            written = buffer.getvalue()
+
+            with open("test/resources/bluesky-profiles-export.csv", "w") as f:
+                f.write(written)
+
+        with open_resource("bluesky-profiles-export.csv") as f:
+            buffer.seek(0)
+            assert list(csv.reader(buffer)) == list(csv.reader(f))
+
+    def test_transform_profile_into_csv_dict(self):
+        normalized_profiles = get_json_resource("bluesky-normalized-profiles.json")
+
+        buffer = StringIO(newline=None)
+        writer = csv.DictWriter(
+            buffer,
+            fieldnames=PROFILE_FIELDS,
+            extrasaction="ignore",
+            restval="",
+            quoting=csv.QUOTE_MINIMAL,
+        )
+        writer.writeheader()
+
+        for profile in normalized_profiles:
+            transform_profile_into_csv_dict(profile)
+            writer.writerow(profile)
+
+        with open_resource("bluesky-profiles-export.csv") as f:
+            buffer.seek(0)
+            assert list(csv.DictReader(buffer)) == list(csv.DictReader(f))
+
+    def test_format_post_as_csv_row(self):
+        normalized_posts = get_json_resource("bluesky-normalized-posts.json")
+
+        buffer = StringIO(newline=None)
+        writer = csv.writer(buffer, quoting=csv.QUOTE_MINIMAL)
+        writer.writerow(POST_FIELDS)
+
+        for source in normalized_posts:
+            for post in source:
+                writer.writerow(format_post_as_csv_row(post))
+
+        if OVERWRITE_TESTS:
+            written = buffer.getvalue()
+
+            with open("test/resources/bluesky-posts-export.csv", "w") as f:
+                f.write(written)
+
+        with open_resource("bluesky-posts-export.csv") as f:
+            buffer.seek(0)
+
+            assert list(csv.reader(buffer)) == list(csv.reader(f))
+
+    def test_transform_post_into_csv_dict(self):
+        normalized_posts = get_json_resource("bluesky-normalized-posts.json")
+
+        buffer = StringIO(newline=None)
+        writer = csv.DictWriter(
+            buffer,
+            fieldnames=POST_FIELDS,
+            extrasaction="ignore",
+            restval="",
+            quoting=csv.QUOTE_MINIMAL,
+        )
+        writer.writeheader()
+
+        for source in normalized_posts:
+            for post in source:
+                transform_post_into_csv_dict(post)
+                writer.writerow(post)
+
+        with open_resource("bluesky-posts-export.csv") as f:
+            buffer.seek(0)
+            assert list(csv.DictReader(buffer)) == list(csv.DictReader(f))

test/bluesky/normalizers_test.py

@@ -0,0 +1,130 @@
+# =============================================================================
+# Twitwi Bluesky Normalizers Unit Tests
+# =============================================================================
+from functools import partial
+from pytz import timezone
+from copy import deepcopy
+
+from twitwi.bluesky import normalize_profile, normalize_post
+
+from test.utils import get_json_resource
+
+
+# Set to True to regenerate test results
+OVERWRITE_TESTS = False
+
+
+FAKE_COLLECTION_TIME = "2025-01-01T00:00:00.000000"
+def set_fake_collection_time(dico):
+    if "collection_time" in dico:
+        dico["collection_time"] = FAKE_COLLECTION_TIME
+    return dico
+
+
+def compare_dicts(_id, d1, d2, ignore_fields=[]):
+    for k in d2.keys():
+        if k not in ignore_fields + ["collection_time"]:
+            assert d1[k] == d2[k], (
+                'Different value for key "%s" with payload data for "%s"'
+                % (
+                    k,
+                    _id,
+                )
+            )
+
+    for k in d1.keys():
+        if k not in ignore_fields:
+            assert k in d2, 'Missing key "%s" with payload data for "%s"' % (k, _id)
+
+
+class TestNormalizers:
+    def test_normalize_profile(self):
+        tz = timezone("Europe/Paris")
+
+        profiles = get_json_resource("bluesky-profiles.json")
+        fn = partial(normalize_profile, locale=tz)
+
+        if OVERWRITE_TESTS:
+            from test.utils import dump_json_resource
+
+            normalized_profiles = [set_fake_collection_time(fn(profile)) for profile in profiles]
+            dump_json_resource(normalized_profiles, "bluesky-normalized-profiles.json")
+
+        expected = get_json_resource("bluesky-normalized-profiles.json")
+
+        for idx, profile in enumerate(profiles):
+            result = fn(profile)
+            assert isinstance(result, dict)
+            assert "collection_time" in result and isinstance(
+                result["collection_time"], str
+            )
+
+            compare_dicts(profile["handle"], result, expected[idx])
+
+    def test_normalize_profile_should_not_mutate(self):
+        profile = get_json_resource("bluesky-profiles.json")[0]
+
+        original_arg = deepcopy(profile)
+
+        normalize_profile(profile)
+
+        assert profile == original_arg
+
+    def test_normalize_post(self):
+        tz = timezone("Europe/Paris")
+
+        posts = get_json_resource("bluesky-posts.json")
+        fn = partial(normalize_post, locale=tz)
+
+        if OVERWRITE_TESTS:
+            from test.utils import dump_json_resource
+
+            normalized_posts = [[set_fake_collection_time(p) for p in fn(post, extract_referenced_posts=True)] for post in posts]
+            dump_json_resource(normalized_posts, "bluesky-normalized-posts.json")
+
+        expected = get_json_resource("bluesky-normalized-posts.json")
+
+        # With referenced tweets
+        for idx, post in enumerate(posts):
+            result = fn(post, extract_referenced_posts=True)
+            assert isinstance(result, list)
+            assert set(p["uri"] for p in result) == set(p["uri"] for p in expected[idx])
+            for idx2, p in enumerate(result):
+                assert "collection_time" in p and isinstance(p["collection_time"], str)
+
+                if "post" in post:
+                    uri = post["post"]["uri"]
+                else:
+                    uri = post["uri"]
+                compare_dicts(uri, p, expected[idx][idx2])
+
+        # With single output
+        for idx, post in enumerate(posts):
+            result = fn(post)
+
+            assert isinstance(result, dict)
+
+            _id = p["uri"]
+            compare_dicts(_id, result, expected[idx][-1])
+
+        # With custom collection_source
+        for post in posts:
+            result = fn(post, collection_source="unit_test")
+
+            assert result["collected_via"] == ["unit_test"]
+
+    def test_normalize_post_should_not_mutate(self):
+        post = get_json_resource("bluesky-posts.json")[0]
+
+        original_arg = deepcopy(post)
+
+        normalize_post(post)
+
+        assert post == original_arg
+
+    def test_normalize_post_should_be_normalized_across_sources(self):
+        # handle same post from different sources (search, get_post and user_feed)
+        pass
+
+    def test_badly_formatted_posts_payload(self):
+        pass

twitwi/__init__.py

@@ -8,8 +8,9 @@ from twitwi.formatters import (
     format_tweet_as_csv_row,
     transform_user_into_csv_dict,
     format_user_as_csv_row,
-    apply_tcat_format
+    apply_tcat_format,
 )
+
 # NOTE: should we drop this from public exports?
 from twitwi.utils import (
     get_dates,
@@ -20,5 +21,21 @@ from twitwi.utils import (
 from twitwi.normalizers import (
     normalize_tweet,
     normalize_user,
-    normalize_tweets_payload_v2
+    normalize_tweets_payload_v2,
 )
+
+__all__ = [
+    "anonymize_normalized_tweet",
+    "transform_tweet_into_csv_dict",
+    "format_tweet_as_csv_row",
+    "transform_user_into_csv_dict",
+    "format_user_as_csv_row",
+    "apply_tcat_format",
+    "get_dates",
+    "custom_normalize_url",
+    "get_timestamp_from_id",
+    "get_dates_from_id",
+    "normalize_tweet",
+    "normalize_user",
+    "normalize_tweets_payload_v2",
+]

twitwi/anonymizers.py

@@ -8,13 +8,12 @@ def redact_quoted_text(text: str) -> str:
 
 
 def redact_rt_text(text: str) -> str:
-    return 'RT: ' + text.split(': ', 1)[1]
+    return "RT: " + text.split(": ", 1)[1]
 
 
 FIELDS_TO_DELETE = [
     # The tweet's url leaks the user
     "url",
-
     # User's place
     "lat",
     "lng",
@@ -23,7 +22,6 @@ FIELDS_TO_DELETE = [
     "place_name",
     "place_type",
     "user_location",
-
     # User info
     "user_created_at",
     "user_description",
@@ -34,16 +32,13 @@ FIELDS_TO_DELETE = [
     "user_timestamp_utc",
     "user_url",
     "user_verified",
-
     # Retweeted user info
     "retweeted_timestamp_utc",
     "retweeted_user",
     "retweeted_user_id",
-
     # Replied user info
     "to_userid",
     "to_username",
-
     # Quoted user info
     "quoted_user",
     "quoted_user_id",
@@ -56,14 +51,13 @@ FIELDS_TO_DELETE = [
 # NOTE: we do not redact mentions either.
 # NOTE: we also don't redact replies.
 def anonymize_normalized_tweet(normalized_tweet) -> None:
-
     # Text mangling
     text = normalized_tweet["text"]
 
-    if normalized_tweet.get('retweeted_id', None) is not None:
+    if normalized_tweet.get("retweeted_id", None) is not None:
         normalized_tweet["text"] = redact_rt_text(text)
 
-    elif normalized_tweet.get('quoted_id', None) is not None:
+    elif normalized_tweet.get("quoted_id", None) is not None:
         normalized_tweet["text"] = redact_quoted_text(text)
 
     for field in FIELDS_TO_DELETE:

twitwi/bluesky/__init__.py

@@ -0,0 +1,16 @@
+from twitwi.bluesky.normalizers import normalize_profile, normalize_post
+from twitwi.bluesky.formatters import (
+    transform_profile_into_csv_dict,
+    format_profile_as_csv_row,
+    transform_post_into_csv_dict,
+    format_post_as_csv_row,
+)
+
+__all__ = [
+    "transform_profile_into_csv_dict",
+    "format_profile_as_csv_row",
+    "transform_post_into_csv_dict",
+    "format_post_as_csv_row",
+    "normalize_profile",
+    "normalize_post",
+]

twitwi/bluesky/constants.py

@@ -0,0 +1,19 @@
+from typing import List, Optional
+
+from twitwi.bluesky.types import BlueskyProfile, BlueskyPost
+
+PROFILE_FIELDS = list(BlueskyProfile.__annotations__.keys())
+
+POST_FIELDS = list(BlueskyPost.__annotations__.keys())
+
+POST_PLURAL_FIELDS = [
+    k
+    for k, v in BlueskyPost.__annotations__.items()
+    if v == List[str] or v == Optional[List[str]]
+]
+
+POST_BOOLEAN_FIELDS = [
+    k
+    for k, v in BlueskyPost.__annotations__.items()
+    if v is bool or v == Optional[bool]
+]

twitwi/bluesky/formatters.py

@@ -0,0 +1,29 @@
+from twitwi.formatters import make_transform_into_csv_dict, make_format_as_csv_row
+from twitwi.bluesky.constants import (
+    PROFILE_FIELDS,
+    POST_FIELDS,
+    POST_PLURAL_FIELDS,
+    POST_BOOLEAN_FIELDS,
+)
+
+
+transform_post_into_csv_dict = make_transform_into_csv_dict(
+    POST_PLURAL_FIELDS, POST_BOOLEAN_FIELDS
+)
+
+format_post_as_csv_row = make_format_as_csv_row(
+    POST_FIELDS, POST_PLURAL_FIELDS, POST_BOOLEAN_FIELDS
+)
+
+
+transform_profile_into_csv_dict = make_transform_into_csv_dict([], [])
+
+format_profile_as_csv_row = make_format_as_csv_row(PROFILE_FIELDS, [], [])
+
+
+__all__ = [
+    "transform_post_into_csv_dict",
+    "format_post_as_csv_row",
+    "transform_profile_into_csv_dict",
+    "format_profile_as_csv_row",
+]