rettiwt-api 1.0.1

package/src/services/helper/Extractors.ts

@@ -0,0 +1,455 @@
+// TYPES
+import { DataErrors } from '../../types/graphql/Errors';
+import RawUser from '../../types/raw/user/User';
+import RawUserFollowers from '../../types/raw/user/Followers';
+import RawUserFollowing from '../../types/raw/user/Following';
+import RawUserLikes from '../../types/raw/user/Likes';
+import RawUserTweets from '../../types/raw/user/Tweets';
+import RawTweet from '../../types/raw/tweet/Tweet';
+import RawTweets from '../../types/raw/tweet/Tweets';
+import RawLikers from '../../types/raw/tweet/Favouriters';
+import RawRetweeters from '../../types/raw/tweet/Retweeters';
+
+// PARSERS
+import * as Parsers from './Parser';
+
+/* USERS */
+
+/**
+ * @returns The raw user account data formatted and sorted into required and additional data
+ * @param res The raw response received from Twitter
+ */
+export function extractUserAccountDetails(res: RawUser): {
+    required: any[],
+    cursor: string,
+    users: any[],
+    tweets: any[]
+} {
+    let required: any[] = [];                                               // To store the reqruied raw data
+    let cursor: string = '';                                                // To store the cursor to next batch
+    let users: any[] = [];                                                  // To store additional user data
+    let tweets: any[] = [];                                                 // To store additional tweet data
+
+    // If user not found or account suspended
+    if (Parsers.isJSONEmpty(res.data) || Parsers.isJSONEmpty(res.data.user) || res.data.user.result.__typename !== 'User') {
+        throw new Error(DataErrors.UserNotFound);
+    }
+
+    // Destructuring user account data
+    required.push(res.data.user.result);
+    users.push(res.data.user.result);
+
+    // Returning the data
+    return {
+        required: required,
+        cursor: cursor,
+        users: users,
+        tweets: tweets
+    };
+}
+
+/**
+ * @returns The raw user following/followers data formatted and sorted into required and additional data
+ * @param res The raw response received from TwitterAPI
+ */
+export function extractUserFollow(res: RawUserFollowers | RawUserFollowing): {
+    required: any[],
+    cursor: string,
+    users: any[],
+    tweets: any[]
+} {
+    let required: any[] = [];                                               // To store the reqruied raw data
+    let cursor: string = '';                                                // To store the cursor to next batch
+    let users: any[] = [];                                                  // To store additional user data
+    let tweets: any[] = [];                                                 // To store additional tweet data
+
+    // If user does not exist
+    if (Parsers.isJSONEmpty(res.data.user)) {
+        throw new Error(DataErrors.UserNotFound);
+    }
+
+    // Extracting the raw list
+    res.data.user.result.timeline.timeline.instructions.forEach(item => {
+        if (item.type === 'TimelineAddEntries') {
+            // Destructuring data
+            item.entries?.forEach(entry => {
+                // If entry is of type user and user account exists
+                if (entry.entryId.indexOf('user') != -1 && entry.content.itemContent?.user_results.result.__typename ==='User') {
+                    required.push(entry.content.itemContent.user_results.result);
+                    users.push(entry.content.itemContent.user_results.result);
+                }
+                // If entry is of type cursor
+                else if (entry.entryId.indexOf('cursor-bottom') != -1) {
+                    cursor = entry.content.value ?? '';
+                }
+            });
+        }
+    });
+
+    // Returning the data
+    return {
+        required: required,
+        cursor: cursor,
+        users: users,
+        tweets: tweets
+    };
+}
+
+/**
+ * @returns The raw user likes data formatted and sorted into required and additional data
+ * @param res The raw response received from TwitterAPI
+ */
+export function extractUserLikes(res: RawUserLikes): {
+    required: any[],
+    cursor: string,
+    users: any[],
+    tweets: any[]
+} {
+    let required: any[] = [];                                               // To store the reqruied raw data
+    let cursor: string = '';                                                // To store the cursor to next batch
+    let users: any[] = [];                                                  // To store additional user data
+    let tweets: any[] = [];                                                 // To store additional tweet data
+
+    // If user does not exist
+    if (Parsers.isJSONEmpty(res.data.user)) {
+        throw new Error(DataErrors.UserNotFound);
+    }
+
+    // Extracting the raw list
+    res.data.user.result.timeline_v2.timeline.instructions.forEach(item => {
+        if (item.type === 'TimelineAddEntries') {
+            // Destructuring data
+            item.entries.forEach(entry => {
+                // If entry is of type tweet and tweet exists
+                if (entry.entryId.indexOf('tweet') != -1 && entry.content.itemContent?.tweet_results.result.__typename === 'Tweet') {
+                    required.push(entry.content.itemContent.tweet_results.result);
+                    users.push(entry.content.itemContent.tweet_results.result.core.user_results.result);
+                    tweets.push(entry.content.itemContent.tweet_results.result);
+                }
+                // If entry is of type cursor
+                else if (entry.entryId.indexOf('cursor-bottom') != -1) {
+                    cursor = entry.content.value ?? '';
+                }
+            });     
+        }
+    });
+
+    // Returning the data
+    return {
+        required: required,
+        cursor: cursor,
+        users: users,
+        tweets: tweets
+    };
+}
+
+/**
+ * @returns The raw tweets data formatted and sorted into required and additional data
+ * @param res The raw response received from TwitterAPI
+ */
+export function extractUserTweets(res: RawUserTweets): {
+    required: any[],
+    cursor: string,
+    users: any[],
+    tweets: any[]
+} {
+    let required: any[] = [];                                               // To store the reqruied raw data
+    let cursor: string = '';                                                // To store the cursor to next batch
+    let users: any[] = [];                                                  // To store additional user data
+    let tweets: any[] = [];                                                 // To store additional tweet data
+
+    // Getting the raw tweet list
+    let dataTweets = res.data.user.result.timeline_v2.timeline.instructions.filter(item => item.type === 'TimelineAddEntries')[0].entries;
+
+    // Destructuring tweets, if not empty
+    if (!Parsers.isJSONEmpty(dataTweets)) {
+        // Iterating through the json array of tweets
+        for (let entry of dataTweets) {
+            // If the entry is a tweet
+            if(entry.entryId.indexOf('tweet') != -1) {
+                required.push(entry.content.itemContent?.tweet_results.result);
+                tweets.push(entry.content.itemContent?.tweet_results.result);
+                users.push(entry.content.itemContent?.tweet_results.result.core.user_results.result);
+            }
+            // If the entry is a cursor
+            else if(entry.entryId.indexOf('cursor-bottom') != -1) {
+                cursor = entry.content.value as string;
+            }
+        }
+    }
+
+    return {
+        required: required,
+        cursor: cursor,
+        users: users,
+        tweets: tweets
+    };
+}
+
+/* TWEETS */
+
+/**
+ * @returns The raw tweets data formatted and sorted into required and additional data
+ * @param res The raw response received from TwitterAPI
+ */
+export function extractTweets(res: RawTweets): {
+    required: any[],
+    cursor: string,
+    users: any[],
+    tweets: any[]
+} {
+    let required: any[] = [];                                               // To store the reqruied raw data
+    let cursor: string = '';                                                // To store the cursor to next batch
+    let users: any[] = [];                                                  // To store additional user data
+    let tweets: any[] = [];                                                 // To store additional tweet data
+
+    // Getting raw tweet list
+    let dataTweets = res.globalObjects.tweets;
+
+    // Getting raw users list
+    let dataUsers = res.globalObjects.users;
+
+    // Destructuring tweets, if not empty
+    if (!Parsers.isJSONEmpty(dataTweets)) {
+        // Iterating through the json array of tweets
+        for (let key of Object.keys(dataTweets)) {
+            required.push({ rest_id: dataTweets[key].id_str, legacy: dataTweets[key] });
+            tweets.push({ rest_id: dataTweets[key].id_str, legacy: dataTweets[key] });
+        }
+    }
+
+    // Destructuring users, if not empty
+    if (!Parsers.isJSONEmpty(dataUsers)) {
+        // Iterating through the json array of users
+        for (let key of Object.keys(dataUsers)) {
+            users.push({ rest_id: dataUsers[key].id_str, legacy: dataUsers[key] });
+        }
+    }
+
+    // Getting the cursor to next batch
+    // If not first batch
+    if (res.timeline.instructions.length > 2) {
+        cursor = res.timeline.instructions[2]?.replaceEntry.entry.content.operation?.cursor.value ?? '';
+    }
+    // If first batch
+    else {
+        cursor = res.timeline.instructions[0].addEntries?.entries.filter(item => item.entryId.indexOf('cursor-bottom') != -1)[0].content.operation?.cursor.value ?? '';
+    }
+
+    // Returning the data
+    return {
+        required: required,
+        cursor: cursor,
+        users: users,
+        tweets: tweets
+    };
+}
+
+/**
+ * @returns The raw tweet data formatted and sorted into required and additional data
+ * @param res The raw response received from TwitterAPI
+ * @param tweetId The rest id of the tweet to fetch
+ */
+export function extractTweet(res: RawTweet, tweetId: string): {
+    required: any[],
+    cursor: string,
+    users: any[],
+    tweets: any[]
+} {
+    let required: any[] = [];                                               // To store the reqruied raw data
+    let cursor: string = '';                                                // To store the cursor to next batch
+    let users: any[] = [];                                                  // To store additional user data
+    let tweets: any[] = [];                                                 // To store additional tweet data
+
+    // If tweet does not exist
+    if (Parsers.isJSONEmpty(res.data)) {
+        throw new Error(DataErrors.TweetNotFound);
+    }
+
+    // Destructuring the received raw data
+    res.data.threaded_conversation_with_injections.instructions.filter(item => item['type'] === 'TimelineAddEntries')[0].entries?.forEach(entry => {
+        // If entry is of type tweet and tweet exists
+        if (entry.entryId.indexOf('tweet') != -1 && entry.content.itemContent?.tweet_results?.result.__typename === 'Tweet') {
+            // If this is the required tweet
+            if (entry.entryId.indexOf(tweetId) != -1) {
+                required.push(entry.content.itemContent.tweet_results.result);
+            }
+            tweets.push(entry.content.itemContent.tweet_results.result);
+            users.push(entry.content.itemContent.tweet_results.result.core.user_results.result);
+        }
+        // If entry if of type conversation
+        else if (entry.entryId.indexOf('conversationthread') != -1) {
+            // Iterating over the conversation
+            entry.content.items?.forEach(item => {
+                // If item is of type tweet and tweet exists
+                if (item.entryId.indexOf('tweet') != -1 && item.item.itemContent.tweet_results?.result.__typename === 'Tweet') {
+                    required.push(item.item.itemContent.tweet_results.result);
+                    tweets.push(item.item.itemContent.tweet_results.result);
+                    users.push(item.item.itemContent.tweet_results.result.core.user_results.result);
+                }
+            });
+        }
+    });
+
+    // Returning the data
+    return {
+        required: required,
+        cursor: cursor,
+        users: users,
+        tweets: tweets
+    };
+}
+
+/**
+ * @returns The raw tweet likers data formatted and sorted into required and additional data
+ * @param res The raw response received from TwitterAPI
+ */
+export function extractTweetLikers(res: RawLikers): {
+    required: any[],
+    cursor: string,
+    users: any[],
+    tweets: any[]
+} {
+    let required: any[] = [];                                               // To store the reqruied raw data
+    let cursor: string = '';                                                // To store the cursor to next batch
+    let users: any[] = [];                                                  // To store additional user data
+    let tweets: any[] = [];                                                 // To store additional tweet data
+
+    // If tweet does not exist
+    if (Parsers.isJSONEmpty(res.data.favoriters_timeline)) {
+        throw new Error(DataErrors.TweetNotFound);
+    }
+
+    // Destructuring raw list of likers
+    res.data.favoriters_timeline.timeline.instructions.filter(item => item.type === 'TimelineAddEntries')[0].entries.forEach(entry => {
+        // If entry is of type user and user exists
+        if (entry.entryId.indexOf('user') != -1 && entry.content.itemContent?.user_results.result.__typename === 'User') {
+            required.push(entry.content.itemContent.user_results.result);
+            users.push(entry.content.itemContent.user_results.result);
+        }
+        // If entry is of type cursor
+        else if (entry.entryId.indexOf('cursor-bottom') != -1) {
+            cursor = entry.content.value ?? '';
+        }
+    });
+
+    // Returning the data
+    return {
+        required: required,
+        cursor: cursor,
+        users: users,
+        tweets: tweets
+    };
+}
+
+/**
+ * @returns The raw tweet retweeters data formatted and sorted into required and additional data
+ * @param res The raw response received from TwitterAPI
+ */
+export function extractTweetRetweeters(res: RawRetweeters): {
+    required: any[],
+    cursor: string,
+    users: any[],
+    tweets: any[]
+} {
+    let required: any[] = [];                                               // To store the reqruied raw data
+    let cursor: string = '';                                                // To store the cursor to next batch
+    let users: any[] = [];                                                  // To store additional user data
+    let tweets: any[] = [];                                                 // To store additional tweet data
+
+    // If tweet does not exist
+    if (Parsers.isJSONEmpty(res.data.retweeters_timeline)) {
+        throw new Error(DataErrors.TweetNotFound);
+    }
+
+    // Destructuring raw list of retweeters
+    res.data.retweeters_timeline.timeline.instructions.filter(item => item.type === 'TimelineAddEntries')[0].entries.forEach(entry => {
+        // If entry is of type user and user exists
+        if (entry.entryId.indexOf('user') != -1 && entry.content.itemContent?.user_results.result.__typename === 'User') {
+            required.push(entry.content.itemContent.user_results.result);
+            users.push(entry.content.itemContent.user_results.result);
+        }
+        // If entry is of type cursor
+        else if (entry.entryId.indexOf('cursor-bottom') != -1) {
+            cursor = entry.content.value ?? '';
+        }
+    });
+
+    // Returning the data
+    return {
+        required: required,
+        cursor: cursor,
+        users: users,
+        tweets: tweets
+    };
+}
+
+/**
+ * @returns The raw tweet replies data formatted and sorted into required and additional data
+ * @param res The raw response received from TwitterAPI
+ * @param tweetId The id of the tweet whose replies must be extracted
+ */
+export function extractTweetReplies(res: RawTweet, tweetId: string): {
+    required: any[],
+    cursor: string,
+    users: any[],
+    tweets: any[]
+} {
+    let required: any[] = [];                                               // To store the reqruied raw data
+    let cursor: string = '';                                                // To store the cursor to next batch
+    let users: any[] = [];                                                  // To store additional user data
+    let tweets: any[] = [];                                                 // To store additional tweet data
+
+    // If tweet does not exist
+    if (Parsers.isJSONEmpty(res.data)) {
+        throw new Error(DataErrors.TweetNotFound);
+    }
+
+    // Destructuring the received raw data
+    //@ts-ignore
+    res.data.threaded_conversation_with_injections.instructions.filter(item => item.type === 'TimelineAddEntries')[0].entries.map(entry => {
+        // If entry is of type tweet
+        if (entry.entryId.indexOf('tweet') != -1) {
+            // If tweet exists
+            if(entry.content.itemContent?.tweet_results?.result.__typename === 'Tweet') {
+                tweets.push(entry.content.itemContent.tweet_results.result);
+                users.push(entry.content.itemContent.tweet_results.result.core.user_results.result);
+            }
+        }
+        // If entry if of type conversation/reply
+        else if (entry.entryId.indexOf('conversationthread') != -1) {
+            // If tweet exists
+            if(entry.content.items?.at(0)?.item.itemContent.tweet_results?.result.__typename === 'Tweet') {
+                // Adding the 1st entry, which is a reply, to required list
+                required.push(entry.content.items[0].item.itemContent.tweet_results?.result);
+                tweets.push(entry.content.items[0].item.itemContent.tweet_results?.result);
+                users.push(entry.content.items[0].item.itemContent.tweet_results?.result.core.user_results.result);
+            }
+            
+            // Iterating over the rest of the conversation
+            //@ts-ignore
+            entry.content.items.forEach(item => {
+                // If item is of type tweet
+                if (item.entryId.indexOf('tweet') != -1) {
+                    // If tweet exists
+                    if(item.item.itemContent.tweet_results?.result.__typename === 'Tweet') {
+                        tweets.push(item.item.itemContent.tweet_results.result);
+                        users.push(item.item.itemContent.tweet_results.result.core.user_results.result);
+                    }
+                }
+            });
+        }
+        // If entry is of type bottom cursor
+        else if (entry.entryId.indexOf('cursor-bottom') != -1) {
+            cursor = entry.content.itemContent?.value ?? '';
+        }
+    });
+
+    // Returning the data
+    return {
+        required: required,
+        cursor: cursor,
+        users: users,
+        tweets: tweets
+    };
+}

package/src/services/helper/Headers.ts

@@ -0,0 +1,45 @@
+// TYPES
+import { GuestCredentials, AuthCredentials } from '../../types/Authentication';
+
+/**
+ * @returns The header required for making authorized HTTP requests
+ * @param authToken The authentication token received from Twitter
+ * @param csrfToken The csrf token received from Twitter
+ * @param cookie The cookie associated with the logged in account
+ */
+export function authorizedHeader(authCred: AuthCredentials): any {
+    return [
+        `sec-ch-ua: "Not_A Brand";v="99", "Microsoft Edge";v="109", "Chromium";v="109"`,
+        `x-twitter-client-language: en`,
+        `x-csrf-token: ${authCred.csrfToken}`,
+        `sec-ch-ua-mobile: ?0`,
+        `authorization: ${authCred.authToken}`,
+        `User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.61`,
+        `x-twitter-auth-type: OAuth2Session`,
+        `x-twitter-active-user: yes`,
+        `sec-ch-ua-platform: "Windows"`,
+        `Accept: */*`,
+        `host: api.twitter.com`,
+        `Cookie: ${authCred.cookie}`
+    ];
+}
+
+export function guestHeader(guestCred: GuestCredentials): any {
+    return [
+        'authority: api.twitter.com',
+        'accept: */*',
+        'accept-language: en-US,en;q=0.9',
+        `authorization: ${guestCred.authToken}`,
+        'content-type: application/json',
+        'origin: https://twitter.com',
+        'referer: https://twitter.com/',
+        'sec-ch-ua: ^\^"Not_A Brand^\^";v=^\^"99^\^", ^\^"Microsoft Edge^\^";v=^\^"109^\^", ^\^"Chromium^\^";v=^\^"109^\^"',
+        'sec-ch-ua-mobile: ?0',
+        'sec-ch-ua-platform: ^\^"Windows^\^"',
+        'sec-fetch-dest: empty',
+        'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.70',
+        `x-guest-token: ${guestCred.guestToken}`,
+        'x-twitter-active-user: yes',
+        'x-twitter-client-language: en'
+    ];
+}

package/src/services/helper/Parser.ts

@@ -0,0 +1,108 @@
+import { TweetFilter } from '../../types/Tweet';
+
+/**
+ * @returns Whether the given json object is empty or not
+ * @param data The input JSON object which needs to be checked
+ */
+export function isJSONEmpty(data: any): boolean {
+    // If the JSON has any keys, it's not empty
+    if(Object.keys(data).length == 0) {
+        return true;
+    }
+    // Else, it's empty
+    else {
+        return false;
+    }
+}
+
+/**
+ * @returns The value associated with the given key inside the given json
+ * @param data The json data within which to search for the value
+ * @param key The key to search for
+ * @param last Whether to begin searching from the end
+ */
+export function findJSONKey(data: any, key: string, last: boolean = false): any {
+    let jsonStr: string = JSON.stringify(data);                                 // To store the input data as string
+    let extStr: string = '';                                                    // To store the extracted string
+    let len: number = jsonStr.length;                                           // To store length of input data
+
+    /**
+     * Getting the position to start extracting data from
+     * This the position just after the key plus ":"
+     * */
+    let start: number = !last ? (jsonStr.indexOf(`"${key}"`) + `"${key}":`.length) : (jsonStr.lastIndexOf(`"${key}"`) + `"${key}":`.length);
+
+    for (let i = start; i < len; i++) {
+        // Getting each character
+        let char: string = jsonStr[i];
+
+        // If not ending of value
+        if (char != ',' && char != '\n') {
+            extStr += char;
+        }
+        // If ending of value
+        else {
+            break;
+        }
+    }
+
+    // Removing begginning and ending quotes from string
+    /**
+     * I don't know how this regex I used works. I just copied it from StackOverflow
+     * Here is the link to the thread: https://stackoverflow.com/q/19156148
+     */
+    extStr = extStr.replace(/^"|"$/g, '');
+
+    return extStr;
+}
+
+/**
+ * @returns A list of data from a singleton data
+ * @param data The data to be converted to a list
+ */
+export function dataToList(data: any | any[]): any[] {
+    // If data is already a list
+    if (Array.isArray(data)) {
+        return data;
+    }
+    // If data is not array
+    else {
+        return [data];
+    }
+}
+
+/**
+ * @param text The text to be normalized
+ * @returns The text after being formatted to remove unnecessary characters
+ */
+export function normalizeText(text: string): string {
+    let normalizedText: string = '';                                                // To store the normalized text
+    
+    // Removing unnecessary full stops, and other characters
+    normalizedText = text.replace(/\n/g, '.').replace(/[.]+[\s+.\s+]+/g, '. ');
+    
+    // Adding full-stop to the end if does not exist already
+    normalizedText = normalizedText.endsWith('.') ? normalizedText : (normalizedText + '.');
+
+    return normalizedText;
+}
+
+/**
+ * @param filter The tweet filter to use for getting filtered tweets
+ * @returns The same tweet filter, in a URL query format string
+ */
+export function toQueryString(filter: TweetFilter): string {
+    // Concatenating the input filter arguments to a URL query formatted string
+    return [
+        filter.words ? filter.words.join(' ') : '',
+        filter.hashtags ? `(${filter.hashtags.map(hashtag => '%23' + hashtag).join(' OR ')})` : '',
+        filter.fromUsers ? `(${filter.fromUsers.map(user => `from:${user}`).join(' OR ')})` : '',
+        filter.toUsers ? `(${filter.toUsers.map(user => `to:${user}`).join(' OR ')})` : '',
+        filter.mentions ? `(${filter.mentions.map(mention => '%40' + mention).join(' OR ')})` : '',
+        filter.startDate ? `since:${filter.startDate}` : '',
+        filter.endDate ? `until:${filter.endDate}` : '',
+        filter.quoted ? `quoted_tweet_id:${filter.quoted}` : ''
+    ]
+    .filter(item => item !== '()' && item !== '')
+    .join(' ') + (!filter.links ? '%20-filter%3Alinks' : '');
+}