rettiwt-api 5.0.0-alpha.0 → 5.0.0-alpha.2

package/src/services/{public → internal}/AuthService.ts

@@ -1,23 +1,24 @@
-import { Auth } from 'rettiwt-auth';
+import { Agent } from 'https';
 
-import { EApiErrors } from '../../enums/Api';
-import { IRettiwtConfig } from '../../types/RettiwtConfig';
+import axios from 'axios';
 
-import { FetcherService } from './FetcherService';
+import { EApiErrors } from '../../enums/Api';
+import { AuthCredential } from '../../models/auth/AuthCredential';
 
 /**
  * The services that handles authentication.
  *
  * @public
  */
-export class AuthService extends FetcherService {
+export class AuthService {
+	/** The HTTPS Agent to use for requests to Twitter API. */
+	private readonly _httpsAgent: Agent;
+
 	/**
-	 * @param config - The config object for configuring the `Rettiwt` instance.
-	 *
-	 * @internal
+	 * @param httpsAgent - The HTTPS agent to use. If none is provided, default is used.
 	 */
-	public constructor(config?: IRettiwtConfig) {
-		super(config);
+	public constructor(httpsAgent?: Agent) {
+		this._httpsAgent = httpsAgent ?? new Agent();
 	}
 
 	/**
@@ -94,57 +95,24 @@ export class AuthService extends FetcherService {
 	 * });
 	 * ```
 	 */
-	public async guest(): Promise<string> {
-		// Getting a new guest key
-		const guestKey: string = (await new Auth().getGuestCredential()).guestToken ?? '';
-
-		return guestKey;
-	}
-
-	/**
-	 * Login to twitter using account credentials.
-	 *
-	 * @param email - The email id associated with the Twitter account.
-	 * @param userName - The username associated with the Twitter account.
-	 * @param password - The password to the Twitter account.
-	 *
-	 * @returns The `API_KEY` for the Twitter account.
-	 *
-	 * @example
-	 * ```
-	 * import { Rettiwt } from 'rettiwt-api';
-	 *
-	 * // Creating a new Rettiwt instance
-	 * const rettiwt = new Rettiwt();
-	 *
-	 * // Logging in an getting the API_KEY
-	 * rettiwt.auth.login("email@domain.com", "username", "password")
-	 * .then(apiKey => {
-	 * 	// Use the API_KEY
-	 * 	...
-	 * })
-	 * .catch(err => {
-	 * 	console.log(err);
-	 * });
-	 * ```
-	 *
-	 * @remarks
-	 * Interchanging `email` and `userName` works too.
-	 */
-	public async login(email: string, userName: string, password: string): Promise<string> {
-		// Logging in and getting the credentials
-		let apiKey: string =
-			((
-				await new Auth({ proxyUrl: this.authProxyUrl }).getUserCredential({
-					email: email,
-					userName: userName,
-					password: password,
-				})
-			).toHeader().cookie as string) ?? '';
+	public async guest(): Promise<AuthCredential> {
+		// Creating a new blank credential
+		const cred: AuthCredential = new AuthCredential();
 
-		// Converting the credentials to base64 string
-		apiKey = AuthService.encodeCookie(apiKey);
+		// Getting the guest token
+		await axios
+			.post<{
+				/* eslint-disable @typescript-eslint/naming-convention */
+				guest_token: string;
+				/* eslint-enable @typescript-eslint/naming-convention */
+			}>('https://api.twitter.com/1.1/guest/activate.json', undefined, {
+				headers: cred.toHeader(),
+				httpsAgent: this._httpsAgent,
+			})
+			.then((res) => {
+				cred.guestToken = res.data.guest_token;
+			});
 
-		return apiKey;
+		return cred;
 	}
 }

package/src/services/internal/LogService.ts

@@ -1,5 +1,3 @@
-import chalk from 'chalk';
-
 import { ELogActions } from '../../enums/Logging';
 
 /**
@@ -9,20 +7,7 @@ import { ELogActions } from '../../enums/Logging';
  */
 export class LogService {
 	/** Whether logging is enabled or not. */
-	public static enabled: boolean = false;
-
-	/**
-	 * @param action - The action to be logged.
-	 *
-	 * @returns - The colored text representing the action.
-	 */
-	private static getColoredAction(action: ELogActions): string {
-		if (action == ELogActions.WARNING) {
-			return chalk.yellow(action);
-		} else {
-			return chalk.green(action);
-		}
-	}
+	public static enabled = false;
 
 	/**
 	 * Logs the given data.
@@ -35,12 +20,12 @@ export class LogService {
 		// Proceed to log only if logging is enabled
 		if (this.enabled) {
 			// Preparing the log message
-			const logPrefix: string = chalk.blue('Rettiwt-API');
+			const logPrefix = 'Rettiwt-API';
 			const logTime: string = new Date().toISOString();
-			const logAction: string = LogService.getColoredAction(action);
+			const logAction: string = action;
 			const logData: string = JSON.stringify(data);
 
-			const logMessage: string = `[${logPrefix}] [${logTime}] [${logAction}] ${logData}`;
+			const logMessage = `[${logPrefix}] [${logTime}] [${logAction}] ${logData}`;
 
 			// Logging
 			console.log(logMessage);

package/src/services/internal/TidService.ts

@@ -0,0 +1,147 @@
+import { Agent } from 'http';
+
+import axios from 'axios';
+import * as htmlParser from 'node-html-parser';
+
+import { ELogActions } from '../../enums/Logging';
+
+import { calculateClientTransactionIdHeader } from '../../helper/TidUtils';
+
+import { ITidDynamicArgs } from '../../types/auth/TidDynamicArgs';
+import { ITidProvider } from '../../types/auth/TidProvider';
+
+import { LogService } from './LogService';
+
+/**
+ * Handles transaction ID generation for requests to Twitter.
+ *
+ * @internal
+ */
+export class TidService implements ITidProvider {
+	private readonly _cdnUrl: string;
+	private readonly _httpsAgent: Agent;
+	private readonly _requestHeaders: NonNullable<unknown>;
+	private _dynamicArgs?: ITidDynamicArgs;
+
+	/**
+	 * @param httpsAgent - The HTTPS agent to use. If none is provided, default is used.
+	 */
+	public constructor(httpsAgent?: Agent) {
+		this._cdnUrl = 'https://abs.twimg.com/responsive-web/client-web';
+		this._httpsAgent = httpsAgent ?? new Agent();
+		this._requestHeaders = {
+			/* eslint-disable @typescript-eslint/naming-convention */
+
+			Authority: 'x.com',
+			'Accept-Language': 'en-US,en;q=0.9',
+			'Cache-Control': 'no-cache',
+			Referer: 'https://x.com',
+			'User-Agent':
+				'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
+			'X-Twitter-Active-User': 'yes',
+			'X-Twitter-Client-Language': 'en',
+
+			/* eslint-enable @typescript-eslint/naming-convention */
+		};
+	}
+
+	/**
+	 * Fetches the dynamic args embedded in the homepage.
+	 *
+	 * @returns The new dynamic args.
+	 */
+	private async getDynamicArgs(): Promise<ITidDynamicArgs> {
+		const html = await this.getHomepageHtml();
+		const root = htmlParser.parse(html);
+		const keyElement = root.querySelector("[name='twitter-site-verification']");
+		const frameElements = root.querySelectorAll("[id^='loading-x-anim']");
+
+		return {
+			verificationKey: keyElement?.getAttribute('content') ?? '',
+			frames: frameElements.map((el) => this.parseFrameElement(el)),
+			indices: await this.getKeyBytesIndices(html),
+		};
+	}
+
+	/**
+	 * Fetches the HTML content of Twitter's homepage.
+	 *
+	 * @returns The stringified HTML content of the homepage.
+	 */
+	private async getHomepageHtml(): Promise<string> {
+		const response = await axios.get<string>('https://x.com', {
+			headers: this._requestHeaders,
+			httpAgent: this._httpsAgent,
+			httpsAgent: this._httpsAgent,
+		});
+
+		return response.data;
+	}
+
+	private async getKeyBytesIndices(html: string): Promise<number[]> {
+		const ondemandFileMatch = html.match(/ondemand\.s":"([^"]+)"/);
+		if (!ondemandFileMatch || !ondemandFileMatch[1]) {
+			LogService.log(ELogActions.WARNING, { message: 'ondemand.s file not found' });
+
+			return [0, 0, 0, 0];
+		}
+
+		const onDemandFileHash = ondemandFileMatch ? ondemandFileMatch[1] : '';
+		const response = await axios.get<string>(`${this._cdnUrl}/ondemand.s.${onDemandFileHash}a.js`, {
+			httpAgent: this._httpsAgent,
+			httpsAgent: this._httpsAgent,
+		});
+		const match = response.data.matchAll(/(\(\w\[(\d{1,2})],\s*16\))+?/gm);
+
+		return Array.from(match).map((m) => Number(m[2]));
+	}
+
+	private parseFrameElement(element: htmlParser.HTMLElement): number[][] {
+		const pathElement = element.children[0].children[1];
+		const value = pathElement.getAttribute('d');
+		if (!value) {
+			return [[]];
+		}
+
+		const rawFrames = value.substring(9).split('C');
+
+		return rawFrames.map((str) => str.replaceAll(/\D+/g, ' ').trim().split(' ')).map((arr) => arr.map(Number));
+	}
+
+	/**
+	 * Generate an `x-client-transaction-id` for the specific URL method and path.
+	 *
+	 * @param method - The target method.
+	 * @param path - The target path.
+	 *
+	 * @returns The specific `x-client-transaction-id` token.
+	 */
+	public async generate(method: string, path: string): Promise<string | undefined> {
+		try {
+			if (!this._dynamicArgs) {
+				this._dynamicArgs = await this.getDynamicArgs();
+			}
+
+			const { verificationKey, frames, indices } = this._dynamicArgs;
+
+			return calculateClientTransactionIdHeader({
+				keyword: 'obfiowerehiring',
+				method: method,
+				path: path,
+				verificationKey: verificationKey,
+				frames: frames,
+				indices: indices,
+				extraByte: 3,
+			});
+		} catch {
+			return;
+		}
+	}
+
+	/**
+	 * Refreshes the dynamic args from the homepage.
+	 */
+	public async refreshDynamicArgs(): Promise<void> {
+		this._dynamicArgs = await this.getDynamicArgs();
+	}
+}

package/src/services/public/FetcherService.ts

@@ -1,8 +1,5 @@
-import https, { Agent } from 'https';
-
 import axios from 'axios';
-import { HttpsProxyAgent } from 'https-proxy-agent';
-import { Auth, AuthCredential } from 'rettiwt-auth';
+import { Cookie } from 'cookiejar';
 
 import { allowGuestAuthentication, fetchResources, postResources } from '../../collections/Groups';
 import { requests } from '../../collections/Requests';
@@ -11,13 +8,18 @@ import { ELogActions } from '../../enums/Logging';
 import { EResourceType } from '../../enums/Resource';
 import { FetchArgs } from '../../models/args/FetchArgs';
 import { PostArgs } from '../../models/args/PostArgs';
+import { AuthCredential } from '../../models/auth/AuthCredential';
+import { RettiwtConfig } from '../../models/RettiwtConfig';
+import { IFetchArgs } from '../../types/args/FetchArgs';
+import { IPostArgs } from '../../types/args/PostArgs';
+import { ITidHeader } from '../../types/auth/TidHeader';
+import { ITidProvider } from '../../types/auth/TidProvider';
 import { IErrorHandler } from '../../types/ErrorHandler';
-import { IRettiwtConfig } from '../../types/RettiwtConfig';
 
+import { AuthService } from '../internal/AuthService';
 import { ErrorService } from '../internal/ErrorService';
 import { LogService } from '../internal/LogService';
-
-import { AuthService } from './AuthService';
+import { TidService } from '../internal/TidService';
 
 /**
  * The base service that handles all HTTP requests.
@@ -25,39 +27,35 @@ import { AuthService } from './AuthService';
  * @public
  */
 export class FetcherService {
-	/** The api key to use for authenticating against Twitter API as user. */
-	private readonly _apiKey?: string;
+	/** The AuthService instance to use. */
+	private readonly _auth: AuthService;
+
+	/** The delay/delay function to use (ms). */
+	private readonly _delay?: number | (() => number | Promise<number>);
 
 	/** The service used to handle HTTP and API errors */
 	private readonly _errorHandler: IErrorHandler;
 
-	/** The guest key to use for authenticating against Twitter API as guest. */
-	private readonly _guestKey?: string;
-
-	/** The URL To the proxy server to use for all others. */
-	private readonly _proxyUrl?: URL;
+	/** Service responsible for generating the `x-client-transaction-id` header. */
+	private readonly _tidProvider: ITidProvider;
 
 	/** The max wait time for a response. */
 	private readonly _timeout: number;
 
-	/** The URL to the proxy server to use only for authentication. */
-	protected readonly authProxyUrl?: URL;
-
-	/** The id of the authenticated user (if any). */
-	protected readonly userId?: string;
+	/** The config object. */
+	protected readonly config: RettiwtConfig;
 
 	/**
 	 * @param config - The config object for configuring the Rettiwt instance.
 	 */
-	public constructor(config?: IRettiwtConfig) {
-		LogService.enabled = config?.logging ?? false;
-		this._apiKey = config?.apiKey;
-		this._guestKey = config?.guestKey;
-		this.userId = config?.apiKey ? AuthService.getUserId(config.apiKey) : undefined;
-		this.authProxyUrl = config?.authProxyUrl ?? config?.proxyUrl;
-		this._proxyUrl = config?.proxyUrl;
-		this._timeout = config?.timeout ?? 0;
-		this._errorHandler = config?.errorHandler ?? new ErrorService();
+	public constructor(config: RettiwtConfig) {
+		LogService.enabled = config.logging ?? false;
+		this.config = config;
+		this._delay = config.delay;
+		this._errorHandler = config.errorHandler ?? new ErrorService();
+		this._tidProvider = config.tidProvider ?? new TidService(config.httpsAgent);
+		this._timeout = config.timeout ?? 0;
+		this._auth = new AuthService(config.httpsAgent);
 	}
 
 	/**
@@ -69,10 +67,10 @@ export class FetcherService {
 	 */
 	private checkAuthorization(resource: EResourceType): void {
 		// Logging
-		LogService.log(ELogActions.AUTHORIZATION, { authenticated: this.userId != undefined });
+		LogService.log(ELogActions.AUTHORIZATION, { authenticated: this.config.userId != undefined });
 
 		// Checking authorization status
-		if (!allowGuestAuthentication.includes(resource) && this.userId == undefined) {
+		if (!allowGuestAuthentication.includes(resource) && this.config.userId == undefined) {
 			throw new Error(EApiErrors.RESOURCE_NOT_ALLOWED);
 		}
 	}
@@ -83,42 +81,46 @@ export class FetcherService {
 	 * @returns The generated AuthCredential
 	 */
 	private async getCredential(): Promise<AuthCredential> {
-		if (this._apiKey) {
+		if (this.config.apiKey) {
 			// Logging
 			LogService.log(ELogActions.GET, { target: 'USER_CREDENTIAL' });
 
-			return new AuthCredential(AuthService.decodeCookie(this._apiKey).split(';'));
-		} else if (this._guestKey) {
-			// Logging
-			LogService.log(ELogActions.GET, { target: 'GUEST_CREDENTIAL' });
-
-			return new AuthCredential(undefined, this._guestKey);
+			return new AuthCredential(
+				AuthService.decodeCookie(this.config.apiKey)
+					.split(';')
+					.map((item) => new Cookie(item)),
+			);
 		} else {
 			// Logging
 			LogService.log(ELogActions.GET, { target: 'NEW_GUEST_CREDENTIAL' });
 
-			return await new Auth({ proxyUrl: this.authProxyUrl }).getGuestCredential();
+			return this._auth.guest();
 		}
 	}
 
 	/**
-	 * Gets the https agent based on whether a proxy is used or not.
+	 * Generates the header for the transaction ID.
 	 *
-	 * @param proxyUrl - Optional URL with proxy configuration to use for requests to Twitter API.
+	 * @param method - The target method.
+	 * @param url - The target URL.
 	 *
-	 * @returns The https agent to use.
+	 * @returns The header containing the transaction ID.
 	 */
-	private getHttpsAgent(proxyUrl?: URL): Agent {
-		if (proxyUrl) {
-			// Logging
-			LogService.log(ELogActions.GET, { target: 'HTTPS_PROXY_AGENT' });
-
-			return new HttpsProxyAgent(proxyUrl);
+	private async getTransactionHeader(method: string, url: string): Promise<ITidHeader | undefined> {
+		// Getting the URL path excluding all params
+		const path = new URL(url).pathname.split('?')[0].trim();
+
+		// Generating the transaction ID
+		const tid = await this._tidProvider.generate(method.toUpperCase(), path);
+
+		if (tid) {
+			return {
+				/* eslint-disable @typescript-eslint/naming-convention */
+				'x-client-transaction-id': tid,
+				/* eslint-enable @typescript-eslint/naming-convention */
+			};
 		} else {
-			// Logging
-			LogService.log(ELogActions.GET, { target: 'HTTPS_AGENT' });
-
-			return new https.Agent();
+			return undefined;
 		}
 	}
 
@@ -130,7 +132,7 @@ export class FetcherService {
 	 *
 	 * @returns The validated args.
 	 */
-	private validateArgs(resource: EResourceType, args: FetchArgs | PostArgs): FetchArgs | PostArgs | undefined {
+	private validateArgs(resource: EResourceType, args: IFetchArgs | IPostArgs): FetchArgs | PostArgs | undefined {
 		if (fetchResources.includes(resource)) {
 			// Logging
 			LogService.log(ELogActions.VALIDATE, { target: 'FETCH_ARGS' });
@@ -144,6 +146,29 @@ export class FetcherService {
 		}
 	}
 
+	/**
+	 * Introduces a delay using the configured delay/delay function.
+	 */
+	private async wait(): Promise<void> {
+		// If no delay is set, skip
+		if (this._delay == undefined) {
+			return;
+		}
+
+		/** The delay (in ms) to use. */
+		let delay = 0;
+
+		// Getting the delay
+		if (this._delay && typeof this._delay == 'number') {
+			delay = this._delay;
+		} else if (this._delay && typeof this._delay == 'function') {
+			delay = await this._delay();
+		}
+
+		// Awaiting for the delay time
+		await new Promise((resolve) => setTimeout(resolve, delay));
+	}
+
 	/**
 	 * Makes an HTTP request according to the given parameters.
 	 *
@@ -172,7 +197,7 @@ export class FetcherService {
 	 * })
 	 * ```
 	 */
-	public async request<T>(resource: EResourceType, args: FetchArgs | PostArgs): Promise<T> {
+	public async request<T = unknown>(resource: EResourceType, args: IFetchArgs | IPostArgs): Promise<T> {
 		// Logging
 		LogService.log(ELogActions.REQUEST, { resource: resource, args: args });
 
@@ -182,9 +207,6 @@ export class FetcherService {
 		// Validating args
 		args = this.validateArgs(resource, args)!;
 
-		// Getting HTTPS agent
-		const httpsAgent: Agent = this.getHttpsAgent(this._proxyUrl);
-
 		// Getting credentials from key
 		const cred: AuthCredential = await this.getCredential();
 
@@ -192,13 +214,21 @@ export class FetcherService {
 		const config = requests[resource](args);
 
 		// Setting additional request parameters
-		config.headers = { ...config.headers, ...cred.toHeader() };
-		config.httpAgent = httpsAgent;
-		config.httpsAgent = httpsAgent;
+		config.headers = {
+			...config.headers,
+			...cred.toHeader(),
+			...(await this.getTransactionHeader(config.method ?? '', config.url ?? '')),
+			...this.config.headers,
+		};
+		config.httpAgent = this.config.httpsAgent;
+		config.httpsAgent = this.config.httpsAgent;
 		config.timeout = this._timeout;
 
 		// Sending the request
 		try {
+			// Introducing a delay
+			await this.wait();
+
 			// Returning the reponse body
 			return (await axios<T>(config)).data;
 		} catch (error) {

package/src/services/public/ListService.ts

@@ -1,11 +1,11 @@
-import { IListMembersResponse, IListTweetsResponse } from 'rettiwt-core';
-
 import { extractors } from '../../collections/Extractors';
 import { EResourceType } from '../../enums/Resource';
 import { CursoredData } from '../../models/data/CursoredData';
 import { Tweet } from '../../models/data/Tweet';
 import { User } from '../../models/data/User';
-import { IRettiwtConfig } from '../../types/RettiwtConfig';
+import { RettiwtConfig } from '../../models/RettiwtConfig';
+import { IListMembersResponse } from '../../types/raw/list/Members';
+import { IListTweetsResponse } from '../../types/raw/list/Tweets';
 
 import { FetcherService } from './FetcherService';
 
@@ -15,7 +15,7 @@ export class ListService extends FetcherService {
 	 *
 	 * @internal
 	 */
-	public constructor(config?: IRettiwtConfig) {
+	public constructor(config: RettiwtConfig) {
 		super(config);
 	}