triangle-utils 1.4.68 → 1.4.70

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,5 +3,8 @@ export declare class UtilsTwitter {
3
3
  constructor(twitter_api_key: string | undefined);
4
4
  get_user(user_id: string): Promise<any>;
5
5
  get_user_by_username(username: string): Promise<any>;
6
- get_tweets(user_id: string): Promise<any>;
6
+ get_tweets(user_id: string, options: {
7
+ compile?: boolean;
8
+ min_twitter_tweet_id?: string;
9
+ }): Promise<any>;
7
10
  }
@@ -1,107 +1,111 @@
1
1
  import { Client } from "@xdevplatform/xdk";
2
+ import { UtilsMisc } from "./UtilsMisc.js";
3
+ const user_fields = [
4
+ "affiliation",
5
+ // "confirmed_email",
6
+ "connection_status",
7
+ "created_at",
8
+ "description",
9
+ "entities",
10
+ "id",
11
+ "is_identity_verified",
12
+ "location",
13
+ "most_recent_tweet_id",
14
+ "name",
15
+ "parody",
16
+ "pinned_tweet_id",
17
+ "profile_banner_url",
18
+ "profile_image_url",
19
+ "protected",
20
+ // "public_metrics",
21
+ // "receives_your_dm",
22
+ "subscription",
23
+ "subscription_type",
24
+ "url",
25
+ "username",
26
+ "verified",
27
+ "verified_followers_count",
28
+ "verified_type",
29
+ "withheld"
30
+ ];
31
+ const tweet_fields = [
32
+ "article",
33
+ "attachments",
34
+ "author_id",
35
+ "card_uri",
36
+ "community_id",
37
+ "context_annotations",
38
+ "conversation_id",
39
+ "created_at",
40
+ "display_text_range",
41
+ "edit_controls",
42
+ "edit_history_tweet_ids",
43
+ "entities",
44
+ "geo",
45
+ "id",
46
+ "in_reply_to_user_id",
47
+ "lang",
48
+ "matched_media_notes",
49
+ "media_metadata",
50
+ "note_request_suggestions",
51
+ "note_tweet",
52
+ "paid_partnership",
53
+ "possibly_sensitive",
54
+ "referenced_tweets",
55
+ "reply_settings",
56
+ "scopes",
57
+ "source",
58
+ "suggested_source_links",
59
+ "suggested_source_links_with_counts",
60
+ "text",
61
+ "withheld"
62
+ ];
2
63
  export class UtilsTwitter {
3
64
  twitter;
4
65
  constructor(twitter_api_key) {
5
66
  this.twitter = new Client({ bearerToken: twitter_api_key });
6
67
  }
7
68
  async get_user(user_id) {
8
- const result = await this.twitter.users.getById(user_id, { userFields: [
9
- "affiliation",
10
- // "confirmed_email",
11
- "connection_status",
12
- "created_at",
13
- "description",
14
- "entities",
15
- "id",
16
- "is_identity_verified",
17
- "location",
18
- "most_recent_tweet_id",
19
- "name",
20
- "parody",
21
- "pinned_tweet_id",
22
- "profile_banner_url",
23
- "profile_image_url",
24
- "protected",
25
- // "public_metrics",
26
- // "receives_your_dm",
27
- "subscription",
28
- "subscription_type",
29
- "url",
30
- "username",
31
- "verified",
32
- "verified_followers_count",
33
- "verified_type",
34
- "withheld"
35
- ] })
69
+ const result = await this.twitter.users.getById(user_id, { userFields: user_fields })
36
70
  .then(response => response.data);
37
71
  return result;
38
72
  }
39
73
  async get_user_by_username(username) {
40
- const result = await this.twitter.users.getByUsername(username, { userFields: [
41
- "affiliation",
42
- // "confirmed_email",
43
- "connection_status",
44
- "created_at",
45
- "description",
46
- "entities",
47
- "id",
48
- "is_identity_verified",
49
- "location",
50
- "most_recent_tweet_id",
51
- "name",
52
- "parody",
53
- "pinned_tweet_id",
54
- "profile_banner_url",
55
- "profile_image_url",
56
- "protected",
57
- // "public_metrics",
58
- // "receives_your_dm",
59
- "subscription",
60
- "subscription_type",
61
- "url",
62
- "username",
63
- "verified",
64
- "verified_followers_count",
65
- "verified_type",
66
- "withheld"
67
- ] })
74
+ const result = await this.twitter.users.getByUsername(username, { userFields: user_fields })
68
75
  .then(response => response.data);
69
76
  return result;
70
77
  }
71
- async get_tweets(user_id) {
72
- const response = await this.twitter.users.getPosts(user_id, { tweetFields: [
73
- "article",
74
- "attachments",
75
- "author_id",
76
- "card_uri",
77
- "community_id",
78
- "context_annotations",
79
- "conversation_id",
80
- "created_at",
81
- "display_text_range",
82
- "edit_controls",
83
- "edit_history_tweet_ids",
84
- "entities",
85
- "geo",
86
- "id",
87
- "in_reply_to_user_id",
88
- "lang",
89
- "matched_media_notes",
90
- "media_metadata",
91
- "note_request_suggestions",
92
- "note_tweet",
93
- "paid_partnership",
94
- "possibly_sensitive",
95
- "referenced_tweets",
96
- "reply_settings",
97
- "scopes",
98
- "source",
99
- "suggested_source_links",
100
- "suggested_source_links_with_counts",
101
- "text",
102
- "withheld"
103
- ] });
104
- console.log("RESPONSE", response);
105
- return response.data;
78
+ async get_tweets(user_id, options) {
79
+ const compile = options.compile !== undefined ? options.compile : true;
80
+ const results = [];
81
+ let token = undefined;
82
+ while (true) {
83
+ let response = undefined;
84
+ for (let i = 0;; i++) {
85
+ try {
86
+ response = await this.twitter.users.getPosts(user_id, {
87
+ paginationToken: token,
88
+ sinceId: options.min_twitter_tweet_id,
89
+ tweetFields: tweet_fields
90
+ });
91
+ break;
92
+ }
93
+ catch (error) {
94
+ console.log(error.stack);
95
+ if (error.toString().includes("429")) {
96
+ console.log(30000);
97
+ UtilsMisc.wait(30000);
98
+ continue;
99
+ }
100
+ return [];
101
+ }
102
+ }
103
+ results.push(...(response.data || []));
104
+ token = (response.meta || {}).nextToken;
105
+ console.log("Token:", token);
106
+ if (token === undefined || !compile) {
107
+ return results;
108
+ }
109
+ }
106
110
  }
107
111
  }
@@ -37,6 +37,9 @@ export class UtilsXAI {
37
37
  tools: [
38
38
  {
39
39
  type: "x_search"
40
+ },
41
+ {
42
+ type: "web_search"
40
43
  }
41
44
  ],
42
45
  temperature: 0,
package/dist/src/f.js CHANGED
@@ -15,26 +15,26 @@ const config = {
15
15
  };
16
16
  console.log(config);
17
17
  const utils = new TriangleUtils(config);
18
- const text = await utils.xai.grok_simple_query("grok-4.3", "Find the X username corresponding to the candidate of the committee \"Darializa for Congress\"", {
19
- print_usage: true,
20
- json_format: {
21
- type: "object",
22
- properties: {
23
- username: {
24
- type: "string"
25
- },
26
- explanation: {
27
- type: "string"
28
- }
29
- },
30
- required: [
31
- "username",
32
- "explanation"
33
- ],
34
- additionalProperties: false
35
- }
36
- });
37
- console.log(text);
18
+ // const text = await utils.xai.grok_simple_query("grok-4.3", "Find the X username corresponding to the candidate of the committee \"Darializa for Congress\"", {
19
+ // print_usage : true,
20
+ // json_format : {
21
+ // type : "object",
22
+ // properties : {
23
+ // username : {
24
+ // type: "string"
25
+ // },
26
+ // explanation : {
27
+ // type: "string"
28
+ // }
29
+ // },
30
+ // required : [
31
+ // "username",
32
+ // "explanation"
33
+ // ],
34
+ // additionalProperties : false
35
+ // }
36
+ // })
37
+ // console.log(text)
38
38
  // const foods = await utils.dynamodb.query("triage_docket_documents:register_document_id", { register_document_id : "E6-17065" })
39
39
  // console.log(foods)
40
40
  // const text = await utils.bee.get("https://www.doyourjobs.org", { return_page_text : true })
@@ -100,7 +100,8 @@ console.log(text);
100
100
  // "withheld"
101
101
  // ] })
102
102
  // console.log(JSON.stringify(user, null, 4))
103
- // const tweets = await x.users.getPosts(user.data?.id || "", { tweetFields : [
103
+ // const tweets = await x.users.getPosts("1603470469608374272", {
104
+ // tweetFields : [
104
105
  // "article",
105
106
  // "attachments",
106
107
  // "author_id",
@@ -131,10 +132,87 @@ console.log(text);
131
132
  // "suggested_source_links_with_counts",
132
133
  // "text",
133
134
  // "withheld"
135
+ // ], expansions : [
136
+ // "article.cover_media",
137
+ // "article.media_entities",
138
+ // "attachments.media_keys",
139
+ // "attachments.media_source_tweet",
140
+ // "attachments.poll_ids",
141
+ // "author_id",
142
+ // "edit_history_tweet_ids",
143
+ // "entities.mentions.username",
144
+ // "geo.place_id",
145
+ // "in_reply_to_user_id",
146
+ // "entities.note.mentions.username",
147
+ // "referenced_tweets.id",
148
+ // "referenced_tweets.id.attachments.media_keys",
149
+ // "referenced_tweets.id.author_id"
134
150
  // ]})
135
- // console.log(tweets)
151
+ // console.log(JSON.stringify(tweets, null, 4))
136
152
  // for (const tweet of tweets.data || []) {
137
153
  // console.log(JSON.stringify(tweet, undefined, 4))
138
154
  // }
139
- // const tweet = await x.posts.getReposts("2068675973722128815")
140
- // console.log(tweets)
155
+ // const tweet = await x.posts.getById("2061603004248154195", { tweetFields : [
156
+ // "article",
157
+ // "attachments",
158
+ // "author_id",
159
+ // "card_uri",
160
+ // "community_id",
161
+ // "context_annotations",
162
+ // "conversation_id",
163
+ // "created_at",
164
+ // "display_text_range",
165
+ // "edit_controls",
166
+ // "edit_history_tweet_ids",
167
+ // "entities",
168
+ // "geo",
169
+ // "id",
170
+ // "in_reply_to_user_id",
171
+ // "lang",
172
+ // "matched_media_notes",
173
+ // "media_metadata",
174
+ // // "non_public_metrics",
175
+ // "note_request_suggestions",
176
+ // "note_tweet",
177
+ // // "organic_metrics",
178
+ // "paid_partnership",
179
+ // "possibly_sensitive",
180
+ // // "promoted_metrics",
181
+ // "public_metrics",
182
+ // "referenced_tweets",
183
+ // "reply_settings",
184
+ // "scopes",
185
+ // "source",
186
+ // "suggested_source_links",
187
+ // "suggested_source_links_with_counts",
188
+ // "text",
189
+ // "withheld"
190
+ // ], expansions : [
191
+ // "article.cover_media",
192
+ // "article.media_entities",
193
+ // "attachments.media_keys",
194
+ // "attachments.media_source_tweet",
195
+ // "attachments.poll_ids",
196
+ // "author_id",
197
+ // "edit_history_tweet_ids",
198
+ // "entities.mentions.username",
199
+ // "geo.place_id",
200
+ // "in_reply_to_user_id",
201
+ // "entities.note.mentions.username",
202
+ // "referenced_tweets.id",
203
+ // "referenced_tweets.id.attachments.media_keys",
204
+ // "referenced_tweets.id.author_id"
205
+ // ], pollFields : [
206
+ // "duration_minutes",
207
+ // "end_datetime",
208
+ // "id",
209
+ // "options",
210
+ // "voting_status"
211
+ // ]
212
+ // })
213
+ // console.log(JSON.stringify(tweet, undefined, 4))
214
+ const results = await utils.twitter.get_tweets("1603470469608374272", {
215
+ compile: true,
216
+ min_twitter_tweet_id: "2061610563679985813"
217
+ });
218
+ console.log(results);
package/package.json CHANGED
@@ -1,48 +1,48 @@
1
- {
2
- "name": "triangle-utils",
3
- "version": "1.4.68",
4
- "main": "dist/src/index.js",
5
- "types": "dist/src/index.d.ts",
6
- "directories": {
7
- "test": "vitest"
8
- },
9
- "scripts": {
10
- "test": "vitest",
11
- "build": "tsc && tsc-alias"
12
- },
13
- "author": "",
14
- "license": "ISC",
15
- "description": "",
16
- "type": "module",
17
- "dependencies": {
18
- "@anthropic-ai/sdk": "^0.94.0",
19
- "@aws-sdk/client-bedrock-runtime": "^3.953.0",
20
- "@aws-sdk/client-cognito-identity-provider": "^3.1004.0",
21
- "@aws-sdk/client-dynamodb": "^3.953.0",
22
- "@aws-sdk/client-s3": "^3.953.0",
23
- "@aws-sdk/client-s3vectors": "^3.953.0",
24
- "@aws-sdk/client-secrets-manager": "^3.965.0",
25
- "@aws-sdk/client-ses": "^3.1032.0",
26
- "@aws-sdk/s3-request-presigner": "^3.953.0",
27
- "@types/jsdom": "^28.0.1",
28
- "@types/node": "^25.2.3",
29
- "@types/nodemailer": "^7.0.9",
30
- "@xdevplatform/xdk": "^0.5.0",
31
- "googleapis": "^170.0.0",
32
- "jsdom": "^29.0.1",
33
- "scrapingbee": "^1.8.2"
34
- },
35
- "devDependencies": {
36
- "@eslint/js": "^9.39.2",
37
- "@types/jest": "^30.0.0",
38
- "eslint": "^9.39.2",
39
- "globals": "^17.3.0",
40
- "jiti": "^2.6.1",
41
- "ts-jest": "^29.4.6",
42
- "tsc-alias": "^1.8.16",
43
- "tsx": "^4.21.0",
44
- "typescript": "^5.9.3",
45
- "typescript-eslint": "^8.55.0",
46
- "vitest": "^4.0.18"
47
- }
48
- }
1
+ {
2
+ "name": "triangle-utils",
3
+ "version": "1.4.70",
4
+ "main": "dist/src/index.js",
5
+ "types": "dist/src/index.d.ts",
6
+ "directories": {
7
+ "test": "vitest"
8
+ },
9
+ "scripts": {
10
+ "test": "vitest",
11
+ "build": "tsc && tsc-alias"
12
+ },
13
+ "author": "",
14
+ "license": "ISC",
15
+ "description": "",
16
+ "type": "module",
17
+ "dependencies": {
18
+ "@anthropic-ai/sdk": "^0.94.0",
19
+ "@aws-sdk/client-bedrock-runtime": "^3.953.0",
20
+ "@aws-sdk/client-cognito-identity-provider": "^3.1004.0",
21
+ "@aws-sdk/client-dynamodb": "^3.953.0",
22
+ "@aws-sdk/client-s3": "^3.953.0",
23
+ "@aws-sdk/client-s3vectors": "^3.953.0",
24
+ "@aws-sdk/client-secrets-manager": "^3.965.0",
25
+ "@aws-sdk/client-ses": "^3.1032.0",
26
+ "@aws-sdk/s3-request-presigner": "^3.953.0",
27
+ "@types/jsdom": "^28.0.1",
28
+ "@types/node": "^25.2.3",
29
+ "@types/nodemailer": "^7.0.9",
30
+ "@xdevplatform/xdk": "^0.5.0",
31
+ "googleapis": "^170.0.0",
32
+ "jsdom": "^29.0.1",
33
+ "scrapingbee": "^1.8.2"
34
+ },
35
+ "devDependencies": {
36
+ "@eslint/js": "^9.39.2",
37
+ "@types/jest": "^30.0.0",
38
+ "eslint": "^9.39.2",
39
+ "globals": "^17.3.0",
40
+ "jiti": "^2.6.1",
41
+ "ts-jest": "^29.4.6",
42
+ "tsc-alias": "^1.8.16",
43
+ "tsx": "^4.21.0",
44
+ "typescript": "^5.9.3",
45
+ "typescript-eslint": "^8.55.0",
46
+ "vitest": "^4.0.18"
47
+ }
48
+ }
@@ -1,4 +1,67 @@
1
1
  import { Client } from "@xdevplatform/xdk"
2
+ import { UtilsMisc } from "./UtilsMisc"
3
+
4
+ const user_fields = [
5
+ "affiliation",
6
+ // "confirmed_email",
7
+ "connection_status",
8
+ "created_at",
9
+ "description",
10
+ "entities",
11
+ "id",
12
+ "is_identity_verified",
13
+ "location",
14
+ "most_recent_tweet_id",
15
+ "name",
16
+ "parody",
17
+ "pinned_tweet_id",
18
+ "profile_banner_url",
19
+ "profile_image_url",
20
+ "protected",
21
+ // "public_metrics",
22
+ // "receives_your_dm",
23
+ "subscription",
24
+ "subscription_type",
25
+ "url",
26
+ "username",
27
+ "verified",
28
+ "verified_followers_count",
29
+ "verified_type",
30
+ "withheld"
31
+ ]
32
+
33
+ const tweet_fields = [
34
+ "article",
35
+ "attachments",
36
+ "author_id",
37
+ "card_uri",
38
+ "community_id",
39
+ "context_annotations",
40
+ "conversation_id",
41
+ "created_at",
42
+ "display_text_range",
43
+ "edit_controls",
44
+ "edit_history_tweet_ids",
45
+ "entities",
46
+ "geo",
47
+ "id",
48
+ "in_reply_to_user_id",
49
+ "lang",
50
+ "matched_media_notes",
51
+ "media_metadata",
52
+ "note_request_suggestions",
53
+ "note_tweet",
54
+ "paid_partnership",
55
+ "possibly_sensitive",
56
+ "referenced_tweets",
57
+ "reply_settings",
58
+ "scopes",
59
+ "source",
60
+ "suggested_source_links",
61
+ "suggested_source_links_with_counts",
62
+ "text",
63
+ "withheld"
64
+ ]
2
65
 
3
66
  export class UtilsTwitter {
4
67
 
@@ -9,105 +72,50 @@ export class UtilsTwitter {
9
72
  }
10
73
 
11
74
  async get_user(user_id : string) : Promise<any> {
12
- const result = await this.twitter.users.getById(user_id, { userFields : [
13
- "affiliation",
14
- // "confirmed_email",
15
- "connection_status",
16
- "created_at",
17
- "description",
18
- "entities",
19
- "id",
20
- "is_identity_verified",
21
- "location",
22
- "most_recent_tweet_id",
23
- "name",
24
- "parody",
25
- "pinned_tweet_id",
26
- "profile_banner_url",
27
- "profile_image_url",
28
- "protected",
29
- // "public_metrics",
30
- // "receives_your_dm",
31
- "subscription",
32
- "subscription_type",
33
- "url",
34
- "username",
35
- "verified",
36
- "verified_followers_count",
37
- "verified_type",
38
- "withheld"
39
- ] })
75
+ const result = await this.twitter.users.getById(user_id, { userFields : user_fields })
40
76
  .then(response => response.data)
41
77
  return result
42
78
  }
43
79
 
44
80
  async get_user_by_username(username : string) : Promise<any> {
45
- const result = await this.twitter.users.getByUsername(username, { userFields : [
46
- "affiliation",
47
- // "confirmed_email",
48
- "connection_status",
49
- "created_at",
50
- "description",
51
- "entities",
52
- "id",
53
- "is_identity_verified",
54
- "location",
55
- "most_recent_tweet_id",
56
- "name",
57
- "parody",
58
- "pinned_tweet_id",
59
- "profile_banner_url",
60
- "profile_image_url",
61
- "protected",
62
- // "public_metrics",
63
- // "receives_your_dm",
64
- "subscription",
65
- "subscription_type",
66
- "url",
67
- "username",
68
- "verified",
69
- "verified_followers_count",
70
- "verified_type",
71
- "withheld"
72
- ] })
81
+ const result = await this.twitter.users.getByUsername(username, { userFields : user_fields })
73
82
  .then(response => response.data)
74
83
  return result
75
84
  }
76
85
 
77
- async get_tweets(user_id : string) : Promise<any> {
78
- const response = await this.twitter.users.getPosts(user_id, { tweetFields : [
79
- "article",
80
- "attachments",
81
- "author_id",
82
- "card_uri",
83
- "community_id",
84
- "context_annotations",
85
- "conversation_id",
86
- "created_at",
87
- "display_text_range",
88
- "edit_controls",
89
- "edit_history_tweet_ids",
90
- "entities",
91
- "geo",
92
- "id",
93
- "in_reply_to_user_id",
94
- "lang",
95
- "matched_media_notes",
96
- "media_metadata",
97
- "note_request_suggestions",
98
- "note_tweet",
99
- "paid_partnership",
100
- "possibly_sensitive",
101
- "referenced_tweets",
102
- "reply_settings",
103
- "scopes",
104
- "source",
105
- "suggested_source_links",
106
- "suggested_source_links_with_counts",
107
- "text",
108
- "withheld"
109
- ]})
110
- console.log("RESPONSE", response)
111
- return response.data
86
+ async get_tweets(user_id : string, options : {
87
+ compile? : boolean
88
+ min_twitter_tweet_id? : string
89
+ }) : Promise<any> {
90
+ const compile = options.compile !== undefined ? options.compile : true
91
+ const results : any[] = []
92
+ let token : string | undefined = undefined
93
+ while (true) {
94
+ let response : any = undefined
95
+ for (let i = 0; ; i++) {
96
+ try {
97
+ response = await this.twitter.users.getPosts(user_id, {
98
+ paginationToken : token,
99
+ sinceId : options.min_twitter_tweet_id,
100
+ tweetFields : tweet_fields
101
+ })
102
+ break
103
+ } catch (error : any) {
104
+ console.log(error.stack)
105
+ if (error.toString().includes("429")) {
106
+ console.log(30000)
107
+ UtilsMisc.wait(30000)
108
+ continue
109
+ }
110
+ return []
111
+ }
112
+ }
113
+ results.push(...(response.data || []))
114
+ token = (response.meta || {}).nextToken
115
+ console.log("Token:", token)
116
+ if (token === undefined || !compile) {
117
+ return results
118
+ }
119
+ }
112
120
  }
113
121
  }
package/src/UtilsXAI.ts CHANGED
@@ -64,6 +64,9 @@ export class UtilsXAI {
64
64
  tools : [
65
65
  {
66
66
  type : "x_search"
67
+ },
68
+ {
69
+ type : "web_search"
67
70
  }
68
71
  ],
69
72
  temperature : 0,
package/src/f.ts CHANGED
@@ -24,26 +24,26 @@ console.log(config)
24
24
 
25
25
  const utils = new TriangleUtils(config)
26
26
 
27
- const text = await utils.xai.grok_simple_query("grok-4.3", "Find the X username corresponding to the candidate of the committee \"Darializa for Congress\"", {
28
- print_usage : true,
29
- json_format : {
30
- type : "object",
31
- properties : {
32
- username : {
33
- type: "string"
34
- },
35
- explanation : {
36
- type: "string"
37
- }
38
- },
39
- required : [
40
- "username",
41
- "explanation"
42
- ],
43
- additionalProperties : false
44
- }
45
- })
46
- console.log(text)
27
+ // const text = await utils.xai.grok_simple_query("grok-4.3", "Find the X username corresponding to the candidate of the committee \"Darializa for Congress\"", {
28
+ // print_usage : true,
29
+ // json_format : {
30
+ // type : "object",
31
+ // properties : {
32
+ // username : {
33
+ // type: "string"
34
+ // },
35
+ // explanation : {
36
+ // type: "string"
37
+ // }
38
+ // },
39
+ // required : [
40
+ // "username",
41
+ // "explanation"
42
+ // ],
43
+ // additionalProperties : false
44
+ // }
45
+ // })
46
+ // console.log(text)
47
47
 
48
48
  // const foods = await utils.dynamodb.query("triage_docket_documents:register_document_id", { register_document_id : "E6-17065" })
49
49
 
@@ -123,9 +123,12 @@ console.log(text)
123
123
  // "withheld"
124
124
  // ] })
125
125
 
126
+
126
127
  // console.log(JSON.stringify(user, null, 4))
127
128
 
128
- // const tweets = await x.users.getPosts(user.data?.id || "", { tweetFields : [
129
+ // const tweets = await x.users.getPosts("1603470469608374272", {
130
+
131
+ // tweetFields : [
129
132
  // "article",
130
133
  // "attachments",
131
134
  // "author_id",
@@ -156,14 +159,93 @@ console.log(text)
156
159
  // "suggested_source_links_with_counts",
157
160
  // "text",
158
161
  // "withheld"
162
+ // ], expansions : [
163
+ // "article.cover_media",
164
+ // "article.media_entities",
165
+ // "attachments.media_keys",
166
+ // "attachments.media_source_tweet",
167
+ // "attachments.poll_ids",
168
+ // "author_id",
169
+ // "edit_history_tweet_ids",
170
+ // "entities.mentions.username",
171
+ // "geo.place_id",
172
+ // "in_reply_to_user_id",
173
+ // "entities.note.mentions.username",
174
+ // "referenced_tweets.id",
175
+ // "referenced_tweets.id.attachments.media_keys",
176
+ // "referenced_tweets.id.author_id"
159
177
  // ]})
160
178
 
161
- // console.log(tweets)
179
+ // console.log(JSON.stringify(tweets, null, 4))
162
180
 
163
181
  // for (const tweet of tweets.data || []) {
164
182
  // console.log(JSON.stringify(tweet, undefined, 4))
165
183
  // }
166
184
 
167
- // const tweet = await x.posts.getReposts("2068675973722128815")
185
+ // const tweet = await x.posts.getById("2061603004248154195", { tweetFields : [
186
+ // "article",
187
+ // "attachments",
188
+ // "author_id",
189
+ // "card_uri",
190
+ // "community_id",
191
+ // "context_annotations",
192
+ // "conversation_id",
193
+ // "created_at",
194
+ // "display_text_range",
195
+ // "edit_controls",
196
+ // "edit_history_tweet_ids",
197
+ // "entities",
198
+ // "geo",
199
+ // "id",
200
+ // "in_reply_to_user_id",
201
+ // "lang",
202
+ // "matched_media_notes",
203
+ // "media_metadata",
204
+ // // "non_public_metrics",
205
+ // "note_request_suggestions",
206
+ // "note_tweet",
207
+ // // "organic_metrics",
208
+ // "paid_partnership",
209
+ // "possibly_sensitive",
210
+ // // "promoted_metrics",
211
+ // "public_metrics",
212
+ // "referenced_tweets",
213
+ // "reply_settings",
214
+ // "scopes",
215
+ // "source",
216
+ // "suggested_source_links",
217
+ // "suggested_source_links_with_counts",
218
+ // "text",
219
+ // "withheld"
220
+ // ], expansions : [
221
+ // "article.cover_media",
222
+ // "article.media_entities",
223
+ // "attachments.media_keys",
224
+ // "attachments.media_source_tweet",
225
+ // "attachments.poll_ids",
226
+ // "author_id",
227
+ // "edit_history_tweet_ids",
228
+ // "entities.mentions.username",
229
+ // "geo.place_id",
230
+ // "in_reply_to_user_id",
231
+ // "entities.note.mentions.username",
232
+ // "referenced_tweets.id",
233
+ // "referenced_tweets.id.attachments.media_keys",
234
+ // "referenced_tweets.id.author_id"
235
+ // ], pollFields : [
236
+ // "duration_minutes",
237
+ // "end_datetime",
238
+ // "id",
239
+ // "options",
240
+ // "voting_status"
241
+ // ]
242
+ // })
243
+
244
+ // console.log(JSON.stringify(tweet, undefined, 4))
245
+
246
+ const results = await utils.twitter.get_tweets("1603470469608374272", {
247
+ compile : true,
248
+ min_twitter_tweet_id : "2061610563679985813"
249
+ })
168
250
 
169
- // console.log(tweets)
251
+ console.log(results)
@@ -1,186 +0,0 @@
1
- import { JSDOM } from "jsdom"
2
-
3
- import { UtilsBee } from "./UtilsBee"
4
- import { UtilsMisc } from "./UtilsMisc"
5
-
6
- export class Tweet {
7
- readonly tweet_id : string
8
- readonly username : string
9
- readonly url : string
10
- readonly tweet_time : string
11
- readonly tweet_content : string
12
- readonly quote? : string
13
- readonly html : string
14
- readonly text : string
15
-
16
- constructor(tweet : any) {
17
- if (!Tweet.is(tweet)) {
18
- throw Error("Invalid input.")
19
- }
20
- this.tweet_id = tweet.tweet_id
21
- this.username = tweet.username
22
- this.url = tweet.url
23
- this.tweet_time = tweet.tweet_time
24
- this.tweet_content = tweet.tweet_content
25
- this.quote = tweet.quote
26
- this.html = tweet.html
27
- this.text = tweet.text
28
- }
29
-
30
- static is(tweet : any) : tweet is Tweet {
31
- return (
32
- tweet !== undefined &&
33
- typeof tweet.tweet_id === "string" &&
34
- typeof tweet.username === "string" &&
35
- typeof tweet.url === "string" &&
36
- typeof tweet.tweet_time === "string" &&
37
- typeof tweet.tweet_content === "string" &&
38
- (tweet.quote === undefined || typeof tweet.quote === "string") &&
39
- (typeof tweet.html === "string") &&
40
- (typeof tweet.text === "string")
41
- )
42
- }
43
- }
44
-
45
- function parse_tweet_element(tweet_element : Element) : Tweet | undefined {
46
- if (tweet_element.className.includes("show-more")) {
47
- return undefined
48
- }
49
- const tweet_link = tweet_element.querySelector(".tweet-link")
50
- if (tweet_link === null) {
51
- return undefined
52
- }
53
- const url_element = tweet_link.getAttribute("href")
54
- const url = url_element !== null ? url_element.replace("#m", "") : undefined
55
- if (url === undefined) {
56
- return undefined
57
- }
58
- const username = url.split("/")[1]
59
- const tweet_number = url.split("/")[3].padStart(20, "0")
60
- const tweet_date = tweet_element.querySelector(".tweet-date")?.querySelector("a")?.getAttribute("title")?.replace(" · ", " ")
61
- if (tweet_date === undefined) {
62
- return undefined
63
- }
64
- const tweet_time = (new Date(tweet_date)).toISOString()
65
- const tweet_id = tweet_time.substring(0, 16) + "|" + tweet_number
66
- const tweet_content = tweet_element.querySelector(".tweet-content")?.innerHTML?.replaceAll(/\<[^\>]+\>/g, "")
67
- if (tweet_content === undefined) {
68
- return undefined
69
- }
70
- const tweet_quote = tweet_element.querySelector(".quote-big")
71
- const quote = tweet_quote !== null ?
72
- tweet_quote.innerHTML
73
- .replaceAll(/\<[^\>]+\>/g, "")
74
- .replaceAll(/[^\n\S\r]+/g, " ").split("\n").map(line => line.trim()).filter(line => line !== "").join("\n")
75
- :
76
- undefined
77
- const text = "Tweet from @" + username + " on " + tweet_time + ":\n\n" +
78
- tweet_content +
79
- (quote !== undefined ? ("\n\nQuote:\n\n" + quote): "")
80
- const html = tweet_element.innerHTML
81
- const tweet : Tweet = {
82
- tweet_id : tweet_id,
83
- username : username,
84
- url : url,
85
- tweet_time : tweet_time,
86
- tweet_content : tweet_content,
87
- quote : quote,
88
- html : html,
89
- text : text
90
- }
91
- return tweet
92
- }
93
-
94
- function parse_nitter_html(nitter_html : string) {
95
- const dom = new JSDOM(nitter_html)
96
- const root = dom.window.document.body
97
- const tweet_elements = root.querySelectorAll(".timeline-item")
98
- const tweets = Array.from(tweet_elements).map(parse_tweet_element).filter(tweet => tweet !== undefined)
99
- const next_button = root.querySelector(".show-more:not(.timeline-item)")
100
- const next_button_link = next_button !== null ? next_button.querySelector("a") : null
101
- const next_nitter_query = (next_button_link !== null ? next_button_link.getAttribute("href") : null) || undefined
102
- return { tweets : tweets, next_nitter_query : next_nitter_query }
103
- }
104
-
105
-
106
- export class UtilsNitter extends UtilsBee {
107
-
108
- constructor(scraping_bee_api_key : string | undefined) {
109
- super(scraping_bee_api_key)
110
- }
111
-
112
- async get_tweets(username : string, options? : { min_tweet_id? : string, min_tweet_time? : string }) : Promise<Tweet[]> {
113
- const min_tweet_id = options !== undefined ? options.min_tweet_id : undefined
114
- const min_tweet_time = options !== undefined ? options.min_tweet_time : undefined
115
- console.log("Querying Twitter: @" + username, "min_tweet_id:", min_tweet_id, "min_tweet_time:", min_tweet_time)
116
- let nitter_query : string | undefined = ""
117
- const tweets = []
118
- while (nitter_query !== undefined) {
119
- const nitter_url = "https://nitter.net/" + username + nitter_query
120
- console.log(nitter_url)
121
- let nitter_html = undefined
122
- for (let i = 0; i < 10; i++) {
123
- nitter_html = await this.get(nitter_url, { render_js : false })
124
- if (nitter_html !== undefined && nitter_html !== "" && !(nitter_html instanceof Error)) {
125
- break
126
- }
127
- console.log("Failed to query Nitter, trying again.")
128
- await UtilsMisc.wait((2 + 2 * i) * 1000)
129
- }
130
- if (nitter_html === undefined || nitter_html === "" || nitter_html instanceof Error) {
131
- console.log("Nitter failed for url", nitter_url)
132
- return []
133
- }
134
- const nitter_data = parse_nitter_html(nitter_html)
135
- nitter_query = nitter_data.next_nitter_query
136
- for (const tweet of nitter_data.tweets) {
137
- if (min_tweet_id !== undefined && tweet.tweet_id.localeCompare(min_tweet_id) < 0) {
138
- nitter_query = undefined
139
- }
140
- if (min_tweet_time !== undefined && tweet.tweet_time.localeCompare(min_tweet_time) < 0) {
141
- nitter_query = undefined
142
- }
143
- tweets.push(tweet)
144
- }
145
- }
146
- return tweets
147
- }
148
-
149
- async search(query : string, options? : { min_tweet_id? : string, min_tweet_time? : string }) : Promise<Tweet[]> {
150
- const min_tweet_id = options !== undefined ? options.min_tweet_id : undefined
151
- const min_tweet_time = options !== undefined ? options.min_tweet_time : undefined
152
-
153
- console.log("Querying Twitter:", query, "min_tweet_id:", min_tweet_id, "min_tweet_time:", min_tweet_time)
154
- let nitter_query : string | undefined = "?f=tweets&q=" + query + "&e-nativeretweets=on"
155
- const tweets = []
156
- while (nitter_query !== undefined) {
157
- const nitter_url = "https://nitter.net/search" + nitter_query
158
- console.log(nitter_url)
159
- let nitter_html = undefined
160
- for (let i = 0; i < 10; i++) {
161
- nitter_html = await this.get(nitter_url, { render_js : false })
162
- if (nitter_html !== undefined && nitter_html !== "" && !(nitter_html instanceof Error)) {
163
- break
164
- }
165
- console.log("Failed to query Nitter, trying again.")
166
- await UtilsMisc.wait((2 + 2 * i) * 1000)
167
- }
168
- if (nitter_html === undefined || nitter_html === "" || nitter_html instanceof Error) {
169
- console.log("Nitter failed for url", nitter_url)
170
- return []
171
- }
172
- const nitter_data = parse_nitter_html(nitter_html)
173
- nitter_query = nitter_data.next_nitter_query
174
- for (const tweet of nitter_data.tweets) {
175
- if (min_tweet_id !== undefined && tweet.tweet_id.localeCompare(min_tweet_id) < 0) {
176
- nitter_query = undefined
177
- }
178
- if (min_tweet_time !== undefined && tweet.tweet_time.localeCompare(min_tweet_time) < 0) {
179
- nitter_query = undefined
180
- }
181
- tweets.push(tweet)
182
- }
183
- }
184
- return tweets
185
- }
186
- }