user-agents 1.1.8 → 2.0.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,48 +1,69 @@
1
1
  {
2
2
  "name": "user-agents",
3
- "version": "1.1.8",
3
+ "version": "2.0.0-alpha.3",
4
4
  "description": "A JavaScript library for generating random user agents. ",
5
- "main": "dist/index.js",
5
+ "main": "dist/index.cjs",
6
+ "module": "dist/index.js",
6
7
  "repository": "git@github.com:intoli/user-agents.git",
7
8
  "author": "Intoli, LLC <contact@intoli.com>",
9
+ "files": [
10
+ "dist/",
11
+ "src/**/*",
12
+ "!src/user-agents.json",
13
+ "!src/user-agents.json.gz"
14
+ ],
15
+ "exports": {
16
+ "import": "dist/index.js",
17
+ "require": "dist/index.cjs"
18
+ },
8
19
  "license": "BSD-2-Clause",
9
20
  "private": false,
21
+ "type": "module",
10
22
  "scripts": {
11
- "build": "NODE_ENV=production webpack",
12
- "gunzip-data": "babel-node src/gunzip-data.js src/user-agents.json.gz",
13
- "lint": "eslint src/",
23
+ "build": "tsup",
24
+ "postbuild": "yarn gunzip-data && cp src/user-agents.json dist/",
25
+ "gunzip-data": "node --loader ts-node/esm src/gunzip-data.ts src/user-agents.json.gz",
26
+ "lint": "eslint src/ && prettier --check src/",
14
27
  "postversion": "git push && git push --tags",
15
- "test": "NODE_ENV=testing mocha --exit --require @babel/register",
16
- "update-data": "babel-node src/update-data.js src/user-agents.json.gz"
28
+ "test": "NODE_ENV=testing mocha --exit --require @babel/register --extensions '.ts, .js'",
29
+ "update-data": "node --loader ts-node/esm src/update-data.ts src/user-agents.json.gz"
17
30
  },
18
31
  "devDependencies": {
19
- "@babel/cli": "^7.12.16",
20
- "@babel/core": "^7.12.16",
21
- "@babel/eslint-parser": "^7.12.16",
22
- "@babel/node": "^7.12.16",
23
- "@babel/plugin-proposal-class-properties": "^7.12.13",
24
- "@babel/plugin-proposal-object-rest-spread": "^7.12.13",
25
- "@babel/plugin-transform-classes": "^7.12.13",
26
- "@babel/preset-env": "^7.12.16",
27
- "@babel/register": "^7.12.13",
28
- "babel-loader": "^8.2.2",
32
+ "@babel/cli": "^7.23.0",
33
+ "@babel/core": "^7.23.2",
34
+ "@babel/plugin-proposal-class-properties": "^7.18.6",
35
+ "@babel/plugin-proposal-object-rest-spread": "^7.20.7",
36
+ "@babel/plugin-syntax-import-assertions": "^7.22.5",
37
+ "@babel/plugin-transform-classes": "^7.22.15",
38
+ "@babel/preset-env": "^7.23.2",
39
+ "@babel/preset-typescript": "^7.23.2",
40
+ "@babel/register": "^7.22.15",
41
+ "@types/lodash.clonedeep": "^4.5.8",
42
+ "@types/ua-parser-js": "^0.7.38",
43
+ "@typescript-eslint/eslint-plugin": "^6.9.0",
44
+ "@typescript-eslint/parser": "^6.9.0",
45
+ "babel-loader": "^9.1.3",
29
46
  "babel-plugin-add-module-exports": "^1.0.4",
30
47
  "babel-preset-power-assert": "^3.0.0",
31
48
  "dynamoose": "^3.2.1",
32
- "eslint": "^7.19.0",
33
- "eslint-config-airbnb": "^16.1.0",
34
- "eslint-loader": "^4.0.2",
35
- "eslint-plugin-import": "^2.22.1",
49
+ "esbuild": "^0.19.5",
50
+ "eslint": "^8.52.0",
51
+ "eslint-config-airbnb": "^19.0.4",
52
+ "eslint-config-prettier": "^9.0.0",
53
+ "eslint-plugin-import": "^2.29.0",
36
54
  "fast-json-stable-stringify": "^2.1.0",
37
- "imports-loader": "^2.0.0",
55
+ "imports-loader": "^4.0.1",
38
56
  "isbot": "^3.7.0",
39
- "mocha": "^8.3.0",
57
+ "mocha": "^10.2.0",
40
58
  "power-assert": "^1.6.1",
41
- "random": "^2.2.0",
42
- "source-map-support": "^0.5.19",
43
- "ua-parser-js": "^1.0.36",
44
- "webpack": "^5.21.2",
45
- "webpack-cli": "^4.5.0"
59
+ "prettier": "^3.0.3",
60
+ "random": "^4.1.0",
61
+ "source-map-support": "^0.5.21",
62
+ "ts-node": "^10.9.1",
63
+ "tsup": "^7.2.0",
64
+ "typescript": "^5.2.2",
65
+ "typescript-language-server": "^4.0.0",
66
+ "ua-parser-js": "^1.0.36"
46
67
  },
47
68
  "dependencies": {
48
69
  "lodash.clonedeep": "^4.5.0"
@@ -1,8 +1,9 @@
1
1
  import fs from 'fs';
2
+ import { argv } from 'process';
3
+ import { fileURLToPath } from 'url';
2
4
  import { gunzipSync } from 'zlib';
3
5
 
4
-
5
- const gunzipData = (inputFilename) => {
6
+ const gunzipData = (inputFilename?: string) => {
6
7
  if (!inputFilename || !inputFilename.endsWith('.gz')) {
7
8
  throw new Error('Filename must be specified and end with `.gz` for gunzipping.');
8
9
  }
@@ -12,11 +13,9 @@ const gunzipData = (inputFilename) => {
12
13
  fs.writeFileSync(outputFilename, data);
13
14
  };
14
15
 
15
-
16
- if (!module.parent) {
16
+ if (fileURLToPath(import.meta.url) === argv[1]) {
17
17
  const inputFilename = process.argv[2];
18
18
  gunzipData(inputFilename);
19
19
  }
20
20
 
21
-
22
21
  export default gunzipData;
package/src/index.ts ADDED
@@ -0,0 +1,4 @@
1
+ import { Filter, UserAgent, UserAgentData } from './user-agent';
2
+
3
+ export default UserAgent;
4
+ export type { Filter, UserAgentData };
@@ -1,20 +1,31 @@
1
1
  /* eslint-disable import/no-extraneous-dependencies */
2
- import fs from "fs";
3
- import { gzipSync } from "zlib";
2
+ import fs from 'fs';
3
+ import { argv } from 'process';
4
+ import { fileURLToPath } from 'url';
5
+ import { gzipSync } from 'zlib';
4
6
 
5
- import * as dynamoose from "dynamoose";
6
- import stableStringify from "fast-json-stable-stringify";
7
- import isbot from "isbot";
8
- import random from "random";
9
- import UAParser from "ua-parser-js";
7
+ import dynamoose from 'dynamoose';
8
+ import { Item } from 'dynamoose/dist/Item.js';
9
+ import stableStringify from 'fast-json-stable-stringify';
10
+ import isbot from 'isbot';
11
+ import random from 'random';
12
+ import UAParser from 'ua-parser-js';
13
+
14
+ import { UserAgentData } from './user-agent';
10
15
 
11
16
  const ddb = new dynamoose.aws.ddb.DynamoDB({
12
- region: "us-east-2",
17
+ region: 'us-east-2',
13
18
  });
14
19
  dynamoose.aws.ddb.set(ddb);
15
20
 
16
- const SubmissionModel = dynamoose.model(
17
- "userAgentsAnalyticsSubmissionTable",
21
+ const SubmissionModel = dynamoose.model<
22
+ {
23
+ ip: string;
24
+ profile: { [key: string]: unknown };
25
+ } & UserAgentData &
26
+ Item
27
+ >(
28
+ 'userAgentsAnalyticsSubmissionTable',
18
29
  new dynamoose.Schema(
19
30
  {
20
31
  id: {
@@ -25,8 +36,8 @@ const SubmissionModel = dynamoose.model(
25
36
  profile: Object,
26
37
  },
27
38
  {
28
- saveUnknown: ["profile.**"],
29
- timestamps: { createdAt: "timestamp", updatedAt: undefined },
39
+ saveUnknown: ['profile.**'],
40
+ timestamps: { createdAt: 'timestamp', updatedAt: undefined },
30
41
  },
31
42
  ),
32
43
  { create: false, update: false },
@@ -37,13 +48,11 @@ const getUserAgentTable = async (limit = 1e4) => {
37
48
 
38
49
  // Scan through all recent profiles keeping track of the count of each.
39
50
  let lastKey = null;
40
- const countsByProfile = {};
41
- let totalCount = 0;
42
- let uniqueCount = 0;
43
- let ipAddressAlreadySeen = {};
51
+ const countsByProfile: { [stringifiedProfile: string]: number } = {};
52
+ const ipAddressAlreadySeen: { [ipAddress: string]: boolean } = {};
44
53
  do {
45
54
  const scan = SubmissionModel.scan(
46
- new dynamoose.Condition().filter("timestamp").gt(minimumTimestamp),
55
+ new dynamoose.Condition().filter('timestamp').gt(minimumTimestamp),
47
56
  );
48
57
  if (lastKey) {
49
58
  scan.startAt(lastKey);
@@ -62,10 +71,8 @@ const getUserAgentTable = async (limit = 1e4) => {
62
71
  const stringifiedProfile = stableStringify(profile);
63
72
  if (!countsByProfile[stringifiedProfile]) {
64
73
  countsByProfile[stringifiedProfile] = 0;
65
- uniqueCount += 1;
66
74
  }
67
75
  countsByProfile[stringifiedProfile] += 1;
68
- totalCount += 1;
69
76
  });
70
77
 
71
78
  lastKey = response.lastKey;
@@ -76,17 +83,17 @@ const getUserAgentTable = async (limit = 1e4) => {
76
83
  Object.entries(countsByProfile).forEach(([stringifiedProfile, count]) => {
77
84
  const unnormalizedWeight =
78
85
  Array(2 * count)
79
- .fill()
86
+ .fill(undefined)
80
87
  .reduce((sum) => sum + n()() ** 2, 0) / 2;
81
88
  countsByProfile[stringifiedProfile] = unnormalizedWeight;
82
89
  });
83
90
 
84
91
  // Accumulate the profiles and add/remove a few properties to match the historical format.
85
- const profiles = [];
86
- for (let stringifiedProfile in countsByProfile) {
92
+ const profiles: UserAgentData[] = [];
93
+ Object.entries(countsByProfile).forEach(([stringifiedProfile, weight]) => {
87
94
  if (countsByProfile.hasOwnProperty(stringifiedProfile)) {
88
95
  const profile = JSON.parse(stringifiedProfile);
89
- profile.weight = countsByProfile[stringifiedProfile];
96
+ profile.weight = weight;
90
97
  delete profile.sessionId;
91
98
 
92
99
  // Deleting these because they weren't in the old format, but we should leave them in...
@@ -98,24 +105,19 @@ const getUserAgentTable = async (limit = 1e4) => {
98
105
  const device = parser.getDevice();
99
106
  // Sketchy, but I validated this on historical data and it is a 100% match.
100
107
  profile.deviceCategory =
101
- { mobile: "mobile", tablet: "tablet", undefined: "desktop" }[
102
- `${device.type}`
103
- ] ?? "desktop";
108
+ { mobile: 'mobile', tablet: 'tablet', undefined: 'desktop' }[`${device.type}`] ?? 'desktop';
104
109
 
105
110
  profiles.push(profile);
106
111
  delete countsByProfile[stringifiedProfile];
107
112
  }
108
- }
113
+ });
109
114
 
110
115
  // Sort by descending weight.
111
116
  profiles.sort((a, b) => b.weight - a.weight);
112
117
 
113
118
  // Apply the count limit and normalize the weights.
114
119
  profiles.splice(limit);
115
- const totalWeight = profiles.reduce(
116
- (total, profile) => total + profile.weight,
117
- 0,
118
- );
120
+ const totalWeight = profiles.reduce((total, profile) => total + profile.weight, 0);
119
121
  profiles.forEach((profile) => {
120
122
  profile.weight /= totalWeight;
121
123
  });
@@ -123,18 +125,16 @@ const getUserAgentTable = async (limit = 1e4) => {
123
125
  return profiles;
124
126
  };
125
127
 
126
- if (!module.parent) {
128
+ if (fileURLToPath(import.meta.url) === argv[1]) {
127
129
  const filename = process.argv[2];
128
130
  if (!filename) {
129
- throw new Error(
130
- "An output filename must be passed as an argument to the command.",
131
- );
131
+ throw new Error('An output filename must be passed as an argument to the command.');
132
132
  }
133
133
  getUserAgentTable()
134
134
  .then(async (userAgents) => {
135
135
  const stringifiedUserAgents = JSON.stringify(userAgents, null, 2);
136
136
  // Compress the content if the extension ends with `.gz`.
137
- const content = filename.endsWith(".gz")
137
+ const content = filename.endsWith('.gz')
138
138
  ? gzipSync(stringifiedUserAgents)
139
139
  : stringifiedUserAgents;
140
140
  fs.writeFileSync(filename, content);
@@ -0,0 +1,215 @@
1
+ import cloneDeep from 'lodash.clonedeep';
2
+
3
+ import untypedUserAgents from './user-agents.json' assert { type: 'json' };
4
+
5
+ const userAgents: UserAgentData[] = untypedUserAgents as UserAgentData[];
6
+
7
+ type NestedValueOf<T> = T extends object ? T[keyof T] | NestedValueOf<T[keyof T]> : T;
8
+
9
+ export type Filter<T extends UserAgentData | NestedValueOf<UserAgentData> = UserAgentData> =
10
+ | ((parentObject: T) => boolean)
11
+ | RegExp
12
+ | Array<Filter<T>>
13
+ | { [key: string]: Filter<T> }
14
+ | string;
15
+
16
+ export interface UserAgentData {
17
+ appName: 'Netscape';
18
+ connection: {
19
+ downlink: number;
20
+ effectiveType: '3g' | '4g';
21
+ rtt: number;
22
+ downlinkMax?: number | null;
23
+ type?: 'cellular' | 'wifi';
24
+ };
25
+ platform:
26
+ | 'iPad'
27
+ | 'iPhone'
28
+ | 'Linux aarch64'
29
+ | 'Linux armv81'
30
+ | 'Linux armv8l'
31
+ | 'Linux x86_64'
32
+ | 'MacIntel'
33
+ | 'Win32';
34
+ pluginsLength: number;
35
+ screenHeight: number;
36
+ screenWidth: number;
37
+ userAgent: string;
38
+ vendor: 'Apple Computer, Inc.' | 'Google Inc.' | '';
39
+ weight: number;
40
+ }
41
+
42
+ declare module './user-agent' {
43
+ export interface UserAgent extends Readonly<UserAgentData> {
44
+ readonly cumulativeWeightIndexPairs: Array<[number, number]>;
45
+ readonly data: UserAgentData;
46
+ (): UserAgent;
47
+ }
48
+ }
49
+
50
+ // Normalizes the total weight to 1 and constructs a cumulative distribution.
51
+ const makeCumulativeWeightIndexPairs = (
52
+ weightIndexPairs: Array<[number, number]>,
53
+ ): Array<[number, number]> => {
54
+ const totalWeight = weightIndexPairs.reduce((sum, [weight]) => sum + weight, 0);
55
+ let sum = 0;
56
+ return weightIndexPairs.map(([weight, index]) => {
57
+ sum += weight / totalWeight;
58
+ return [sum, index];
59
+ });
60
+ };
61
+
62
+ // Precompute these so that we can quickly generate unfiltered user agents.
63
+ const defaultWeightIndexPairs: Array<[number, number]> = userAgents.map(({ weight }, index) => [
64
+ weight,
65
+ index,
66
+ ]);
67
+ const defaultCumulativeWeightIndexPairs = makeCumulativeWeightIndexPairs(defaultWeightIndexPairs);
68
+
69
+ // Turn the various filter formats into a single filter function that acts on raw user agents.
70
+ const constructFilter = <T extends UserAgentData | NestedValueOf<UserAgentData>>(
71
+ filters: Filter<T>,
72
+ accessor: (parentObject: T) => T | NestedValueOf<T> = (parentObject: T): T => parentObject,
73
+ ): ((profile: T) => boolean) => {
74
+ // WARNING: This type and a lot of the types in here are wrong, but I can't get TypeScript to
75
+ // resolve things correctly so this will have to do for now.
76
+ let childFilters: Array<(parentObject: T) => boolean>;
77
+ if (typeof filters === 'function') {
78
+ childFilters = [filters];
79
+ } else if (filters instanceof RegExp) {
80
+ childFilters = [
81
+ (value: T | NestedValueOf<T>) =>
82
+ typeof value === 'object' && value && 'userAgent' in value && value.userAgent
83
+ ? filters.test(value.userAgent)
84
+ : filters.test(value as string),
85
+ ];
86
+ } else if (filters instanceof Array) {
87
+ childFilters = filters.map((childFilter) => constructFilter(childFilter));
88
+ } else if (typeof filters === 'object') {
89
+ childFilters = Object.entries(filters).map(([key, valueFilter]) =>
90
+ constructFilter(
91
+ valueFilter as Filter<T>,
92
+ (parentObject: T): T | NestedValueOf<T> =>
93
+ (parentObject as unknown as { [key: string]: NestedValueOf<T> })[key] as NestedValueOf<T>,
94
+ ),
95
+ );
96
+ } else {
97
+ childFilters = [
98
+ (value: T | NestedValueOf<T>) =>
99
+ typeof value === 'object' && value && 'userAgent' in value && value.userAgent
100
+ ? filters === value.userAgent
101
+ : filters === value,
102
+ ];
103
+ }
104
+
105
+ return (parentObject: T) => {
106
+ try {
107
+ const value = accessor(parentObject);
108
+ return childFilters.every((childFilter) => childFilter(value as T));
109
+ } catch (error) {
110
+ // This happens when a user-agent lacks a nested property.
111
+ return false;
112
+ }
113
+ };
114
+ };
115
+
116
+ // Construct normalized cumulative weight index pairs given the filters.
117
+ const constructCumulativeWeightIndexPairsFromFilters = (
118
+ filters?: Filter<UserAgentData>,
119
+ ): Array<[number, number]> => {
120
+ if (!filters) {
121
+ return defaultCumulativeWeightIndexPairs;
122
+ }
123
+
124
+ const filter = constructFilter(filters);
125
+
126
+ const weightIndexPairs: Array<[number, number]> = [];
127
+ userAgents.forEach((rawUserAgent, index) => {
128
+ if (filter(rawUserAgent)) {
129
+ weightIndexPairs.push([rawUserAgent.weight, index]);
130
+ }
131
+ });
132
+ return makeCumulativeWeightIndexPairs(weightIndexPairs);
133
+ };
134
+
135
+ const setCumulativeWeightIndexPairs = (
136
+ userAgent: UserAgent,
137
+ cumulativeWeightIndexPairs: Array<[number, number]>,
138
+ ) => {
139
+ Object.defineProperty(userAgent, 'cumulativeWeightIndexPairs', {
140
+ configurable: true,
141
+ enumerable: false,
142
+ writable: false,
143
+ value: cumulativeWeightIndexPairs,
144
+ });
145
+ };
146
+
147
+ export class UserAgent extends Function {
148
+ constructor(filters?: Filter) {
149
+ super();
150
+ setCumulativeWeightIndexPairs(this, constructCumulativeWeightIndexPairsFromFilters(filters));
151
+ if (this.cumulativeWeightIndexPairs.length === 0) {
152
+ throw new Error('No user agents matched your filters.');
153
+ }
154
+
155
+ this.randomize();
156
+
157
+ // eslint-disable-next-line no-constructor-return
158
+ return new Proxy(this, {
159
+ apply: () => this.random(),
160
+ get: (target, property, receiver) => {
161
+ const dataCandidate =
162
+ target.data &&
163
+ typeof property === 'string' &&
164
+ Object.prototype.hasOwnProperty.call(target.data, property) &&
165
+ Object.prototype.propertyIsEnumerable.call(target.data, property);
166
+ if (dataCandidate) {
167
+ const value = target.data[property as keyof UserAgentData];
168
+ if (value !== undefined) {
169
+ return value;
170
+ }
171
+ }
172
+
173
+ return Reflect.get(target, property, receiver);
174
+ },
175
+ });
176
+ }
177
+
178
+ static random = (filters: Filter) => {
179
+ try {
180
+ return new UserAgent(filters);
181
+ } catch (error) {
182
+ return null;
183
+ }
184
+ };
185
+
186
+ //
187
+ // Standard Object Methods
188
+ //
189
+
190
+ [Symbol.toPrimitive] = (): string => this.data.userAgent;
191
+
192
+ toString = (): string => this.data.userAgent;
193
+
194
+ random = (): UserAgent => {
195
+ const userAgent = new UserAgent();
196
+ setCumulativeWeightIndexPairs(userAgent, this.cumulativeWeightIndexPairs);
197
+ userAgent.randomize();
198
+ return userAgent;
199
+ };
200
+
201
+ randomize = (): void => {
202
+ // Find a random raw random user agent.
203
+ const randomNumber = Math.random();
204
+ const [, index] =
205
+ this.cumulativeWeightIndexPairs.find(
206
+ ([cumulativeWeight]) => cumulativeWeight > randomNumber,
207
+ ) ?? [];
208
+ if (index == null) {
209
+ throw new Error('Error finding a random user agent.');
210
+ }
211
+ const rawUserAgent = userAgents[index];
212
+
213
+ (this as { data: UserAgentData }).data = cloneDeep(rawUserAgent);
214
+ };
215
+ }
package/.babelrc DELETED
@@ -1,31 +0,0 @@
1
- {
2
- "env": {
3
- "testing": {
4
- "presets": [
5
- ["@babel/preset-env", {
6
- "targets": {
7
- "node": "current"
8
- }
9
- }],
10
- "power-assert"
11
- ]
12
- }
13
- },
14
- "presets": [["@babel/preset-env", {
15
- "modules": "commonjs",
16
- "targets": {
17
- "browsers": [
18
- "last 2 chrome versions",
19
- "last 2 firefox versions",
20
- ],
21
- "node": "6.10"
22
- }
23
- }]],
24
- "plugins": [
25
- "@babel/plugin-proposal-class-properties",
26
- "@babel/plugin-proposal-object-rest-spread",
27
- "@babel/plugin-transform-classes",
28
- "babel-plugin-add-module-exports"
29
- ]
30
- }
31
-