user-agents 1.1.13 → 2.0.0-alpha.10

package/package.json

@@ -1,47 +1,71 @@
 {
   "name": "user-agents",
-  "version": "1.1.13",
+  "version": "2.0.0-alpha.10",
   "description": "A JavaScript library for generating random user agents. ",
-  "main": "dist/index.js",
-  "repository": "git@github.com:intoli/user-agents.git",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/intoli/user-agents.git"
+  },
   "author": "Intoli, LLC <contact@intoli.com>",
+  "files": [
+    "dist/",
+    "src/**/*",
+    "!src/user-agents.json",
+    "!src/user-agents.json.gz"
+  ],
+  "exports": {
+    "import": "./dist/index.js",
+    "require": "./dist/index.cjs"
+  },
   "license": "BSD-2-Clause",
   "private": false,
+  "type": "module",
   "scripts": {
-    "build": "NODE_ENV=production webpack",
-    "gunzip-data": "babel-node src/gunzip-data.js src/user-agents.json.gz",
-    "lint": "eslint src/",
-    "test": "NODE_ENV=testing mocha --exit --require @babel/register",
-    "update-data": "babel-node src/update-data.js src/user-agents.json.gz"
+    "build": "tsup",
+    "postbuild": "yarn gunzip-data && cp src/user-agents.json dist/",
+    "gunzip-data": "node --loader ts-node/esm src/gunzip-data.ts src/user-agents.json.gz",
+    "lint": "eslint src/ && prettier --check src/",
+    "test": "NODE_ENV=testing mocha --exit --require @babel/register --extensions '.ts, .js'",
+    "update-data": "node --loader ts-node/esm src/update-data.ts src/user-agents.json.gz"
   },
   "devDependencies": {
-    "@babel/cli": "^7.12.16",
-    "@babel/core": "^7.12.16",
-    "@babel/eslint-parser": "^7.12.16",
-    "@babel/node": "^7.12.16",
-    "@babel/plugin-proposal-class-properties": "^7.12.13",
-    "@babel/plugin-proposal-object-rest-spread": "^7.12.13",
-    "@babel/plugin-transform-classes": "^7.12.13",
-    "@babel/preset-env": "^7.12.16",
-    "@babel/register": "^7.12.13",
-    "babel-loader": "^8.2.2",
+    "@babel/cli": "^7.23.0",
+    "@babel/core": "^7.23.2",
+    "@babel/plugin-proposal-class-properties": "^7.18.6",
+    "@babel/plugin-proposal-object-rest-spread": "^7.20.7",
+    "@babel/plugin-syntax-import-assertions": "^7.22.5",
+    "@babel/plugin-transform-classes": "^7.22.15",
+    "@babel/preset-env": "^7.23.2",
+    "@babel/preset-typescript": "^7.23.2",
+    "@babel/register": "^7.22.15",
+    "@types/lodash.clonedeep": "^4.5.8",
+    "@types/ua-parser-js": "^0.7.38",
+    "@typescript-eslint/eslint-plugin": "^6.9.0",
+    "@typescript-eslint/parser": "^6.9.0",
+    "babel-loader": "^9.1.3",
     "babel-plugin-add-module-exports": "^1.0.4",
     "babel-preset-power-assert": "^3.0.0",
     "dynamoose": "^3.2.1",
-    "eslint": "^7.19.0",
-    "eslint-config-airbnb": "^16.1.0",
-    "eslint-loader": "^4.0.2",
-    "eslint-plugin-import": "^2.22.1",
+    "esbuild": "^0.19.5",
+    "eslint": "^8.52.0",
+    "eslint-config-airbnb": "^19.0.4",
+    "eslint-config-prettier": "^9.0.0",
+    "eslint-plugin-import": "^2.29.0",
     "fast-json-stable-stringify": "^2.1.0",
-    "imports-loader": "^2.0.0",
+    "imports-loader": "^4.0.1",
     "isbot": "^3.7.0",
-    "mocha": "^8.3.0",
+    "mocha": "^10.2.0",
     "power-assert": "^1.6.1",
-    "random": "^2.2.0",
-    "source-map-support": "^0.5.19",
-    "ua-parser-js": "^1.0.36",
-    "webpack": "^5.21.2",
-    "webpack-cli": "^4.5.0"
+    "prettier": "^3.0.3",
+    "random": "^4.1.0",
+    "source-map-support": "^0.5.21",
+    "ts-node": "^10.9.1",
+    "tsup": "^7.2.0",
+    "typescript": "^5.2.2",
+    "typescript-language-server": "^4.0.0",
+    "ua-parser-js": "^1.0.36"
   },
   "dependencies": {
     "lodash.clonedeep": "^4.5.0"

package/src/{gunzip-data.js → gunzip-data.ts}

@@ -1,8 +1,9 @@
 import fs from 'fs';
+import { argv } from 'process';
+import { fileURLToPath } from 'url';
 import { gunzipSync } from 'zlib';
 
-
-const gunzipData = (inputFilename) => {
+const gunzipData = (inputFilename?: string) => {
   if (!inputFilename || !inputFilename.endsWith('.gz')) {
     throw new Error('Filename must be specified and end with `.gz` for gunzipping.');
   }
@@ -12,11 +13,9 @@ const gunzipData = (inputFilename) => {
   fs.writeFileSync(outputFilename, data);
 };
 
-
-if (!module.parent) {
+if (fileURLToPath(import.meta.url) === argv[1]) {
   const inputFilename = process.argv[2];
   gunzipData(inputFilename);
 }
 
-
 export default gunzipData;

package/src/index.ts

@@ -0,0 +1,4 @@
+import { Filter, UserAgent, UserAgentData } from './user-agent';
+
+export default UserAgent;
+export type { Filter, UserAgentData };

package/src/{update-data.js → update-data.ts}

@@ -1,20 +1,31 @@
 /* eslint-disable import/no-extraneous-dependencies */
-import fs from "fs";
-import { gzipSync } from "zlib";
+import fs from 'fs';
+import { argv } from 'process';
+import { fileURLToPath } from 'url';
+import { gzipSync } from 'zlib';
 
-import * as dynamoose from "dynamoose";
-import stableStringify from "fast-json-stable-stringify";
-import isbot from "isbot";
-import random from "random";
-import UAParser from "ua-parser-js";
+import dynamoose from 'dynamoose';
+import { Item } from 'dynamoose/dist/Item.js';
+import stableStringify from 'fast-json-stable-stringify';
+import isbot from 'isbot';
+import random from 'random';
+import UAParser from 'ua-parser-js';
+
+import { UserAgentData } from './user-agent';
 
 const ddb = new dynamoose.aws.ddb.DynamoDB({
-  region: "us-east-2",
+  region: 'us-east-2',
 });
 dynamoose.aws.ddb.set(ddb);
 
-const SubmissionModel = dynamoose.model(
-  "userAgentsAnalyticsSubmissionTable",
+const SubmissionModel = dynamoose.model<
+  {
+    ip: string;
+    profile: { [key: string]: unknown };
+  } & UserAgentData &
+    Item
+>(
+  'userAgentsAnalyticsSubmissionTable',
   new dynamoose.Schema(
     {
       id: {
@@ -25,8 +36,8 @@ const SubmissionModel = dynamoose.model(
       profile: Object,
     },
     {
-      saveUnknown: ["profile.**"],
-      timestamps: { createdAt: "timestamp", updatedAt: undefined },
+      saveUnknown: ['profile.**'],
+      timestamps: { createdAt: 'timestamp', updatedAt: undefined },
     },
   ),
   { create: false, update: false },
@@ -37,13 +48,11 @@ const getUserAgentTable = async (limit = 1e4) => {
 
   // Scan through all recent profiles keeping track of the count of each.
   let lastKey = null;
-  const countsByProfile = {};
-  let totalCount = 0;
-  let uniqueCount = 0;
-  let ipAddressAlreadySeen = {};
+  const countsByProfile: { [stringifiedProfile: string]: number } = {};
+  const ipAddressAlreadySeen: { [ipAddress: string]: boolean } = {};
   do {
     const scan = SubmissionModel.scan(
-      new dynamoose.Condition().filter("timestamp").gt(minimumTimestamp),
+      new dynamoose.Condition().filter('timestamp').gt(minimumTimestamp),
     );
     if (lastKey) {
       scan.startAt(lastKey);
@@ -62,10 +71,8 @@ const getUserAgentTable = async (limit = 1e4) => {
       const stringifiedProfile = stableStringify(profile);
       if (!countsByProfile[stringifiedProfile]) {
         countsByProfile[stringifiedProfile] = 0;
-        uniqueCount += 1;
       }
       countsByProfile[stringifiedProfile] += 1;
-      totalCount += 1;
     });
 
     lastKey = response.lastKey;
@@ -76,46 +83,37 @@ const getUserAgentTable = async (limit = 1e4) => {
   Object.entries(countsByProfile).forEach(([stringifiedProfile, count]) => {
     const unnormalizedWeight =
       Array(2 * count)
-        .fill()
+        .fill(undefined)
         .reduce((sum) => sum + n()() ** 2, 0) / 2;
     countsByProfile[stringifiedProfile] = unnormalizedWeight;
   });
 
   // Accumulate the profiles and add/remove a few properties to match the historical format.
-  const profiles = [];
-  for (let stringifiedProfile in countsByProfile) {
+  const profiles: UserAgentData[] = [];
+  Object.entries(countsByProfile).forEach(([stringifiedProfile, weight]) => {
     if (countsByProfile.hasOwnProperty(stringifiedProfile)) {
       const profile = JSON.parse(stringifiedProfile);
-      profile.weight = countsByProfile[stringifiedProfile];
+      profile.weight = weight;
       delete profile.sessionId;
 
-      // Deleting these because they weren't in the old format, but we should leave them in...
-      delete profile.language;
-      delete profile.oscpu;
-
       // Find the device category.
       const parser = new UAParser(profile.userAgent);
       const device = parser.getDevice();
       // Sketchy, but I validated this on historical data and it is a 100% match.
       profile.deviceCategory =
-        { mobile: "mobile", tablet: "tablet", undefined: "desktop" }[
-          `${device.type}`
-        ] ?? "desktop";
+        { mobile: 'mobile', tablet: 'tablet', undefined: 'desktop' }[`${device.type}`] ?? 'desktop';
 
       profiles.push(profile);
       delete countsByProfile[stringifiedProfile];
     }
-  }
+  });
 
   // Sort by descending weight.
   profiles.sort((a, b) => b.weight - a.weight);
 
   // Apply the count limit and normalize the weights.
   profiles.splice(limit);
-  const totalWeight = profiles.reduce(
-    (total, profile) => total + profile.weight,
-    0,
-  );
+  const totalWeight = profiles.reduce((total, profile) => total + profile.weight, 0);
   profiles.forEach((profile) => {
     profile.weight /= totalWeight;
   });
@@ -123,18 +121,16 @@ const getUserAgentTable = async (limit = 1e4) => {
   return profiles;
 };
 
-if (!module.parent) {
+if (fileURLToPath(import.meta.url) === argv[1]) {
   const filename = process.argv[2];
   if (!filename) {
-    throw new Error(
-      "An output filename must be passed as an argument to the command.",
-    );
+    throw new Error('An output filename must be passed as an argument to the command.');
   }
   getUserAgentTable()
     .then(async (userAgents) => {
       const stringifiedUserAgents = JSON.stringify(userAgents, null, 2);
       // Compress the content if the extension ends with `.gz`.
-      const content = filename.endsWith(".gz")
+      const content = filename.endsWith('.gz')
         ? gzipSync(stringifiedUserAgents)
         : stringifiedUserAgents;
       fs.writeFileSync(filename, content);

package/src/user-agent.ts

@@ -0,0 +1,217 @@
+import cloneDeep from 'lodash.clonedeep';
+
+import untypedUserAgents from './user-agents.json' assert { type: 'json' };
+
+const userAgents: UserAgentData[] = untypedUserAgents as UserAgentData[];
+
+type NestedValueOf<T> = T extends object ? T[keyof T] | NestedValueOf<T[keyof T]> : T;
+
+export type Filter<T extends UserAgentData | NestedValueOf<UserAgentData> = UserAgentData> =
+  | ((parentObject: T) => boolean)
+  | RegExp
+  | Array<Filter<T>>
+  | { [key: string]: Filter<T> }
+  | string;
+
+export interface UserAgentData {
+  appName: 'Netscape';
+  connection: {
+    downlink: number;
+    effectiveType: '3g' | '4g';
+    rtt: number;
+    downlinkMax?: number | null;
+    type?: 'cellular' | 'wifi';
+  };
+  language?: string | null;
+  oscpu?: string | null;
+  platform:
+    | 'iPad'
+    | 'iPhone'
+    | 'Linux aarch64'
+    | 'Linux armv81'
+    | 'Linux armv8l'
+    | 'Linux x86_64'
+    | 'MacIntel'
+    | 'Win32';
+  pluginsLength: number;
+  screenHeight: number;
+  screenWidth: number;
+  userAgent: string;
+  vendor: 'Apple Computer, Inc.' | 'Google Inc.' | '';
+  weight: number;
+}
+
+declare module './user-agent' {
+  export interface UserAgent extends Readonly<UserAgentData> {
+    readonly cumulativeWeightIndexPairs: Array<[number, number]>;
+    readonly data: UserAgentData;
+    (): UserAgent;
+  }
+}
+
+// Normalizes the total weight to 1 and constructs a cumulative distribution.
+const makeCumulativeWeightIndexPairs = (
+  weightIndexPairs: Array<[number, number]>,
+): Array<[number, number]> => {
+  const totalWeight = weightIndexPairs.reduce((sum, [weight]) => sum + weight, 0);
+  let sum = 0;
+  return weightIndexPairs.map(([weight, index]) => {
+    sum += weight / totalWeight;
+    return [sum, index];
+  });
+};
+
+// Precompute these so that we can quickly generate unfiltered user agents.
+const defaultWeightIndexPairs: Array<[number, number]> = userAgents.map(({ weight }, index) => [
+  weight,
+  index,
+]);
+const defaultCumulativeWeightIndexPairs = makeCumulativeWeightIndexPairs(defaultWeightIndexPairs);
+
+// Turn the various filter formats into a single filter function that acts on raw user agents.
+const constructFilter = <T extends UserAgentData | NestedValueOf<UserAgentData>>(
+  filters: Filter<T>,
+  accessor: (parentObject: T) => T | NestedValueOf<T> = (parentObject: T): T => parentObject,
+): ((profile: T) => boolean) => {
+  // WARNING: This type and a lot of the types in here are wrong, but I can't get TypeScript to
+  // resolve things correctly so this will have to do for now.
+  let childFilters: Array<(parentObject: T) => boolean>;
+  if (typeof filters === 'function') {
+    childFilters = [filters];
+  } else if (filters instanceof RegExp) {
+    childFilters = [
+      (value: T | NestedValueOf<T>) =>
+        typeof value === 'object' && value && 'userAgent' in value && value.userAgent
+          ? filters.test(value.userAgent)
+          : filters.test(value as string),
+    ];
+  } else if (filters instanceof Array) {
+    childFilters = filters.map((childFilter) => constructFilter(childFilter));
+  } else if (typeof filters === 'object') {
+    childFilters = Object.entries(filters).map(([key, valueFilter]) =>
+      constructFilter(
+        valueFilter as Filter<T>,
+        (parentObject: T): T | NestedValueOf<T> =>
+          (parentObject as unknown as { [key: string]: NestedValueOf<T> })[key] as NestedValueOf<T>,
+      ),
+    );
+  } else {
+    childFilters = [
+      (value: T | NestedValueOf<T>) =>
+        typeof value === 'object' && value && 'userAgent' in value && value.userAgent
+          ? filters === value.userAgent
+          : filters === value,
+    ];
+  }
+
+  return (parentObject: T) => {
+    try {
+      const value = accessor(parentObject);
+      return childFilters.every((childFilter) => childFilter(value as T));
+    } catch (error) {
+      // This happens when a user-agent lacks a nested property.
+      return false;
+    }
+  };
+};
+
+// Construct normalized cumulative weight index pairs given the filters.
+const constructCumulativeWeightIndexPairsFromFilters = (
+  filters?: Filter<UserAgentData>,
+): Array<[number, number]> => {
+  if (!filters) {
+    return defaultCumulativeWeightIndexPairs;
+  }
+
+  const filter = constructFilter(filters);
+
+  const weightIndexPairs: Array<[number, number]> = [];
+  userAgents.forEach((rawUserAgent, index) => {
+    if (filter(rawUserAgent)) {
+      weightIndexPairs.push([rawUserAgent.weight, index]);
+    }
+  });
+  return makeCumulativeWeightIndexPairs(weightIndexPairs);
+};
+
+const setCumulativeWeightIndexPairs = (
+  userAgent: UserAgent,
+  cumulativeWeightIndexPairs: Array<[number, number]>,
+) => {
+  Object.defineProperty(userAgent, 'cumulativeWeightIndexPairs', {
+    configurable: true,
+    enumerable: false,
+    writable: false,
+    value: cumulativeWeightIndexPairs,
+  });
+};
+
+export class UserAgent extends Function {
+  constructor(filters?: Filter) {
+    super();
+    setCumulativeWeightIndexPairs(this, constructCumulativeWeightIndexPairsFromFilters(filters));
+    if (this.cumulativeWeightIndexPairs.length === 0) {
+      throw new Error('No user agents matched your filters.');
+    }
+
+    this.randomize();
+
+    // eslint-disable-next-line no-constructor-return
+    return new Proxy(this, {
+      apply: () => this.random(),
+      get: (target, property, receiver) => {
+        const dataCandidate =
+          target.data &&
+          typeof property === 'string' &&
+          Object.prototype.hasOwnProperty.call(target.data, property) &&
+          Object.prototype.propertyIsEnumerable.call(target.data, property);
+        if (dataCandidate) {
+          const value = target.data[property as keyof UserAgentData];
+          if (value !== undefined) {
+            return value;
+          }
+        }
+
+        return Reflect.get(target, property, receiver);
+      },
+    });
+  }
+
+  static random = (filters: Filter) => {
+    try {
+      return new UserAgent(filters);
+    } catch (error) {
+      return null;
+    }
+  };
+
+  //
+  // Standard Object Methods
+  //
+
+  [Symbol.toPrimitive] = (): string => this.data.userAgent;
+
+  toString = (): string => this.data.userAgent;
+
+  random = (): UserAgent => {
+    const userAgent = new UserAgent();
+    setCumulativeWeightIndexPairs(userAgent, this.cumulativeWeightIndexPairs);
+    userAgent.randomize();
+    return userAgent;
+  };
+
+  randomize = (): void => {
+    // Find a random raw random user agent.
+    const randomNumber = Math.random();
+    const [, index] =
+      this.cumulativeWeightIndexPairs.find(
+        ([cumulativeWeight]) => cumulativeWeight > randomNumber,
+      ) ?? [];
+    if (index == null) {
+      throw new Error('Error finding a random user agent.');
+    }
+    const rawUserAgent = userAgents[index];
+
+    (this as { data: UserAgentData }).data = cloneDeep(rawUserAgent);
+  };
+}

package/.babelrc

@@ -1,31 +0,0 @@
-{
-  "env": {
-    "testing": {
-      "presets": [
-        ["@babel/preset-env", {
-          "targets": {
-            "node": "current"
-          }
-        }],
-        "power-assert"
-      ]
-    }
-  },
-  "presets": [["@babel/preset-env", {
-    "modules": "commonjs",
-    "targets": {
-      "browsers": [
-        "last 2 chrome versions",
-        "last 2 firefox versions",
-      ],
-      "node": "6.10"
-    }
-  }]],
-  "plugins": [
-      "@babel/plugin-proposal-class-properties",
-      "@babel/plugin-proposal-object-rest-spread",
-      "@babel/plugin-transform-classes",
-      "babel-plugin-add-module-exports"
-  ]
-}
-