webpack 5.79.0 → 5.80.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of webpack might be problematic. Click here for more details.

package/README.md CHANGED
@@ -35,6 +35,9 @@
35
35
  <a href="https://gitter.im/webpack/webpack">
36
36
  <img src="https://badges.gitter.im/webpack/webpack.svg">
37
37
  </a>
38
+ <a href="https://twitter.com/Webpack">
39
+ <img src="https://img.shields.io/twitter/follow/Webpack?style=social">
40
+ </a>
38
41
  <h1>webpack</h1>
39
42
  <p>
40
43
  Webpack is a module bundler. Its main purpose is to bundle JavaScript files for usage in a browser, yet it is also capable of transforming, bundling, or packaging just about any resource or asset.
@@ -1020,7 +1020,7 @@ If changing the source code is not an option there is also a resolve options cal
1020
1020
  context,
1021
1021
  item.loader,
1022
1022
  resolveContext,
1023
- (err, result) => {
1023
+ (err, result, resolveRequest) => {
1024
1024
  if (
1025
1025
  err &&
1026
1026
  /^[^/]*$/.test(item.loader) &&
@@ -1047,8 +1047,18 @@ If changing the source code is not an option there is also a resolve options cal
1047
1047
  if (err) return callback(err);
1048
1048
 
1049
1049
  const parsedResult = this._parseResourceWithoutFragment(result);
1050
+
1051
+ const type = /\.mjs$/i.test(parsedResult.path)
1052
+ ? "module"
1053
+ : /\.cjs$/i.test(parsedResult.path)
1054
+ ? "commonjs"
1055
+ : resolveRequest.descriptionFileData === undefined
1056
+ ? undefined
1057
+ : resolveRequest.descriptionFileData.type;
1058
+
1050
1059
  const resolved = {
1051
1060
  loader: parsedResult.path,
1061
+ type,
1052
1062
  options:
1053
1063
  item.options === undefined
1054
1064
  ? parsedResult.query
@@ -9,6 +9,7 @@ const CaseSensitiveModulesWarning = require("./CaseSensitiveModulesWarning");
9
9
 
10
10
  /** @typedef {import("./Compiler")} Compiler */
11
11
  /** @typedef {import("./Module")} Module */
12
+ /** @typedef {import("./NormalModule")} NormalModule */
12
13
 
13
14
  class WarnCaseSensitiveModulesPlugin {
14
15
  /**
@@ -25,6 +26,17 @@ class WarnCaseSensitiveModulesPlugin {
25
26
  const moduleWithoutCase = new Map();
26
27
  for (const module of compilation.modules) {
27
28
  const identifier = module.identifier();
29
+
30
+ // Ignore `data:` URLs, because it's not a real path
31
+ if (
32
+ /** @type {NormalModule} */
33
+ (module).resourceResolveData !== undefined &&
34
+ /** @type {NormalModule} */
35
+ (module).resourceResolveData.encodedContent !== undefined
36
+ ) {
37
+ continue;
38
+ }
39
+
28
40
  const lowerIdentifier = identifier.toLowerCase();
29
41
  let map = moduleWithoutCase.get(lowerIdentifier);
30
42
  if (map === undefined) {
@@ -108,9 +108,17 @@ const encodeDataUri = (encoding, source) => {
108
108
 
109
109
  const decodeDataUriContent = (encoding, content) => {
110
110
  const isBase64 = encoding === "base64";
111
- return isBase64
112
- ? Buffer.from(content, "base64")
113
- : Buffer.from(decodeURIComponent(content), "ascii");
111
+
112
+ if (isBase64) {
113
+ return Buffer.from(content, "base64");
114
+ }
115
+
116
+ // If we can't decode return the original body
117
+ try {
118
+ return Buffer.from(decodeURIComponent(content), "ascii");
119
+ } catch (_) {
120
+ return Buffer.from(content, "ascii");
121
+ }
114
122
  };
115
123
 
116
124
  const JS_TYPES = new Set(["javascript"]);
@@ -17,21 +17,54 @@ const walkCssTokens = require("./walkCssTokens");
17
17
 
18
18
  /** @typedef {import("../Parser").ParserState} ParserState */
19
19
  /** @typedef {import("../Parser").PreparsedAst} PreparsedAst */
20
-
21
20
  const CC_LEFT_CURLY = "{".charCodeAt(0);
22
21
  const CC_RIGHT_CURLY = "}".charCodeAt(0);
23
22
  const CC_COLON = ":".charCodeAt(0);
24
23
  const CC_SLASH = "/".charCodeAt(0);
25
24
  const CC_SEMICOLON = ";".charCodeAt(0);
26
25
 
27
- const cssUnescape = str => {
28
- return str.replace(/\\([0-9a-fA-F]{1,6}[ \t\n\r\f]?|[\s\S])/g, match => {
29
- if (match.length > 2) {
30
- return String.fromCharCode(parseInt(match.slice(1).trim(), 16));
31
- } else {
32
- return match[1];
26
+ // https://www.w3.org/TR/css-syntax-3/#newline
27
+ // We don't have `preprocessing` stage, so we need specify all of them
28
+ const STRING_MULTILINE = /\\[\n\r\f]/g;
29
+ // https://www.w3.org/TR/css-syntax-3/#whitespace
30
+ const TRIM_WHITE_SPACES = /(^[ \t\n\r\f]*|[ \t\n\r\f]*$)/g;
31
+ const UNESCAPE = /\\([0-9a-fA-F]{1,6}[ \t\n\r\f]?|[\s\S])/g;
32
+ const IMAGE_SET_FUNCTION = /^(-\w+-)?image-set$/i;
33
+
34
+ const normalizeUrl = (str, isString) => {
35
+ // Remove extra spaces and newlines:
36
+ // `url("im\
37
+ // g.png")`
38
+ if (isString) {
39
+ str = str.replace(STRING_MULTILINE, "");
40
+ }
41
+
42
+ str = str
43
+ // Remove unnecessary spaces from `url(" img.png ")`
44
+ .replace(TRIM_WHITE_SPACES, "")
45
+ // Unescape
46
+ .replace(UNESCAPE, match => {
47
+ if (match.length > 2) {
48
+ return String.fromCharCode(parseInt(match.slice(1).trim(), 16));
49
+ } else {
50
+ return match[1];
51
+ }
52
+ });
53
+
54
+ if (/^data:/i.test(str)) {
55
+ return str;
56
+ }
57
+
58
+ if (str.includes("%")) {
59
+ // Convert `url('%2E/img.png')` -> `url('./img.png')`
60
+ try {
61
+ str = decodeURIComponent(str);
62
+ } catch (error) {
63
+ // Ignore
33
64
  }
34
- });
65
+ }
66
+
67
+ return str;
35
68
  };
36
69
 
37
70
  class LocConverter {
@@ -137,8 +170,11 @@ class CssParser extends Parser {
137
170
  let modeData = undefined;
138
171
  let singleClassSelector = undefined;
139
172
  let lastIdentifier = undefined;
140
- const modeStack = [];
141
173
  let awaitRightParenthesis = false;
174
+ /** @type [string, number, number][] */
175
+ const functionStack = [];
176
+ const modeStack = [];
177
+
142
178
  const isTopLevelLocal = () =>
143
179
  modeData === "local" ||
144
180
  (this.defaultMode === "local" && modeData === undefined);
@@ -304,8 +340,11 @@ class CssParser extends Parser {
304
340
  isSelector: () => {
305
341
  return mode !== CSS_MODE_IN_RULE && mode !== CSS_MODE_IN_LOCAL_RULE;
306
342
  },
307
- url: (input, start, end, contentStart, contentEnd) => {
308
- const value = cssUnescape(input.slice(contentStart, contentEnd));
343
+ url: (input, start, end, contentStart, contentEnd, isString) => {
344
+ let value = normalizeUrl(
345
+ input.slice(contentStart, contentEnd),
346
+ isString
347
+ );
309
348
  switch (mode) {
310
349
  case CSS_MODE_AT_IMPORT_EXPECT_URL: {
311
350
  modeData.url = value;
@@ -321,6 +360,15 @@ class CssParser extends Parser {
321
360
  )} at ${start} during ${explainMode(mode)}`
322
361
  );
323
362
  default: {
363
+ if (
364
+ // Ignore `url(#highlight)` URLs
365
+ /^#/.test(value) ||
366
+ // Ignore `url()`, `url('')` and `url("")`, they are valid by spec
367
+ value.length === 0
368
+ ) {
369
+ break;
370
+ }
371
+
324
372
  const dep = new CssUrlDependency(value, [start, end], "url");
325
373
  const { line: sl, column: sc } = locConverter.get(start);
326
374
  const { line: el, column: ec } = locConverter.get(end);
@@ -335,10 +383,44 @@ class CssParser extends Parser {
335
383
  string: (input, start, end) => {
336
384
  switch (mode) {
337
385
  case CSS_MODE_AT_IMPORT_EXPECT_URL: {
338
- modeData.url = cssUnescape(input.slice(start + 1, end - 1));
386
+ modeData.url = normalizeUrl(input.slice(start + 1, end - 1), true);
339
387
  mode = CSS_MODE_AT_IMPORT_EXPECT_SUPPORTS;
340
388
  break;
341
389
  }
390
+ default: {
391
+ // TODO move escaped parsing to tokenizer
392
+ const lastFunction = functionStack[functionStack.length - 1];
393
+
394
+ if (
395
+ lastFunction &&
396
+ (lastFunction[0].replace(/\\/g, "").toLowerCase() === "url" ||
397
+ IMAGE_SET_FUNCTION.test(lastFunction[0].replace(/\\/g, "")))
398
+ ) {
399
+ let value = normalizeUrl(input.slice(start + 1, end - 1), true);
400
+
401
+ if (
402
+ // Ignore `url(#highlight)` URLs
403
+ /^#/.test(value) ||
404
+ // Ignore `url()`, `url('')` and `url("")`, they are valid by spec
405
+ value.length === 0
406
+ ) {
407
+ break;
408
+ }
409
+
410
+ const isUrl =
411
+ lastFunction[0].replace(/\\/g, "").toLowerCase() === "url";
412
+ const dep = new CssUrlDependency(
413
+ value,
414
+ [start, end],
415
+ isUrl ? "string" : "url"
416
+ );
417
+ const { line: sl, column: sc } = locConverter.get(start);
418
+ const { line: el, column: ec } = locConverter.get(end);
419
+ dep.setLoc(sl, sc, el, ec);
420
+ module.addDependency(dep);
421
+ module.addCodeGenerationDependency(dep);
422
+ }
423
+ }
342
424
  }
343
425
  return end;
344
426
  },
@@ -523,6 +605,8 @@ class CssParser extends Parser {
523
605
  return end;
524
606
  },
525
607
  rightParenthesis: (input, start, end) => {
608
+ functionStack.pop();
609
+
526
610
  switch (mode) {
527
611
  case CSS_MODE_TOP_LEVEL: {
528
612
  if (awaitRightParenthesis) {
@@ -537,6 +621,7 @@ class CssParser extends Parser {
537
621
  break;
538
622
  }
539
623
  }
624
+
540
625
  return end;
541
626
  },
542
627
  pseudoClass: (input, start, end) => {
@@ -564,9 +649,14 @@ class CssParser extends Parser {
564
649
  return end;
565
650
  },
566
651
  pseudoFunction: (input, start, end) => {
652
+ let name = input.slice(start, end - 1);
653
+
654
+ functionStack.push([name, start, end]);
655
+
567
656
  switch (mode) {
568
657
  case CSS_MODE_TOP_LEVEL: {
569
- const name = input.slice(start, end - 1).toLowerCase();
658
+ name = name.toLowerCase();
659
+
570
660
  if (this.allowModeSwitch && name === ":global") {
571
661
  modeStack.push(modeData);
572
662
  modeData = "global";
@@ -587,9 +677,14 @@ class CssParser extends Parser {
587
677
  return end;
588
678
  },
589
679
  function: (input, start, end) => {
680
+ let name = input.slice(start, end - 1);
681
+
682
+ functionStack.push([name, start, end]);
683
+
590
684
  switch (mode) {
591
685
  case CSS_MODE_IN_LOCAL_RULE: {
592
- const name = input.slice(start, end - 1).toLowerCase();
686
+ name = name.toLowerCase();
687
+
593
688
  if (name === "var") {
594
689
  let pos = walkCssTokens.eatWhitespaceAndComments(input, end);
595
690
  if (pos === input.length) return pos;
@@ -36,14 +36,16 @@ const CC_FORM_FEED = "\f".charCodeAt(0);
36
36
  const CC_TAB = "\t".charCodeAt(0);
37
37
  const CC_SPACE = " ".charCodeAt(0);
38
38
 
39
- const CC_SLASH = "/".charCodeAt(0);
40
- const CC_BACK_SLASH = "\\".charCodeAt(0);
39
+ const CC_SOLIDUS = "/".charCodeAt(0);
40
+ const CC_REVERSE_SOLIDUS = "\\".charCodeAt(0);
41
41
  const CC_ASTERISK = "*".charCodeAt(0);
42
42
 
43
43
  const CC_LEFT_PARENTHESIS = "(".charCodeAt(0);
44
44
  const CC_RIGHT_PARENTHESIS = ")".charCodeAt(0);
45
45
  const CC_LEFT_CURLY = "{".charCodeAt(0);
46
46
  const CC_RIGHT_CURLY = "}".charCodeAt(0);
47
+ const CC_LEFT_SQUARE = "[".charCodeAt(0);
48
+ const CC_RIGHT_SQUARE = "]".charCodeAt(0);
47
49
 
48
50
  const CC_QUOTATION_MARK = '"'.charCodeAt(0);
49
51
  const CC_APOSTROPHE = "'".charCodeAt(0);
@@ -100,28 +102,46 @@ const _isWhiteSpace = cc => {
100
102
  );
101
103
  };
102
104
 
105
+ const _isIdentStartCodePoint = cc => {
106
+ return (
107
+ (cc >= CC_LOWER_A && cc <= CC_LOWER_Z) ||
108
+ (cc >= CC_UPPER_A && cc <= CC_UPPER_Z) ||
109
+ cc === CC_LOW_LINE ||
110
+ cc >= 0x80
111
+ );
112
+ };
113
+
103
114
  /** @type {CharHandler} */
104
- const consumeSingleCharToken = (input, pos, callbacks) => {
115
+ const consumeDelimToken = (input, pos, callbacks) => {
105
116
  return pos + 1;
106
117
  };
107
118
 
108
119
  /** @type {CharHandler} */
109
- const consumePotentialComment = (input, pos, callbacks) => {
110
- pos++;
111
- if (pos === input.length) return pos;
112
- let cc = input.charCodeAt(pos);
113
- if (cc !== CC_ASTERISK) return pos;
114
- for (;;) {
115
- pos++;
116
- if (pos === input.length) return pos;
117
- cc = input.charCodeAt(pos);
118
- while (cc === CC_ASTERISK) {
120
+ const consumeComments = (input, pos, callbacks) => {
121
+ // If the next two input code point are U+002F SOLIDUS (/) followed by a U+002A
122
+ // ASTERISK (*), consume them and all following code points up to and including
123
+ // the first U+002A ASTERISK (*) followed by a U+002F SOLIDUS (/), or up to an
124
+ // EOF code point. Return to the start of this step.
125
+ //
126
+ // If the preceding paragraph ended by consuming an EOF code point, this is a parse error.
127
+ // But we are silent on errors.
128
+ if (
129
+ input.charCodeAt(pos) === CC_SOLIDUS &&
130
+ input.charCodeAt(pos + 1) === CC_ASTERISK
131
+ ) {
132
+ pos += 1;
133
+ while (pos < input.length) {
134
+ if (
135
+ input.charCodeAt(pos) === CC_ASTERISK &&
136
+ input.charCodeAt(pos + 1) === CC_SOLIDUS
137
+ ) {
138
+ pos += 2;
139
+ break;
140
+ }
119
141
  pos++;
120
- if (pos === input.length) return pos;
121
- cc = input.charCodeAt(pos);
122
- if (cc === CC_SLASH) return pos + 1;
123
142
  }
124
143
  }
144
+ return pos;
125
145
  };
126
146
 
127
147
  /** @type {function(number): CharHandler} */
@@ -144,7 +164,7 @@ const _consumeString = (input, pos, end) => {
144
164
  // bad string
145
165
  return pos;
146
166
  }
147
- if (cc === CC_BACK_SLASH) {
167
+ if (cc === CC_REVERSE_SOLIDUS) {
148
168
  // we don't need to fully parse the escaped code point
149
169
  // just skip over a potential new line
150
170
  pos++;
@@ -165,6 +185,12 @@ const _isIdentifierStartCode = cc => {
165
185
  );
166
186
  };
167
187
 
188
+ const _isTwoCodePointsAreValidEscape = (first, second) => {
189
+ if (first !== CC_REVERSE_SOLIDUS) return false;
190
+ if (_isNewLine(second)) return false;
191
+ return true;
192
+ };
193
+
168
194
  const _isDigit = cc => {
169
195
  return cc >= CC_0 && cc <= CC_9;
170
196
  };
@@ -175,13 +201,13 @@ const _startsIdentifier = (input, pos) => {
175
201
  if (pos === input.length) return false;
176
202
  const cc = input.charCodeAt(pos + 1);
177
203
  if (cc === CC_HYPHEN_MINUS) return true;
178
- if (cc === CC_BACK_SLASH) {
204
+ if (cc === CC_REVERSE_SOLIDUS) {
179
205
  const cc = input.charCodeAt(pos + 2);
180
206
  return !_isNewLine(cc);
181
207
  }
182
208
  return _isIdentifierStartCode(cc);
183
209
  }
184
- if (cc === CC_BACK_SLASH) {
210
+ if (cc === CC_REVERSE_SOLIDUS) {
185
211
  const cc = input.charCodeAt(pos + 1);
186
212
  return !_isNewLine(cc);
187
213
  }
@@ -208,6 +234,7 @@ const consumeMinus = (input, pos, callbacks) => {
208
234
  pos++;
209
235
  if (pos === input.length) return pos;
210
236
  const cc = input.charCodeAt(pos);
237
+ // If the input stream starts with a number, reconsume the current input code point, consume a numeric token, and return it.
211
238
  if (cc === CC_FULL_STOP || _isDigit(cc)) {
212
239
  return consumeNumericToken(input, pos, callbacks);
213
240
  } else if (cc === CC_HYPHEN_MINUS) {
@@ -222,7 +249,7 @@ const consumeMinus = (input, pos, callbacks) => {
222
249
  return callbacks.identifier(input, start, pos);
223
250
  }
224
251
  }
225
- } else if (cc === CC_BACK_SLASH) {
252
+ } else if (cc === CC_REVERSE_SOLIDUS) {
226
253
  if (pos + 1 === input.length) return pos;
227
254
  const cc = input.charCodeAt(pos + 1);
228
255
  if (_isNewLine(cc)) return pos;
@@ -231,11 +258,7 @@ const consumeMinus = (input, pos, callbacks) => {
231
258
  return callbacks.identifier(input, start, pos);
232
259
  }
233
260
  } else if (_isIdentifierStartCode(cc)) {
234
- pos++;
235
- pos = _consumeIdentifier(input, pos);
236
- if (callbacks.identifier !== undefined) {
237
- return callbacks.identifier(input, start, pos);
238
- }
261
+ pos = consumeOtherIdentifier(input, pos - 1, callbacks);
239
262
  }
240
263
  return pos;
241
264
  };
@@ -289,9 +312,10 @@ const consumeOtherIdentifier = (input, pos, callbacks) => {
289
312
  const consumePotentialUrl = (input, pos, callbacks) => {
290
313
  const start = pos;
291
314
  pos = _consumeIdentifier(input, pos);
315
+ const nextPos = pos + 1;
292
316
  if (
293
317
  pos === start + 3 &&
294
- input.slice(start, pos + 1).toLowerCase() === "url("
318
+ input.slice(start, nextPos).toLowerCase() === "url("
295
319
  ) {
296
320
  pos++;
297
321
  let cc = input.charCodeAt(pos);
@@ -301,26 +325,15 @@ const consumePotentialUrl = (input, pos, callbacks) => {
301
325
  cc = input.charCodeAt(pos);
302
326
  }
303
327
  if (cc === CC_QUOTATION_MARK || cc === CC_APOSTROPHE) {
304
- pos++;
305
- const contentStart = pos;
306
- pos = _consumeString(input, pos, cc);
307
- const contentEnd = pos - 1;
308
- cc = input.charCodeAt(pos);
309
- while (_isWhiteSpace(cc)) {
310
- pos++;
311
- if (pos === input.length) return pos;
312
- cc = input.charCodeAt(pos);
328
+ if (callbacks.function !== undefined) {
329
+ return callbacks.function(input, start, nextPos);
313
330
  }
314
- if (cc !== CC_RIGHT_PARENTHESIS) return pos;
315
- pos++;
316
- if (callbacks.url !== undefined)
317
- return callbacks.url(input, start, pos, contentStart, contentEnd);
318
- return pos;
331
+ return nextPos;
319
332
  } else {
320
333
  const contentStart = pos;
321
334
  let contentEnd;
322
335
  for (;;) {
323
- if (cc === CC_BACK_SLASH) {
336
+ if (cc === CC_REVERSE_SOLIDUS) {
324
337
  pos++;
325
338
  if (pos === input.length) return pos;
326
339
  pos++;
@@ -439,7 +452,7 @@ const consumeComma = (input, pos, callbacks) => {
439
452
  const _consumeIdentifier = (input, pos) => {
440
453
  for (;;) {
441
454
  const cc = input.charCodeAt(pos);
442
- if (cc === CC_BACK_SLASH) {
455
+ if (cc === CC_REVERSE_SOLIDUS) {
443
456
  pos++;
444
457
  if (pos === input.length) return pos;
445
458
  pos++;
@@ -513,7 +526,6 @@ const consumeLessThan = (input, pos, callbacks) => {
513
526
  return pos + 1;
514
527
  };
515
528
 
516
- /** @type {CharHandler} */
517
529
  const consumeAt = (input, pos, callbacks) => {
518
530
  const start = pos;
519
531
  pos++;
@@ -527,65 +539,102 @@ const consumeAt = (input, pos, callbacks) => {
527
539
  return pos;
528
540
  };
529
541
 
542
+ /** @type {CharHandler} */
543
+ const consumeReverseSolidus = (input, pos, callbacks) => {
544
+ const start = pos;
545
+ pos++;
546
+ if (pos === input.length) return pos;
547
+ // If the input stream starts with a valid escape, reconsume the current input code point, consume an ident-like token, and return it.
548
+ if (
549
+ _isTwoCodePointsAreValidEscape(
550
+ input.charCodeAt(start),
551
+ input.charCodeAt(pos)
552
+ )
553
+ ) {
554
+ return consumeOtherIdentifier(input, pos - 1, callbacks);
555
+ }
556
+ // Otherwise, this is a parse error. Return a <delim-token> with its value set to the current input code point.
557
+ return pos;
558
+ };
559
+
530
560
  const CHAR_MAP = Array.from({ length: 0x80 }, (_, cc) => {
531
561
  // https://drafts.csswg.org/css-syntax/#consume-token
532
562
  switch (cc) {
563
+ // whitespace
533
564
  case CC_LINE_FEED:
534
565
  case CC_CARRIAGE_RETURN:
535
566
  case CC_FORM_FEED:
536
567
  case CC_TAB:
537
568
  case CC_SPACE:
538
569
  return consumeSpace;
570
+ // U+0022 QUOTATION MARK (")
539
571
  case CC_QUOTATION_MARK:
540
- case CC_APOSTROPHE:
541
572
  return consumeString(cc);
573
+ // U+0023 NUMBER SIGN (#)
542
574
  case CC_NUMBER_SIGN:
543
575
  return consumeNumberSign;
544
- case CC_SLASH:
545
- return consumePotentialComment;
546
- // case CC_LEFT_SQUARE:
547
- // case CC_RIGHT_SQUARE:
548
- // case CC_COMMA:
549
- // case CC_COLON:
550
- // return consumeSingleCharToken;
551
- case CC_COMMA:
552
- return consumeComma;
553
- case CC_SEMICOLON:
554
- return consumeSemicolon;
576
+ // U+0027 APOSTROPHE (')
577
+ case CC_APOSTROPHE:
578
+ return consumeString(cc);
579
+ // U+0028 LEFT PARENTHESIS (()
555
580
  case CC_LEFT_PARENTHESIS:
556
581
  return consumeLeftParenthesis;
582
+ // U+0029 RIGHT PARENTHESIS ())
557
583
  case CC_RIGHT_PARENTHESIS:
558
584
  return consumeRightParenthesis;
559
- case CC_LEFT_CURLY:
560
- return consumeLeftCurlyBracket;
561
- case CC_RIGHT_CURLY:
562
- return consumeRightCurlyBracket;
563
- case CC_COLON:
564
- return consumePotentialPseudo;
585
+ // U+002B PLUS SIGN (+)
565
586
  case CC_PLUS_SIGN:
566
587
  return consumeNumericToken;
567
- case CC_FULL_STOP:
568
- return consumeDot;
588
+ // U+002C COMMA (,)
589
+ case CC_COMMA:
590
+ return consumeComma;
591
+ // U+002D HYPHEN-MINUS (-)
569
592
  case CC_HYPHEN_MINUS:
570
593
  return consumeMinus;
594
+ // U+002E FULL STOP (.)
595
+ case CC_FULL_STOP:
596
+ return consumeDot;
597
+ // U+003A COLON (:)
598
+ case CC_COLON:
599
+ return consumePotentialPseudo;
600
+ // U+003B SEMICOLON (;)
601
+ case CC_SEMICOLON:
602
+ return consumeSemicolon;
603
+ // U+003C LESS-THAN SIGN (<)
571
604
  case CC_LESS_THAN_SIGN:
572
605
  return consumeLessThan;
606
+ // U+0040 COMMERCIAL AT (@)
573
607
  case CC_AT_SIGN:
574
608
  return consumeAt;
609
+ // U+005B LEFT SQUARE BRACKET ([)
610
+ case CC_LEFT_SQUARE:
611
+ return consumeDelimToken;
612
+ // U+005C REVERSE SOLIDUS (\)
613
+ case CC_REVERSE_SOLIDUS:
614
+ return consumeReverseSolidus;
615
+ // U+005D RIGHT SQUARE BRACKET (])
616
+ case CC_RIGHT_SQUARE:
617
+ return consumeDelimToken;
618
+ // U+007B LEFT CURLY BRACKET ({)
619
+ case CC_LEFT_CURLY:
620
+ return consumeLeftCurlyBracket;
621
+ // U+007D RIGHT CURLY BRACKET (})
622
+ case CC_RIGHT_CURLY:
623
+ return consumeRightCurlyBracket;
624
+ // Optimization
575
625
  case CC_LOWER_U:
576
626
  case CC_UPPER_U:
577
627
  return consumePotentialUrl;
578
- case CC_LOW_LINE:
579
- return consumeOtherIdentifier;
580
628
  default:
629
+ // digit
581
630
  if (_isDigit(cc)) return consumeNumericToken;
582
- if (
583
- (cc >= CC_LOWER_A && cc <= CC_LOWER_Z) ||
584
- (cc >= CC_UPPER_A && cc <= CC_UPPER_Z)
585
- ) {
631
+ // ident-start code point
632
+ if (_isIdentStartCodePoint(cc)) {
586
633
  return consumeOtherIdentifier;
587
634
  }
588
- return consumeSingleCharToken;
635
+ // EOF, but we don't have it
636
+ // anything else
637
+ return consumeDelimToken;
589
638
  }
590
639
  });
591
640
 
@@ -595,9 +644,15 @@ const CHAR_MAP = Array.from({ length: 0x80 }, (_, cc) => {
595
644
  * @returns {void}
596
645
  */
597
646
  module.exports = (input, callbacks) => {
647
+ // This section describes how to consume a token from a stream of code points. It will return a single token of any type.
598
648
  let pos = 0;
599
649
  while (pos < input.length) {
650
+ // Consume comments.
651
+ pos = consumeComments(input, pos, callbacks);
652
+
600
653
  const cc = input.charCodeAt(pos);
654
+
655
+ // Consume the next input code point.
601
656
  if (cc < 0x80) {
602
657
  pos = CHAR_MAP[cc](input, pos, callbacks);
603
658
  } else {
@@ -609,7 +664,7 @@ module.exports = (input, callbacks) => {
609
664
  module.exports.eatComments = (input, pos) => {
610
665
  loop: for (;;) {
611
666
  const cc = input.charCodeAt(pos);
612
- if (cc === CC_SLASH) {
667
+ if (cc === CC_SOLIDUS) {
613
668
  if (pos === input.length) return pos;
614
669
  let cc = input.charCodeAt(pos + 1);
615
670
  if (cc !== CC_ASTERISK) return pos;
@@ -622,7 +677,7 @@ module.exports.eatComments = (input, pos) => {
622
677
  pos++;
623
678
  if (pos === input.length) return pos;
624
679
  cc = input.charCodeAt(pos);
625
- if (cc === CC_SLASH) {
680
+ if (cc === CC_SOLIDUS) {
626
681
  pos++;
627
682
  continue loop;
628
683
  }
@@ -636,7 +691,7 @@ module.exports.eatComments = (input, pos) => {
636
691
  module.exports.eatWhitespaceAndComments = (input, pos) => {
637
692
  loop: for (;;) {
638
693
  const cc = input.charCodeAt(pos);
639
- if (cc === CC_SLASH) {
694
+ if (cc === CC_SOLIDUS) {
640
695
  if (pos === input.length) return pos;
641
696
  let cc = input.charCodeAt(pos + 1);
642
697
  if (cc !== CC_ASTERISK) return pos;
@@ -649,7 +704,7 @@ module.exports.eatWhitespaceAndComments = (input, pos) => {
649
704
  pos++;
650
705
  if (pos === input.length) return pos;
651
706
  cc = input.charCodeAt(pos);
652
- if (cc === CC_SLASH) {
707
+ if (cc === CC_SOLIDUS) {
653
708
  pos++;
654
709
  continue loop;
655
710
  }