userpath-js 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/index.js +2 -2
- package/dist/index.js +2 -2
- package/dist/server/index.js +7 -18
- package/dist/types/src/browser/libs/referrer.d.ts.map +1 -1
- package/dist/types/src/server/crawler-detection.d.ts +74 -0
- package/dist/types/src/server/crawler-detection.d.ts.map +1 -0
- package/dist/types/src/server/crawler.d.ts +74 -0
- package/dist/types/src/server/crawler.d.ts.map +1 -0
- package/dist/types/src/server/index.d.ts +1 -0
- package/dist/types/src/server/index.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"referrer.d.ts","sourceRoot":"","sources":["../../../../../src/browser/libs/referrer.ts"],"names":[],"mappings":"AAsCA;;;;GAIG;AACH,wBAAgB,WAAW,IAAI,MAAM,
|
|
1
|
+
{"version":3,"file":"referrer.d.ts","sourceRoot":"","sources":["../../../../../src/browser/libs/referrer.ts"],"names":[],"mappings":"AAsCA;;;;GAIG;AACH,wBAAgB,WAAW,IAAI,MAAM,CAiDpC;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,IAAI,CAE7C;AAED;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,GAAG,IAAI,CAEpD"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Comprehensive crawler detection library for server-side use
|
|
3
|
+
* Includes traditional crawlers, AI/LLM crawlers, and other automated agents
|
|
4
|
+
*
|
|
5
|
+
* Sources:
|
|
6
|
+
* - https://momenticmarketing.com/blog/ai-search-crawlers-bots
|
|
7
|
+
* - https://darkvisitors.com/
|
|
8
|
+
* - https://github.com/dmitrizzle/disallow-ai
|
|
9
|
+
*/
|
|
10
|
+
export interface CrawlerPattern {
|
|
11
|
+
pattern: string;
|
|
12
|
+
category: 'ai-training' | 'ai-search' | 'ai-assistant' | 'traditional' | 'social' | 'research' | 'malicious';
|
|
13
|
+
vendor?: string;
|
|
14
|
+
description?: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Comprehensive list of crawler patterns organized by category
|
|
18
|
+
*/
|
|
19
|
+
export declare const CRAWLER_PATTERNS: CrawlerPattern[];
|
|
20
|
+
/**
|
|
21
|
+
* Configuration options for crawler detection
|
|
22
|
+
*/
|
|
23
|
+
export interface CrawlerDetectionOptions {
|
|
24
|
+
/** Categories of crawlers to detect */
|
|
25
|
+
categories?: CrawlerPattern['category'][];
|
|
26
|
+
/** Whether to use case-sensitive matching */
|
|
27
|
+
caseSensitive?: boolean;
|
|
28
|
+
/** Custom patterns to include */
|
|
29
|
+
customPatterns?: string[];
|
|
30
|
+
/** Patterns to exclude from detection */
|
|
31
|
+
excludePatterns?: string[];
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Result of crawler detection
|
|
35
|
+
*/
|
|
36
|
+
export interface CrawlerDetectionResult {
|
|
37
|
+
/** Whether a crawler was detected */
|
|
38
|
+
isCrawler: boolean;
|
|
39
|
+
/** Matched patterns */
|
|
40
|
+
matches: CrawlerPattern[];
|
|
41
|
+
/** The user agent string that was analyzed */
|
|
42
|
+
userAgent: string;
|
|
43
|
+
/** Confidence score (0-1) */
|
|
44
|
+
confidence: number;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Detects if a user agent string belongs to a crawler/bot
|
|
48
|
+
*/
|
|
49
|
+
export declare function detectCrawler(userAgent: string, options?: CrawlerDetectionOptions): CrawlerDetectionResult;
|
|
50
|
+
/**
|
|
51
|
+
* Quick check if user agent is a crawler (simplified interface)
|
|
52
|
+
*/
|
|
53
|
+
export declare function isCrawler(userAgent: string, options?: CrawlerDetectionOptions): boolean;
|
|
54
|
+
/**
|
|
55
|
+
* Check if user agent is an AI/LLM crawler specifically
|
|
56
|
+
*/
|
|
57
|
+
export declare function isAICrawler(userAgent: string): boolean;
|
|
58
|
+
/**
|
|
59
|
+
* Check if user agent is a training data collector
|
|
60
|
+
*/
|
|
61
|
+
export declare function isTrainingCrawler(userAgent: string): boolean;
|
|
62
|
+
/**
|
|
63
|
+
* Generate robots.txt disallow rules for crawler categories
|
|
64
|
+
*/
|
|
65
|
+
export declare function generateRobotsDisallow(categories: CrawlerPattern['category'][], path?: string): string;
|
|
66
|
+
/**
|
|
67
|
+
* Get all known AI/LLM crawler patterns
|
|
68
|
+
*/
|
|
69
|
+
export declare function getAICrawlerPatterns(): CrawlerPattern[];
|
|
70
|
+
/**
|
|
71
|
+
* Get crawler patterns by vendor
|
|
72
|
+
*/
|
|
73
|
+
export declare function getCrawlerPatternsByVendor(vendor: string): CrawlerPattern[];
|
|
74
|
+
//# sourceMappingURL=crawler-detection.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawler-detection.d.ts","sourceRoot":"","sources":["../../../../src/server/crawler-detection.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,cAAc;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,WAAW,GAAG,cAAc,GAAG,aAAa,GAAG,QAAQ,GAAG,UAAU,GAAG,WAAW,CAAC;IAC7G,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,EAAE,cAAc,EAkV5C,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC,uCAAuC;IACvC,UAAU,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;IAC1C,6CAA6C;IAC7C,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iCAAiC;IACjC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,yCAAyC;IACzC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,qCAAqC;IACrC,SAAS,EAAE,OAAO,CAAC;IACnB,uBAAuB;IACvB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,UAAU,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC5B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,uBAA4B,GACnC,sBAAsB,CAsExB;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAEvF;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAKtD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAK5D;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CACrC,UAAU,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,EACxC,IAAI,GAAE,MAAY,GAChB,MAAM,CAwBR;AAED;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,cAAc,EAAE,CAIvD;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,EAAE,CAI3E"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Comprehensive crawler detection library for server-side use
|
|
3
|
+
* Includes traditional crawlers, AI/LLM crawlers, and other automated agents
|
|
4
|
+
*
|
|
5
|
+
* Sources:
|
|
6
|
+
* - https://momenticmarketing.com/blog/ai-search-crawlers-bots
|
|
7
|
+
* - https://darkvisitors.com/
|
|
8
|
+
* - https://github.com/dmitrizzle/disallow-ai
|
|
9
|
+
*/
|
|
10
|
+
export interface CrawlerPattern {
|
|
11
|
+
pattern: string;
|
|
12
|
+
category: 'ai-training' | 'ai-search' | 'ai-assistant' | 'traditional' | 'social' | 'research' | 'malicious';
|
|
13
|
+
vendor?: string;
|
|
14
|
+
description?: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Comprehensive list of crawler patterns organized by category
|
|
18
|
+
*/
|
|
19
|
+
export declare const CRAWLER_PATTERNS: CrawlerPattern[];
|
|
20
|
+
/**
|
|
21
|
+
* Configuration options for crawler detection
|
|
22
|
+
*/
|
|
23
|
+
export interface CrawlerDetectionOptions {
|
|
24
|
+
/** Categories of crawlers to detect */
|
|
25
|
+
categories?: CrawlerPattern['category'][];
|
|
26
|
+
/** Whether to use case-sensitive matching */
|
|
27
|
+
caseSensitive?: boolean;
|
|
28
|
+
/** Custom patterns to include */
|
|
29
|
+
customPatterns?: string[];
|
|
30
|
+
/** Patterns to exclude from detection */
|
|
31
|
+
excludePatterns?: string[];
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Result of crawler detection
|
|
35
|
+
*/
|
|
36
|
+
export interface CrawlerDetectionResult {
|
|
37
|
+
/** Whether a crawler was detected */
|
|
38
|
+
isCrawler: boolean;
|
|
39
|
+
/** Matched patterns */
|
|
40
|
+
matches: CrawlerPattern[];
|
|
41
|
+
/** The user agent string that was analyzed */
|
|
42
|
+
userAgent: string;
|
|
43
|
+
/** Confidence score (0-1) */
|
|
44
|
+
confidence: number;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Detects if a user agent string belongs to a crawler/bot
|
|
48
|
+
*/
|
|
49
|
+
export declare function detectCrawler(userAgent: string, options?: CrawlerDetectionOptions): CrawlerDetectionResult;
|
|
50
|
+
/**
|
|
51
|
+
* Quick check if user agent is a crawler (simplified interface)
|
|
52
|
+
*/
|
|
53
|
+
export declare function isCrawler(userAgent: string, options?: CrawlerDetectionOptions): boolean;
|
|
54
|
+
/**
|
|
55
|
+
* Check if user agent is an AI/LLM crawler specifically
|
|
56
|
+
*/
|
|
57
|
+
export declare function isAICrawler(userAgent: string): boolean;
|
|
58
|
+
/**
|
|
59
|
+
* Check if user agent is a training data collector
|
|
60
|
+
*/
|
|
61
|
+
export declare function isTrainingCrawler(userAgent: string): boolean;
|
|
62
|
+
/**
|
|
63
|
+
* Generate robots.txt disallow rules for crawler categories
|
|
64
|
+
*/
|
|
65
|
+
export declare function generateRobotsDisallow(categories: CrawlerPattern['category'][], path?: string): string;
|
|
66
|
+
/**
|
|
67
|
+
* Get all known AI/LLM crawler patterns
|
|
68
|
+
*/
|
|
69
|
+
export declare function getAICrawlerPatterns(): CrawlerPattern[];
|
|
70
|
+
/**
|
|
71
|
+
* Get crawler patterns by vendor
|
|
72
|
+
*/
|
|
73
|
+
export declare function getCrawlerPatternsByVendor(vendor: string): CrawlerPattern[];
|
|
74
|
+
//# sourceMappingURL=crawler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../../../src/server/crawler.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,cAAc;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,WAAW,GAAG,cAAc,GAAG,aAAa,GAAG,QAAQ,GAAG,UAAU,GAAG,WAAW,CAAC;IAC7G,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,EAAE,cAAc,EAkV5C,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC,uCAAuC;IACvC,UAAU,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;IAC1C,6CAA6C;IAC7C,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iCAAiC;IACjC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,yCAAyC;IACzC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,qCAAqC;IACrC,SAAS,EAAE,OAAO,CAAC;IACnB,uBAAuB;IACvB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,UAAU,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC5B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,uBAA4B,GACnC,sBAAsB,CAsExB;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAEvF;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAKtD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAK5D;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CACrC,UAAU,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,EACxC,IAAI,GAAE,MAAY,GAChB,MAAM,CAwBR;AAED;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,cAAc,EAAE,CAIvD;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,EAAE,CAI3E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/server/index.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,eAAO,MAAM,QAAQ;IACpB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;mBAEO,OAAO,mCAMb;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;KACjB;CA4HF,CAAC;AAEF,cAAc,SAAS,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/server/index.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,eAAO,MAAM,QAAQ;IACpB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;mBAEO,OAAO,mCAMb;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;KACjB;CA4HF,CAAC;AAEF,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC"}
|