webpeel 0.12.0 → 0.12.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/README.md +82 -9
  2. package/dist/cli.js +97 -6
  3. package/dist/cli.js.map +1 -1
  4. package/dist/core/actions.d.ts +28 -0
  5. package/dist/core/actions.d.ts.map +1 -1
  6. package/dist/core/actions.js +60 -0
  7. package/dist/core/actions.js.map +1 -1
  8. package/dist/core/bm25-filter.d.ts +10 -0
  9. package/dist/core/bm25-filter.d.ts.map +1 -1
  10. package/dist/core/bm25-filter.js +40 -0
  11. package/dist/core/bm25-filter.js.map +1 -1
  12. package/dist/core/content-pruner.d.ts +12 -5
  13. package/dist/core/content-pruner.d.ts.map +1 -1
  14. package/dist/core/content-pruner.js +247 -190
  15. package/dist/core/content-pruner.js.map +1 -1
  16. package/dist/core/research.d.ts +67 -0
  17. package/dist/core/research.d.ts.map +1 -0
  18. package/dist/core/research.js +254 -0
  19. package/dist/core/research.js.map +1 -0
  20. package/dist/index.d.ts.map +1 -1
  21. package/dist/index.js +37 -3
  22. package/dist/index.js.map +1 -1
  23. package/dist/mcp/server.js +107 -2
  24. package/dist/mcp/server.js.map +1 -1
  25. package/dist/server/app.d.ts +14 -0
  26. package/dist/server/app.d.ts.map +1 -0
  27. package/dist/server/app.js +189 -0
  28. package/dist/server/app.js.map +1 -0
  29. package/dist/server/auth-store.d.ts +28 -0
  30. package/dist/server/auth-store.d.ts.map +1 -0
  31. package/dist/server/auth-store.js +89 -0
  32. package/dist/server/auth-store.js.map +1 -0
  33. package/dist/server/job-queue.d.ts +93 -0
  34. package/dist/server/job-queue.d.ts.map +1 -0
  35. package/dist/server/job-queue.js +144 -0
  36. package/dist/server/job-queue.js.map +1 -0
  37. package/dist/server/middleware/auth.d.ts +28 -0
  38. package/dist/server/middleware/auth.d.ts.map +1 -0
  39. package/dist/server/middleware/auth.js +183 -0
  40. package/dist/server/middleware/auth.js.map +1 -0
  41. package/dist/server/middleware/rate-limit.d.ts +23 -0
  42. package/dist/server/middleware/rate-limit.d.ts.map +1 -0
  43. package/dist/server/middleware/rate-limit.js +126 -0
  44. package/dist/server/middleware/rate-limit.js.map +1 -0
  45. package/dist/server/middleware/url-validator.d.ts +16 -0
  46. package/dist/server/middleware/url-validator.d.ts.map +1 -0
  47. package/dist/server/middleware/url-validator.js +187 -0
  48. package/dist/server/middleware/url-validator.js.map +1 -0
  49. package/dist/server/pg-auth-store.d.ts +129 -0
  50. package/dist/server/pg-auth-store.d.ts.map +1 -0
  51. package/dist/server/pg-auth-store.js +457 -0
  52. package/dist/server/pg-auth-store.js.map +1 -0
  53. package/dist/server/pg-job-queue.d.ts +60 -0
  54. package/dist/server/pg-job-queue.d.ts.map +1 -0
  55. package/dist/server/pg-job-queue.js +365 -0
  56. package/dist/server/pg-job-queue.js.map +1 -0
  57. package/dist/server/premium/domain-intel.d.ts +17 -0
  58. package/dist/server/premium/domain-intel.d.ts.map +1 -0
  59. package/dist/server/premium/domain-intel.js +134 -0
  60. package/dist/server/premium/domain-intel.js.map +1 -0
  61. package/dist/server/premium/index.d.ts +18 -0
  62. package/dist/server/premium/index.d.ts.map +1 -0
  63. package/dist/server/premium/index.js +36 -0
  64. package/dist/server/premium/index.js.map +1 -0
  65. package/dist/server/premium/swr-cache.d.ts +15 -0
  66. package/dist/server/premium/swr-cache.d.ts.map +1 -0
  67. package/dist/server/premium/swr-cache.js +35 -0
  68. package/dist/server/premium/swr-cache.js.map +1 -0
  69. package/dist/server/routes/activity.d.ts +7 -0
  70. package/dist/server/routes/activity.d.ts.map +1 -0
  71. package/dist/server/routes/activity.js +66 -0
  72. package/dist/server/routes/activity.js.map +1 -0
  73. package/dist/server/routes/agent.d.ts +12 -0
  74. package/dist/server/routes/agent.d.ts.map +1 -0
  75. package/dist/server/routes/agent.js +356 -0
  76. package/dist/server/routes/agent.js.map +1 -0
  77. package/dist/server/routes/answer.d.ts +6 -0
  78. package/dist/server/routes/answer.d.ts.map +1 -0
  79. package/dist/server/routes/answer.js +124 -0
  80. package/dist/server/routes/answer.js.map +1 -0
  81. package/dist/server/routes/batch.d.ts +7 -0
  82. package/dist/server/routes/batch.d.ts.map +1 -0
  83. package/dist/server/routes/batch.js +287 -0
  84. package/dist/server/routes/batch.js.map +1 -0
  85. package/dist/server/routes/cli-usage.d.ts +7 -0
  86. package/dist/server/routes/cli-usage.d.ts.map +1 -0
  87. package/dist/server/routes/cli-usage.js +121 -0
  88. package/dist/server/routes/cli-usage.js.map +1 -0
  89. package/dist/server/routes/compat.d.ts +24 -0
  90. package/dist/server/routes/compat.d.ts.map +1 -0
  91. package/dist/server/routes/compat.js +651 -0
  92. package/dist/server/routes/compat.js.map +1 -0
  93. package/dist/server/routes/extract.d.ts +9 -0
  94. package/dist/server/routes/extract.d.ts.map +1 -0
  95. package/dist/server/routes/extract.js +121 -0
  96. package/dist/server/routes/extract.js.map +1 -0
  97. package/dist/server/routes/fetch.d.ts +7 -0
  98. package/dist/server/routes/fetch.d.ts.map +1 -0
  99. package/dist/server/routes/fetch.js +537 -0
  100. package/dist/server/routes/fetch.js.map +1 -0
  101. package/dist/server/routes/health.d.ts +8 -0
  102. package/dist/server/routes/health.d.ts.map +1 -0
  103. package/dist/server/routes/health.js +36 -0
  104. package/dist/server/routes/health.js.map +1 -0
  105. package/dist/server/routes/jobs.d.ts +8 -0
  106. package/dist/server/routes/jobs.d.ts.map +1 -0
  107. package/dist/server/routes/jobs.js +374 -0
  108. package/dist/server/routes/jobs.js.map +1 -0
  109. package/dist/server/routes/mcp.d.ts +16 -0
  110. package/dist/server/routes/mcp.d.ts.map +1 -0
  111. package/dist/server/routes/mcp.js +475 -0
  112. package/dist/server/routes/mcp.js.map +1 -0
  113. package/dist/server/routes/oauth.d.ts +10 -0
  114. package/dist/server/routes/oauth.d.ts.map +1 -0
  115. package/dist/server/routes/oauth.js +296 -0
  116. package/dist/server/routes/oauth.js.map +1 -0
  117. package/dist/server/routes/screenshot.d.ts +10 -0
  118. package/dist/server/routes/screenshot.d.ts.map +1 -0
  119. package/dist/server/routes/screenshot.js +217 -0
  120. package/dist/server/routes/screenshot.js.map +1 -0
  121. package/dist/server/routes/search.d.ts +7 -0
  122. package/dist/server/routes/search.d.ts.map +1 -0
  123. package/dist/server/routes/search.js +287 -0
  124. package/dist/server/routes/search.js.map +1 -0
  125. package/dist/server/routes/stats.d.ts +7 -0
  126. package/dist/server/routes/stats.d.ts.map +1 -0
  127. package/dist/server/routes/stats.js +65 -0
  128. package/dist/server/routes/stats.js.map +1 -0
  129. package/dist/server/routes/stripe.d.ts +9 -0
  130. package/dist/server/routes/stripe.d.ts.map +1 -0
  131. package/dist/server/routes/stripe.js +233 -0
  132. package/dist/server/routes/stripe.js.map +1 -0
  133. package/dist/server/routes/users.d.ts +9 -0
  134. package/dist/server/routes/users.d.ts.map +1 -0
  135. package/dist/server/routes/users.js +954 -0
  136. package/dist/server/routes/users.js.map +1 -0
  137. package/dist/server/routes/webhooks.d.ts +15 -0
  138. package/dist/server/routes/webhooks.d.ts.map +1 -0
  139. package/dist/server/routes/webhooks.js +73 -0
  140. package/dist/server/routes/webhooks.js.map +1 -0
  141. package/dist/server/sentry.d.ts +14 -0
  142. package/dist/server/sentry.d.ts.map +1 -0
  143. package/dist/server/sentry.js +39 -0
  144. package/dist/server/sentry.js.map +1 -0
  145. package/dist/types.d.ts +13 -0
  146. package/dist/types.d.ts.map +1 -1
  147. package/dist/types.js.map +1 -1
  148. package/package.json +3 -2
@@ -0,0 +1,144 @@
1
+ /**
2
+ * Job queue for async operations
3
+ *
4
+ * Factory creates PostgreSQL-backed queue in production or in-memory queue for local dev.
5
+ * Tracks crawl, batch scrape, and extraction jobs with progress updates.
6
+ */
7
+ import { randomUUID } from 'crypto';
8
+ import { PostgresJobQueue } from './pg-job-queue.js';
9
+ /**
10
+ * In-memory job queue for local development
11
+ */
12
+ export class InMemoryJobQueue {
13
+ jobs = new Map();
14
+ cleanupInterval;
15
+ constructor() {
16
+ // Clean expired jobs every hour
17
+ this.cleanupInterval = setInterval(() => {
18
+ this.cleanExpired();
19
+ }, 60 * 60 * 1000);
20
+ }
21
+ /**
22
+ * Create a new job
23
+ */
24
+ createJob(type, webhook, ownerId) {
25
+ const now = new Date().toISOString();
26
+ const job = {
27
+ id: randomUUID(),
28
+ type,
29
+ status: 'queued',
30
+ progress: 0,
31
+ total: 0,
32
+ completed: 0,
33
+ creditsUsed: 0,
34
+ data: [],
35
+ webhook,
36
+ ownerId,
37
+ createdAt: now,
38
+ updatedAt: now,
39
+ expiresAt: new Date(Date.now() + 25 * 60 * 60 * 1000).toISOString(), // 25h from now (updated on completion)
40
+ };
41
+ this.jobs.set(job.id, job);
42
+ return job;
43
+ }
44
+ /**
45
+ * Get a job by ID
46
+ */
47
+ getJob(id) {
48
+ return this.jobs.get(id) || null;
49
+ }
50
+ /**
51
+ * Update a job
52
+ */
53
+ updateJob(id, update) {
54
+ const job = this.jobs.get(id);
55
+ if (!job)
56
+ return;
57
+ Object.assign(job, update, {
58
+ updatedAt: new Date().toISOString(),
59
+ });
60
+ // When job completes/fails, set expiration to 24h from now
61
+ if (update.status === 'completed' || update.status === 'failed') {
62
+ job.expiresAt = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString();
63
+ }
64
+ // Update progress percentage
65
+ if (job.total > 0) {
66
+ job.progress = Math.round((job.completed / job.total) * 100);
67
+ }
68
+ this.jobs.set(id, job);
69
+ }
70
+ /**
71
+ * Cancel a job
72
+ */
73
+ cancelJob(id) {
74
+ const job = this.jobs.get(id);
75
+ if (!job)
76
+ return false;
77
+ // Can only cancel queued or processing jobs
78
+ if (job.status !== 'queued' && job.status !== 'processing') {
79
+ return false;
80
+ }
81
+ this.updateJob(id, {
82
+ status: 'cancelled',
83
+ expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(),
84
+ });
85
+ return true;
86
+ }
87
+ /**
88
+ * List jobs with optional filters
89
+ */
90
+ listJobs(options) {
91
+ let jobs = Array.from(this.jobs.values());
92
+ if (options?.ownerId) {
93
+ jobs = jobs.filter(j => j.ownerId === options.ownerId);
94
+ }
95
+ if (options?.type) {
96
+ jobs = jobs.filter(j => j.type === options.type);
97
+ }
98
+ if (options?.status) {
99
+ jobs = jobs.filter(j => j.status === options.status);
100
+ }
101
+ // Sort by creation time (newest first)
102
+ jobs.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
103
+ if (options?.limit) {
104
+ jobs = jobs.slice(0, options.limit);
105
+ }
106
+ return jobs;
107
+ }
108
+ /**
109
+ * Remove expired jobs
110
+ */
111
+ cleanExpired() {
112
+ const now = Date.now();
113
+ for (const [id, job] of this.jobs.entries()) {
114
+ if (new Date(job.expiresAt).getTime() < now) {
115
+ this.jobs.delete(id);
116
+ }
117
+ }
118
+ }
119
+ /**
120
+ * Clean up interval on shutdown
121
+ */
122
+ destroy() {
123
+ clearInterval(this.cleanupInterval);
124
+ }
125
+ }
126
+ /**
127
+ * Create job queue based on environment
128
+ * - Uses PostgreSQL if DATABASE_URL is set
129
+ * - Falls back to in-memory for local development
130
+ */
131
+ export function createJobQueue() {
132
+ if (process.env.DATABASE_URL) {
133
+ console.log('Using PostgreSQL job queue');
134
+ return new PostgresJobQueue();
135
+ }
136
+ else {
137
+ console.log('Using in-memory job queue');
138
+ return new InMemoryJobQueue();
139
+ }
140
+ }
141
+ // Legacy global instance for backwards compatibility (deprecated)
142
+ // @deprecated Use createJobQueue() instead
143
+ export const jobQueue = new InMemoryJobQueue();
144
+ //# sourceMappingURL=job-queue.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"job-queue.js","sourceRoot":"","sources":["../../src/server/job-queue.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAsCrD;;GAEG;AACH,MAAM,OAAO,gBAAgB;IACnB,IAAI,GAAqB,IAAI,GAAG,EAAE,CAAC;IACnC,eAAe,CAAiB;IAExC;QACE,gCAAgC;QAChC,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC,GAAG,EAAE;YACtC,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,IAAiB,EAAE,OAAuB,EAAE,OAAgB;QACpE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,GAAG,GAAQ;YACf,EAAE,EAAE,UAAU,EAAE;YAChB,IAAI;YACJ,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,CAAC;YACX,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,CAAC;YACZ,WAAW,EAAE,CAAC;YACd,IAAI,EAAE,EAAE;YACR,OAAO;YACP,OAAO;YACP,SAAS,EAAE,GAAG;YACd,SAAS,EAAE,GAAG;YACd,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,uCAAuC;SAC7G,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC3B,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,EAAU;QACf,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,EAAU,EAAE,MAAoB;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9B,IAAI,CAAC,GAAG;YAAE,OAAO;QAEjB,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE;YACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;QAEH,2DAA2D;QAC3D,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAChE,GAAG,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QAC3E,CAAC;QAED,6BAA6B;QAC7B,IAAI,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YAClB,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,EAAU;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9B,IAAI,CAAC,GAAG;YAAE,OAAO,KAAK,CAAC;QAEvB,4CAA4C;QAC5C,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YAC3D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE;YACjB,MAAM,EAAE,WAAW;YACnB,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE;SACpE,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,OAKR;QACC,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE1C,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;YACrB,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,OAAO,EAAE,IAAI,EAAE,CAAC;YAClB,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;QAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YACpB,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;QACvD,CAAC;QAED,uCAAuC;QACvC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAEvF,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;YACnB,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,YAAY;QACV,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5C,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO;QACL,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACtC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc;IAC5B,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAC1C,OAAO,IAAI,gBAAgB,EAAE,CAAC;IAChC,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACzC,OAAO,IAAI,gBAAgB,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED,kEAAkE;AAClE,2CAA2C;AAC3C,MAAM,CAAC,MAAM,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAC"}
@@ -0,0 +1,28 @@
1
+ /**
2
+ * API key authentication middleware with SOFT LIMIT enforcement
3
+ *
4
+ * Philosophy: Never fully block users. When weekly limits are exceeded,
5
+ * degrade to HTTP-only mode instead of returning 429.
6
+ * BURST limits (hourly) are HARD limits and return 429.
7
+ *
8
+ * Dual auth: Accepts both API keys AND JWT session tokens.
9
+ * API keys are validated via the auth store; JWTs are verified with JWT_SECRET.
10
+ * Dashboard pages use JWT tokens; CLI/SDK users use API keys.
11
+ */
12
+ import { Request, Response, NextFunction } from 'express';
13
+ import { AuthStore, ApiKeyInfo } from '../auth-store.js';
14
+ declare global {
15
+ namespace Express {
16
+ interface Request {
17
+ auth?: {
18
+ keyInfo: ApiKeyInfo | null;
19
+ tier: 'free' | 'starter' | 'pro' | 'enterprise' | 'max';
20
+ rateLimit: number;
21
+ softLimited: boolean;
22
+ extraUsageAvailable: boolean;
23
+ };
24
+ }
25
+ }
26
+ }
27
+ export declare function createAuthMiddleware(authStore: AuthStore): (req: Request, res: Response, next: NextFunction) => Promise<void>;
28
+ //# sourceMappingURL=auth.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"auth.d.ts","sourceRoot":"","sources":["../../../src/server/middleware/auth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE1D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,OAAO,CAAC;QAChB,UAAU,OAAO;YACf,IAAI,CAAC,EAAE;gBACL,OAAO,EAAE,UAAU,GAAG,IAAI,CAAC;gBAC3B,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,GAAG,YAAY,GAAG,KAAK,CAAC;gBACxD,SAAS,EAAE,MAAM,CAAC;gBAClB,WAAW,EAAE,OAAO,CAAC;gBACrB,mBAAmB,EAAE,OAAO,CAAC;aAC9B,CAAC;SACH;KACF;CACF;AAED,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,SAAS,IACzC,KAAK,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,YAAY,mBAiM9D"}
@@ -0,0 +1,183 @@
1
+ /**
2
+ * API key authentication middleware with SOFT LIMIT enforcement
3
+ *
4
+ * Philosophy: Never fully block users. When weekly limits are exceeded,
5
+ * degrade to HTTP-only mode instead of returning 429.
6
+ * BURST limits (hourly) are HARD limits and return 429.
7
+ *
8
+ * Dual auth: Accepts both API keys AND JWT session tokens.
9
+ * API keys are validated via the auth store; JWTs are verified with JWT_SECRET.
10
+ * Dashboard pages use JWT tokens; CLI/SDK users use API keys.
11
+ */
12
+ import jwt from 'jsonwebtoken';
13
+ import { PostgresAuthStore } from '../pg-auth-store.js';
14
+ export function createAuthMiddleware(authStore) {
15
+ return async (req, res, next) => {
16
+ try {
17
+ // Extract API key from Authorization header or X-API-Key header
18
+ const authHeader = req.headers.authorization;
19
+ const apiKeyHeader = req.headers['x-api-key'];
20
+ // SECURITY: Skip API key auth for public/JWT-protected endpoints
21
+ // These routes either need no auth or use their own JWT middleware
22
+ const isPublicEndpoint = req.path === '/health' ||
23
+ req.path.startsWith('/v1/auth/') ||
24
+ req.path === '/v1/webhooks/stripe' ||
25
+ req.path === '/v1/me' ||
26
+ req.path.startsWith('/v1/keys') ||
27
+ req.path === '/v1/usage' ||
28
+ req.path.startsWith('/v1/extra-usage');
29
+ if (isPublicEndpoint) {
30
+ req.auth = {
31
+ keyInfo: null,
32
+ tier: 'free',
33
+ rateLimit: 10,
34
+ softLimited: false,
35
+ extraUsageAvailable: false,
36
+ };
37
+ return next();
38
+ }
39
+ let apiKey = null;
40
+ if (authHeader?.startsWith('Bearer ')) {
41
+ apiKey = authHeader.slice(7);
42
+ }
43
+ else if (apiKeyHeader && typeof apiKeyHeader === 'string') {
44
+ apiKey = apiKeyHeader;
45
+ }
46
+ if (!apiKey) {
47
+ // Allow anonymous free-tier access (125/week, 25/hr burst)
48
+ // This enables the playground and basic usage without signup
49
+ req.auth = {
50
+ keyInfo: null,
51
+ tier: 'free',
52
+ rateLimit: 25, // requests per minute for anonymous
53
+ softLimited: false,
54
+ extraUsageAvailable: false,
55
+ };
56
+ return next();
57
+ }
58
+ // Validate API key if provided
59
+ let keyInfo = null;
60
+ let softLimited = false;
61
+ let extraUsageAvailable = false;
62
+ if (apiKey) {
63
+ keyInfo = await authStore.validateKey(apiKey);
64
+ if (!keyInfo) {
65
+ // API key not found — try JWT session token as fallback.
66
+ // Dashboard uses JWT tokens (from /v1/auth/login or /v1/auth/oauth),
67
+ // while CLI/SDK users use API keys. Support both seamlessly.
68
+ const jwtSecret = process.env.JWT_SECRET;
69
+ if (jwtSecret) {
70
+ try {
71
+ const payload = jwt.verify(apiKey, jwtSecret);
72
+ if (payload.userId) {
73
+ // Valid JWT — treat as authenticated session user.
74
+ // Set req.auth with null keyInfo (no API key) and attach
75
+ // the JWT payload so routes can use req.user.userId.
76
+ req.auth = {
77
+ keyInfo: null,
78
+ tier: payload.tier || 'free',
79
+ rateLimit: 25,
80
+ softLimited: false,
81
+ extraUsageAvailable: false,
82
+ };
83
+ req.user = payload;
84
+ return next();
85
+ }
86
+ }
87
+ catch {
88
+ // Not a valid JWT either — fall through to 401
89
+ }
90
+ }
91
+ res.status(401).json({
92
+ error: 'invalid_key',
93
+ message: 'Invalid or expired API key. Check your key at https://app.webpeel.dev/keys or generate a new one.',
94
+ docs: 'https://webpeel.dev/docs/api-reference#authentication',
95
+ });
96
+ return;
97
+ }
98
+ // Check limits (only for PostgresAuthStore)
99
+ if (authStore instanceof PostgresAuthStore) {
100
+ // HARD LIMIT: Check burst limit first (per-hour cap)
101
+ const { allowed: burstAllowed, burst } = await authStore.checkBurstLimit(apiKey);
102
+ if (!burstAllowed) {
103
+ // Burst limit exceeded - HARD 429 with Retry-After
104
+ const retryAfterSeconds = 60 * parseInt(burst.resetsIn.match(/\d+/)?.[0] || '1', 10);
105
+ res.setHeader('Retry-After', retryAfterSeconds.toString());
106
+ res.setHeader('X-Burst-Limit', burst.limit.toString());
107
+ res.setHeader('X-Burst-Used', burst.count.toString());
108
+ const tier = keyInfo?.tier || 'free';
109
+ const upgradeHint = tier === 'free'
110
+ ? ' Upgrade to Pro ($9/mo) for 100/hr → https://webpeel.dev/#pricing'
111
+ : tier === 'pro'
112
+ ? ' Upgrade to Max ($29/mo) for 500/hr → https://webpeel.dev/#pricing'
113
+ : '';
114
+ res.status(429).json({
115
+ error: 'burst_limit_exceeded',
116
+ message: `Hourly burst limit exceeded (${burst.count}/${burst.limit} on ${tier} plan). Resets in ${burst.resetsIn}.${upgradeHint}`,
117
+ retryAfter: burst.resetsIn,
118
+ plan: tier,
119
+ docs: 'https://webpeel.dev/docs/api-reference#rate-limits',
120
+ });
121
+ return;
122
+ }
123
+ // Add burst headers
124
+ res.setHeader('X-Burst-Limit', burst.limit.toString());
125
+ res.setHeader('X-Burst-Used', burst.count.toString());
126
+ res.setHeader('X-Burst-Remaining', burst.remaining.toString());
127
+ // SOFT LIMIT: Check weekly usage
128
+ const { allowed, usage } = await authStore.checkLimit(apiKey);
129
+ // Check if extra usage is available
130
+ if (!allowed) {
131
+ extraUsageAvailable = await authStore.canUseExtraUsage(apiKey);
132
+ if (!extraUsageAvailable) {
133
+ // Over weekly quota, no extra usage — SOFT LIMIT (degrade to HTTP-only)
134
+ softLimited = true;
135
+ res.setHeader('X-Soft-Limited', 'true');
136
+ res.setHeader('X-Soft-Limit-Reason', 'Weekly quota exceeded. Requests degraded to HTTP-only mode.');
137
+ res.setHeader('X-Upgrade-URL', 'https://webpeel.dev/#pricing');
138
+ }
139
+ }
140
+ // Add weekly usage headers
141
+ if (usage) {
142
+ res.setHeader('X-Weekly-Limit', usage.totalAvailable.toString());
143
+ res.setHeader('X-Weekly-Used', usage.totalUsed.toString());
144
+ res.setHeader('X-Weekly-Remaining', Math.max(0, usage.remaining).toString());
145
+ res.setHeader('X-Weekly-Percent', usage.percentUsed.toString());
146
+ res.setHeader('X-Weekly-Resets-At', usage.resetsAt);
147
+ // Warn if over 80% usage and not using extra usage
148
+ if (usage.percentUsed >= 80 && !softLimited && !extraUsageAvailable) {
149
+ res.setHeader('X-Usage-Warning', `You've used ${usage.percentUsed}% of your weekly quota. Consider upgrading at https://webpeel.dev/#pricing`);
150
+ }
151
+ }
152
+ // Add extra usage headers if available
153
+ const extraInfo = await authStore.getExtraUsageInfo(apiKey);
154
+ if (extraInfo) {
155
+ res.setHeader('X-Extra-Usage-Enabled', extraInfo.enabled ? 'true' : 'false');
156
+ res.setHeader('X-Extra-Usage-Balance', extraInfo.balance.toFixed(2));
157
+ if (extraInfo.enabled) {
158
+ res.setHeader('X-Extra-Usage-Spent', extraInfo.spent.toFixed(2));
159
+ res.setHeader('X-Extra-Usage-Limit', extraInfo.spendingLimit.toFixed(2));
160
+ }
161
+ }
162
+ }
163
+ }
164
+ // Set auth context on request
165
+ req.auth = {
166
+ keyInfo,
167
+ tier: keyInfo?.tier || 'free',
168
+ rateLimit: keyInfo?.rateLimit || 10,
169
+ softLimited,
170
+ extraUsageAvailable,
171
+ };
172
+ next();
173
+ }
174
+ catch (error) {
175
+ const err = error;
176
+ res.status(500).json({
177
+ error: 'auth_error',
178
+ message: err.message || 'Authentication failed',
179
+ });
180
+ }
181
+ };
182
+ }
183
+ //# sourceMappingURL=auth.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"auth.js","sourceRoot":"","sources":["../../../src/server/middleware/auth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,GAAG,MAAM,cAAc,CAAC;AAE/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAgBxD,MAAM,UAAU,oBAAoB,CAAC,SAAoB;IACvD,OAAO,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;QAC/D,IAAI,CAAC;YACH,gEAAgE;YAChE,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC;YAC7C,MAAM,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAE9C,iEAAiE;YACjE,mEAAmE;YACnE,MAAM,gBAAgB,GACpB,GAAG,CAAC,IAAI,KAAK,SAAS;gBACtB,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;gBAChC,GAAG,CAAC,IAAI,KAAK,qBAAqB;gBAClC,GAAG,CAAC,IAAI,KAAK,QAAQ;gBACrB,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;gBAC/B,GAAG,CAAC,IAAI,KAAK,WAAW;gBACxB,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;YAEzC,IAAI,gBAAgB,EAAE,CAAC;gBACrB,GAAG,CAAC,IAAI,GAAG;oBACT,OAAO,EAAE,IAAI;oBACb,IAAI,EAAE,MAAM;oBACZ,SAAS,EAAE,EAAE;oBACb,WAAW,EAAE,KAAK;oBAClB,mBAAmB,EAAE,KAAK;iBAC3B,CAAC;gBACF,OAAO,IAAI,EAAE,CAAC;YAChB,CAAC;YAED,IAAI,MAAM,GAAkB,IAAI,CAAC;YAEjC,IAAI,UAAU,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBACtC,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC/B,CAAC;iBAAM,IAAI,YAAY,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;gBAC5D,MAAM,GAAG,YAAY,CAAC;YACxB,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,2DAA2D;gBAC3D,6DAA6D;gBAC7D,GAAG,CAAC,IAAI,GAAG;oBACT,OAAO,EAAE,IAAI;oBACb,IAAI,EAAE,MAAM;oBACZ,SAAS,EAAE,EAAE,EAAG,oCAAoC;oBACpD,WAAW,EAAE,KAAK;oBAClB,mBAAmB,EAAE,KAAK;iBAC3B,CAAC;gBACF,OAAO,IAAI,EAAE,CAAC;YAChB,CAAC;YAED,+BAA+B;YAC/B,IAAI,OAAO,GAAsB,IAAI,CAAC;YACtC,IAAI,WAAW,GAAG,KAAK,CAAC;YACxB,IAAI,mBAAmB,GAAG,KAAK,CAAC;YAEhC,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,GAAG,MAAM,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAC9C,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,yDAAyD;oBACzD,qEAAqE;oBACrE,6DAA6D;oBAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;oBACzC,IAAI,SAAS,EAAE,CAAC;wBACd,IAAI,CAAC;4BACH,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAI3C,CAAC;4BACF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gCACnB,mDAAmD;gCACnD,yDAAyD;gCACzD,qDAAqD;gCACrD,GAAG,CAAC,IAAI,GAAG;oCACT,OAAO,EAAE,IAAI;oCACb,IAAI,EAAG,OAAO,CAAC,IAAY,IAAI,MAAM;oCACrC,SAAS,EAAE,EAAE;oCACb,WAAW,EAAE,KAAK;oCAClB,mBAAmB,EAAE,KAAK;iCAC3B,CAAC;gCACD,GAAW,CAAC,IAAI,GAAG,OAAO,CAAC;gCAC5B,OAAO,IAAI,EAAE,CAAC;4BAChB,CAAC;wBACH,CAAC;wBAAC,MAAM,CAAC;4BACP,+CAA+C;wBACjD,CAAC;oBACH,CAAC;oBAED,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,aAAa;wBACpB,OAAO,EAAE,mGAAmG;wBAC5G,IAAI,EAAE,uDAAuD;qBAC9D,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,4CAA4C;gBAC5C,IAAI,SAAS,YAAY,iBAAiB,EAAE,CAAC;oBAC3C,qDAAqD;oBACrD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;oBAEjF,IAAI,CAAC,YAAY,EAAE,CAAC;wBAClB,mDAAmD;wBACnD,MAAM,iBAAiB,GAAG,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;wBACrF,GAAG,CAAC,SAAS,CAAC,aAAa,EAAE,iBAAiB,CAAC,QAAQ,EAAE,CAAC,CAAC;wBAC3D,GAAG,CAAC,SAAS,CAAC,eAAe,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;wBACvD,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;wBAEtD,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,MAAM,CAAC;wBACrC,MAAM,WAAW,GAAG,IAAI,KAAK,MAAM;4BACjC,CAAC,CAAC,mEAAmE;4BACrE,CAAC,CAAC,IAAI,KAAK,KAAK;gCAChB,CAAC,CAAC,oEAAoE;gCACtE,CAAC,CAAC,EAAE,CAAC;wBACP,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;4BACnB,KAAK,EAAE,sBAAsB;4BAC7B,OAAO,EAAE,gCAAgC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,OAAO,IAAI,qBAAqB,KAAK,CAAC,QAAQ,IAAI,WAAW,EAAE;4BAClI,UAAU,EAAE,KAAK,CAAC,QAAQ;4BAC1B,IAAI,EAAE,IAAI;4BACV,IAAI,EAAE,oDAAoD;yBAC3D,CAAC,CAAC;wBACH,OAAO;oBACT,CAAC;oBAED,oBAAoB;oBACpB,GAAG,CAAC,SAAS,CAAC,eAAe,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;oBACvD,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;oBACtD,GAAG,CAAC,SAAS,CAAC,mBAAmB,EAAE,KAAK,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAE/D,iCAAiC;oBACjC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;oBAE9D,oCAAoC;oBACpC,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,mBAAmB,GAAG,MAAM,SAAS,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;wBAE/D,IAAI,CAAC,mBAAmB,EAAE,CAAC;4BACzB,wEAAwE;4BACxE,WAAW,GAAG,IAAI,CAAC;4BACnB,GAAG,CAAC,SAAS,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;4BACxC,GAAG,CAAC,SAAS,CAAC,qBAAqB,EAAE,6DAA6D,CAAC,CAAC;4BACpG,GAAG,CAAC,SAAS,CAAC,eAAe,EAAE,8BAA8B,CAAC,CAAC;wBACjE,CAAC;oBACH,CAAC;oBAED,2BAA2B;oBAC3B,IAAI,KAAK,EAAE,CAAC;wBACV,GAAG,CAAC,SAAS,CAAC,gBAAgB,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ,EAAE,CAAC,CAAC;wBACjE,GAAG,CAAC,SAAS,CAAC,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;wBAC3D,GAAG,CAAC,SAAS,CAAC,oBAAoB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;wBAC7E,GAAG,CAAC,SAAS,CAAC,kBAAkB,EAAE,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;wBAChE,GAAG,CAAC,SAAS,CAAC,oBAAoB,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;wBAEpD,mDAAmD;wBACnD,IAAI,KAAK,CAAC,WAAW,IAAI,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,mBAAmB,EAAE,CAAC;4BACpE,GAAG,CAAC,SAAS,CACX,iBAAiB,EACjB,eAAe,KAAK,CAAC,WAAW,4EAA4E,CAC7G,CAAC;wBACJ,CAAC;oBACH,CAAC;oBAED,uCAAuC;oBACvC,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;oBAC5D,IAAI,SAAS,EAAE,CAAC;wBACd,GAAG,CAAC,SAAS,CAAC,uBAAuB,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;wBAC7E,GAAG,CAAC,SAAS,CAAC,uBAAuB,EAAE,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;wBAErE,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;4BACtB,GAAG,CAAC,SAAS,CAAC,qBAAqB,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;4BACjE,GAAG,CAAC,SAAS,CAAC,qBAAqB,EAAE,SAAS,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC3E,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,8BAA8B;YAC9B,GAAG,CAAC,IAAI,GAAG;gBACT,OAAO;gBACP,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,MAAM;gBAC7B,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,EAAE;gBACnC,WAAW;gBACX,mBAAmB;aACpB,CAAC;YAEF,IAAI,EAAE,CAAC;QACT,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAc,CAAC;YAC3B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,YAAY;gBACnB,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,uBAAuB;aAChD,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Sliding window rate limiting middleware
3
+ */
4
+ import { Request, Response, NextFunction } from 'express';
5
+ export declare class RateLimiter {
6
+ private store;
7
+ private windowMs;
8
+ constructor(windowMs?: number);
9
+ /**
10
+ * Check if request is allowed under rate limit
11
+ */
12
+ checkLimit(identifier: string, limit: number): {
13
+ allowed: boolean;
14
+ remaining: number;
15
+ retryAfter?: number;
16
+ };
17
+ /**
18
+ * Clean up old entries (call periodically)
19
+ */
20
+ cleanup(): void;
21
+ }
22
+ export declare function createRateLimitMiddleware(limiter: RateLimiter): (req: Request, res: Response, next: NextFunction) => void;
23
+ //# sourceMappingURL=rate-limit.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rate-limit.d.ts","sourceRoot":"","sources":["../../../src/server/middleware/rate-limit.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAM1D,qBAAa,WAAW;IACtB,OAAO,CAAC,KAAK,CAAqC;IAClD,OAAO,CAAC,QAAQ,CAAS;gBAEb,QAAQ,GAAE,MAAc;IAIpC;;OAEG;IACH,UAAU,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG;QAC7C,OAAO,EAAE,OAAO,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAmCD;;OAEG;IACH,OAAO,IAAI,IAAI;CAWhB;AAeD,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,WAAW,IACpD,KAAK,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,YAAY,KAAG,IAAI,CAgE/D"}
@@ -0,0 +1,126 @@
1
+ /**
2
+ * Sliding window rate limiting middleware
3
+ */
4
+ export class RateLimiter {
5
+ store = new Map();
6
+ windowMs;
7
+ constructor(windowMs = 60000) {
8
+ this.windowMs = windowMs;
9
+ }
10
+ /**
11
+ * Check if request is allowed under rate limit
12
+ */
13
+ checkLimit(identifier, limit) {
14
+ const now = Date.now();
15
+ const windowStart = now - this.windowMs;
16
+ // Get or create entry
17
+ let entry = this.store.get(identifier);
18
+ if (!entry) {
19
+ entry = { timestamps: [] };
20
+ this.store.set(identifier, entry);
21
+ }
22
+ // Remove timestamps outside the window
23
+ entry.timestamps = entry.timestamps.filter(ts => ts > windowStart);
24
+ // Check if limit exceeded
25
+ if (entry.timestamps.length >= limit) {
26
+ const oldestTimestamp = entry.timestamps[0];
27
+ const retryAfter = Math.ceil((oldestTimestamp + this.windowMs - now) / 1000);
28
+ return {
29
+ allowed: false,
30
+ remaining: 0,
31
+ retryAfter,
32
+ };
33
+ }
34
+ // Add current timestamp
35
+ entry.timestamps.push(now);
36
+ return {
37
+ allowed: true,
38
+ remaining: limit - entry.timestamps.length,
39
+ };
40
+ }
41
+ /**
42
+ * Clean up old entries (call periodically)
43
+ */
44
+ cleanup() {
45
+ const now = Date.now();
46
+ const windowStart = now - this.windowMs;
47
+ for (const [identifier, entry] of this.store.entries()) {
48
+ entry.timestamps = entry.timestamps.filter(ts => ts > windowStart);
49
+ if (entry.timestamps.length === 0) {
50
+ this.store.delete(identifier);
51
+ }
52
+ }
53
+ }
54
+ }
55
+ /**
56
+ * Hourly burst limits per tier.
57
+ * These are the hard caps enforced by the in-memory sliding window.
58
+ * Free: 25/hr, Pro: 100/hr, Max: 500/hr (matches pricing page).
59
+ */
60
+ const TIER_BURST_LIMITS = {
61
+ free: 25,
62
+ starter: 50,
63
+ pro: 100,
64
+ enterprise: 250,
65
+ max: 500,
66
+ };
67
+ export function createRateLimitMiddleware(limiter) {
68
+ return (req, res, next) => {
69
+ try {
70
+ // Use API key or real client IP as identifier.
71
+ // Prefer Cloudflare CF-Connecting-IP, then x-forwarded-for first
72
+ // entry (real client), then x-real-ip, then req.ip.
73
+ const forwardedFor = req.headers['x-forwarded-for'];
74
+ const firstForwardedIp = typeof forwardedFor === 'string'
75
+ ? forwardedFor.split(',')[0].trim()
76
+ : Array.isArray(forwardedFor) ? forwardedFor[0] : undefined;
77
+ const clientIp = req.headers['cf-connecting-ip']
78
+ || firstForwardedIp
79
+ || req.headers['x-real-ip']
80
+ || req.ip
81
+ || 'unknown';
82
+ const identifier = req.auth?.keyInfo?.key || clientIp;
83
+ // Use tier-based hourly burst limits (matches the 1-hour sliding window)
84
+ const limit = TIER_BURST_LIMITS[req.auth?.tier || 'free'] || 25;
85
+ const result = limiter.checkLimit(identifier, limit);
86
+ // Calculate reset timestamp
87
+ const now = Date.now();
88
+ const resetTimestamp = Math.ceil((now + limiter['windowMs']) / 1000);
89
+ // Set rate limit headers on ALL responses
90
+ res.setHeader('X-RateLimit-Limit', limit.toString());
91
+ res.setHeader('X-RateLimit-Remaining', Math.max(0, result.remaining).toString());
92
+ res.setHeader('X-RateLimit-Reset', resetTimestamp.toString());
93
+ // Add plan header if authenticated
94
+ if (req.auth?.tier) {
95
+ res.setHeader('X-WebPeel-Plan', req.auth.tier);
96
+ }
97
+ if (!result.allowed) {
98
+ res.setHeader('Retry-After', result.retryAfter.toString());
99
+ const tier = req.auth?.tier || 'free';
100
+ const upgradeHint = tier === 'free'
101
+ ? ' Upgrade to Pro ($9/mo) for 100/hr burst limit → https://webpeel.dev/#pricing'
102
+ : tier === 'pro'
103
+ ? ' Upgrade to Max ($29/mo) for 500/hr burst limit → https://webpeel.dev/#pricing'
104
+ : '';
105
+ res.status(429).json({
106
+ error: 'rate_limited',
107
+ message: `Hourly rate limit exceeded (${limit} requests/hr on ${tier} plan). Try again in ${result.retryAfter}s.${upgradeHint}`,
108
+ retryAfter: result.retryAfter,
109
+ limit,
110
+ plan: tier,
111
+ docs: 'https://webpeel.dev/docs/api-reference#rate-limits',
112
+ });
113
+ return; // Stop processing - rate limit exceeded
114
+ }
115
+ next();
116
+ }
117
+ catch (error) {
118
+ const err = error;
119
+ res.status(500).json({
120
+ error: 'rate_limit_error',
121
+ message: err.message || 'Rate limiting failed',
122
+ });
123
+ }
124
+ };
125
+ }
126
+ //# sourceMappingURL=rate-limit.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rate-limit.js","sourceRoot":"","sources":["../../../src/server/middleware/rate-limit.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,MAAM,OAAO,WAAW;IACd,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IAC1C,QAAQ,CAAS;IAEzB,YAAY,WAAmB,KAAK;QAClC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,UAAkB,EAAE,KAAa;QAK1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QAExC,sBAAsB;QACtB,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC;QAED,uCAAuC;QACvC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC;QAEnE,0BAA0B;QAC1B,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;YACrC,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;YAE7E,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,CAAC;gBACZ,UAAU;aACX,CAAC;QACJ,CAAC;QAED,wBAAwB;QACxB,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE3B,OAAO;YACL,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM;SAC3C,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,OAAO;QACL,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QAExC,KAAK,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC;YACnE,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,iBAAiB,GAA2B;IAChD,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,EAAE;IACX,GAAG,EAAE,GAAG;IACR,UAAU,EAAE,GAAG;IACf,GAAG,EAAE,GAAG;CACT,CAAC;AAEF,MAAM,UAAU,yBAAyB,CAAC,OAAoB;IAC5D,OAAO,CAAC,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAQ,EAAE;QAC/D,IAAI,CAAC;YACH,+CAA+C;YAC/C,iEAAiE;YACjE,oDAAoD;YACpD,MAAM,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACpD,MAAM,gBAAgB,GAAG,OAAO,YAAY,KAAK,QAAQ;gBACvD,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;gBACnC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAE9D,MAAM,QAAQ,GAAI,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAY;mBACvD,gBAAgB;mBACf,GAAG,CAAC,OAAO,CAAC,WAAW,CAAY;mBACpC,GAAG,CAAC,EAAE;mBACN,SAAS,CAAC;YACf,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,QAAQ,CAAC;YAEtD,yEAAyE;YACzE,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAEhE,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAErD,4BAA4B;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;YAErE,0CAA0C;YAC1C,GAAG,CAAC,SAAS,CAAC,mBAAmB,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrD,GAAG,CAAC,SAAS,CAAC,uBAAuB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjF,GAAG,CAAC,SAAS,CAAC,mBAAmB,EAAE,cAAc,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE9D,mCAAmC;YACnC,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;gBACnB,GAAG,CAAC,SAAS,CAAC,gBAAgB,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,GAAG,CAAC,SAAS,CAAC,aAAa,EAAE,MAAM,CAAC,UAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC5D,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC;gBACtC,MAAM,WAAW,GAAG,IAAI,KAAK,MAAM;oBACjC,CAAC,CAAC,+EAA+E;oBACjF,CAAC,CAAC,IAAI,KAAK,KAAK;wBAChB,CAAC,CAAC,gFAAgF;wBAClF,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,cAAc;oBACrB,OAAO,EAAE,+BAA+B,KAAK,mBAAmB,IAAI,wBAAwB,MAAM,CAAC,UAAU,KAAK,WAAW,EAAE;oBAC/H,UAAU,EAAE,MAAM,CAAC,UAAU;oBAC7B,KAAK;oBACL,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE,oDAAoD;iBAC3D,CAAC,CAAC;gBACH,OAAO,CAAC,wCAAwC;YAClD,CAAC;YAED,IAAI,EAAE,CAAC;QACT,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAc,CAAC;YAC3B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,sBAAsB;aAC/C,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}
@@ -0,0 +1,16 @@
1
+ /**
2
+ * URL validation middleware to prevent SSRF attacks
3
+ * Validates URLs BEFORE any network request is made
4
+ */
5
+ /**
6
+ * Validate URL to prevent SSRF attacks
7
+ * Blocks localhost, private IPs, link-local addresses, and non-HTTP(S) protocols
8
+ */
9
+ export declare function validateUrlForSSRF(urlString: string): void;
10
+ /**
11
+ * SSRF Error class
12
+ */
13
+ export declare class SSRFError extends Error {
14
+ constructor(message: string);
15
+ }
16
+ //# sourceMappingURL=url-validator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../../src/server/middleware/url-validator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAgC1D;AAED;;GAEG;AACH,qBAAa,SAAU,SAAQ,KAAK;gBACtB,OAAO,EAAE,MAAM;CAI5B"}