pi-unsloth-webtools 0.3.1 → 0.4.1

package/web-access.ts

@@ -6,7 +6,13 @@ const MAX_CACHEABLE_DOMAIN_LEN = 253;
 const SITE_FILTER_LIMIT = 8;
 const DOTTED_HOST_RE = /^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+$/;
 const PORT_RE = /^[0-9]{1,5}$/;
+export const MAX_SIGNAL_TIMEOUT_MS = 2 ** 31 - 1;
 
+function throwInvalidDomain(value: unknown): never {
+  throw new Error(`Invalid website domain: ${String(value)}`);
+}
+
+const INVALID_HOST_REASON = "Blocked: the URL has an invalid hostname or port.";
 export interface WebsitePolicy {
   allowedDomains: string[];
   blockedDomains: string[];
@@ -19,18 +25,19 @@ export function normalizeDomain(value: unknown): string {
     [...domain].some((char) => char.charCodeAt(0) < 32) ||
     ["\\", "/", "@", "?", "#"].some((char) => domain.includes(char))
   ) {
-    throw new Error(`Invalid website domain: ${String(value)}`);
+    throwInvalidDomain(value);
   }
-  const bracketed = domain.startsWith("[") && domain.endsWith("]");
-  if (domain.startsWith("[") !== domain.endsWith("]")) {
-    throw new Error(`Invalid website domain: ${String(value)}`);
+  const startsBracket = domain.startsWith("[");
+  const endsBracket = domain.endsWith("]");
+  if (startsBracket !== endsBracket) {
+    throwInvalidDomain(value);
   }
-  const stripped = (bracketed ? domain.slice(1, -1) : domain).replace(/\.+$/, "");
+  const stripped = (startsBracket ? domain.slice(1, -1) : domain).replace(/\.+$/, "");
   if (/^[0-9a-fA-F:.]+$/.test(stripped) && stripped.includes(":")) {
     try {
       return compressIpv6(stripped);
     } catch {
-      throw new Error(`Invalid website domain: ${String(value)}`);
+      throwInvalidDomain(value);
     }
   }
   if (stripped.includes(":")) {
@@ -52,13 +59,13 @@ export function normalizeDomain(value: unknown): string {
   try {
     asciiDomain = domainToASCII(stripped).toLowerCase();
   } catch {
-    throw new Error(`Invalid website domain: ${String(value)}`);
+    throwInvalidDomain(value);
   }
   if (
     asciiDomain.length > 253 ||
     !asciiDomain.split(".").every((label) => DOMAIN_LABEL_RE.test(label))
   ) {
-    throw new Error(`Invalid website domain: ${String(value)}`);
+    throwInvalidDomain(value);
   }
   return asciiDomain;
 }
@@ -194,7 +201,7 @@ export function checkUrlAccess(
   try {
     parsed = new URL(candidate);
   } catch {
-    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
+    return [false, INVALID_HOST_REASON, ""];
   }
   const scheme = parsed.protocol.replace(/:$/, "").toLowerCase();
   if (scheme !== "http" && scheme !== "https") {
@@ -204,20 +211,16 @@ export function checkUrlAccess(
     return [false, "Blocked: URLs with credentials or encoded hostnames are not allowed.", ""];
   }
   if (!parsed.hostname) {
-    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
+    return [false, INVALID_HOST_REASON, ""];
   }
-  try {
-    if (parsed.port && !(PORT_RE.test(parsed.port) && Number(parsed.port) >= 1 && Number(parsed.port) <= 65535)) {
-      return [false, "Blocked: the URL has an invalid hostname or port.", ""];
-    }
-  } catch {
-    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
+  if (parsed.port && !(PORT_RE.test(parsed.port) && Number(parsed.port) >= 1 && Number(parsed.port) <= 65535)) {
+    return [false, INVALID_HOST_REASON, ""];
   }
   let hostname: string;
   try {
     hostname = normalizeDomain(parsed.hostname);
   } catch {
-    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
+    return [false, INVALID_HOST_REASON, ""];
   }
   if (!policyAllows(hostname, policy)) {
     return [false, `Blocked: the website access policy disallows ${hostname}.`, hostname];
@@ -323,7 +326,7 @@ const GITHUB_NON_OWNER_SEGMENTS = new Set([
 
 const GITHUB_NAME_RE = /^[A-Za-z0-9_.\-]{1,100}$/;
 
-function githubRepoOwnerRepo(url: string): [string, string] | null {
+function parseGithubRepo(url: string): { owner: string; repo: string; rest: string[] } | null {
   let parsed: URL;
   try {
     parsed = new URL(url);
@@ -333,22 +336,40 @@ function githubRepoOwnerRepo(url: string): [string, string] | null {
   const host = (parsed.hostname ?? "").toLowerCase();
   if (host !== "github.com" && host !== "www.github.com") return null;
   const parts = parsed.pathname.split("/").filter((part) => part.length > 0);
-  if (parts.length !== 2) return null;
-  const [owner, repo] = parts;
+  if (parts.length < 2) return null;
+  const owner = parts[0];
+  const repoRaw = parts[1];
   if (GITHUB_NON_OWNER_SEGMENTS.has(owner.toLowerCase())) return null;
-  const cleanRepo = repo.endsWith(".git") ? repo.slice(0, -4) : repo;
-  if (!GITHUB_NAME_RE.test(owner) || !GITHUB_NAME_RE.test(cleanRepo)) return null;
-  return [owner, cleanRepo];
+  const repo = repoRaw.endsWith(".git") ? repoRaw.slice(0, -4) : repoRaw;
+  if (!GITHUB_NAME_RE.test(owner) || !GITHUB_NAME_RE.test(repo)) return null;
+  return { owner, repo, rest: parts.slice(2) };
+}
+function githubRepoRoot(url: string): { owner: string; repo: string; rest: string[] } | null {
+  const parsed = parseGithubRepo(url);
+  if (!parsed || parsed.rest.length !== 0) return null;
+  return parsed;
 }
 
 export function githubRepoReadmeApiUrl(url: string): string | null {
-  const pair = githubRepoOwnerRepo(url);
-  return pair ? `https://api.github.com/repos/${pair[0]}/${pair[1]}/readme` : null;
+  const parsed = githubRepoRoot(url);
+  if (!parsed) return null;
+  return `https://api.github.com/repos/${parsed.owner}/${parsed.repo}/readme`;
 }
 
 export function githubRepoRawReadmeUrl(url: string): string | null {
-  const pair = githubRepoOwnerRepo(url);
-  return pair ? `https://raw.githubusercontent.com/${pair[0]}/${pair[1]}/HEAD/README.md` : null;
+  const parsed = githubRepoRoot(url);
+  if (!parsed) return null;
+  return `https://raw.githubusercontent.com/${parsed.owner}/${parsed.repo}/HEAD/README.md`;
+}
+export function githubRawContentUrl(url: string): string | null {
+  const parsed = parseGithubRepo(url);
+  if (!parsed || parsed.rest.length < 2) return null;
+  const [kind, ref] = parsed.rest;
+  if (kind !== "blob" && kind !== "raw") return null;
+  if (!ref) return null;
+  const filePath = parsed.rest.slice(2).join("/");
+  if (!filePath) return null;
+  return `https://raw.githubusercontent.com/${parsed.owner}/${parsed.repo}/${ref}/${filePath}`;
 }
 
 function ipv4Octets(ip: string): number[] | null {