dscli-agent 0.1.0__py3-none-any.whl

dscli/scraper.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+# JavaScript evaluated inside the page. Kept as plain strings so the selectors
+# can be changed from config without editing this file.
+
+EXTRACT_LAST_ANSWER_JS = r"""
+(sel) => {
+  const nodes = Array.from(document.querySelectorAll(sel));
+  const el = nodes.length ? nodes[nodes.length - 1] : null;
+  if (!el) return null;
+
+  const SKIP = new Set(['BUTTON','SVG','STYLE','SCRIPT','TEXTAREA','INPUT','NOSCRIPT']);
+  const classOf = (n) => (n.getAttribute && n.getAttribute('class')) || '';
+  const skip = (n) => {
+    if (n.nodeType !== 1) return false;
+    if (SKIP.has(n.tagName)) return true;
+    if (n.matches('[role="button"], .ds-button, .ds-icon, [aria-hidden="true"]')) return true;
+    return false;
+  };
+
+  const languageOf = (pre) => {
+    const code = pre.querySelector('code');
+    const cls = code ? String(code.className.baseVal || code.className || '') : '';
+    const m = cls.match(/language-([\w.+#-]+)/);
+    if (m) return m[1];
+    let cur = pre.parentElement;
+    for (let i = 0; cur && i < 3; i++, cur = cur.parentElement) {
+      if (/code/i.test(classOf(cur))) {
+        const cand = cur.querySelector('[class*="banner"], [class*="header"], [class*="title"]');
+        if (cand) {
+          const t = (cand.innerText || '').trim().split(/\s+/)[0];
+          if (t) return t.slice(0, 20);
+        }
+      }
+    }
+    return '';
+  };
+
+  const codeBlock = (pre) => {
+    const code = pre.querySelector('code') || pre;
+    const text = (code.textContent || '').replace(/\n+$/, '');
+    return '\n```' + languageOf(pre) + '\n' + text + '\n```\n';
+  };
+
+  const walk = (node) => {
+    let out = '';
+    for (const child of node.childNodes) {
+      if (child.nodeType === 3) { out += child.textContent; continue; }
+      if (child.nodeType !== 1 || skip(child)) continue;
+      const tag = child.tagName;
+      const cls = classOf(child);
+
+      if (tag === 'BR') { out += '\n'; continue; }
+      if (tag === 'PRE') { out += codeBlock(child); continue; }
+      if (tag === 'HR') { out += '\n---\n'; continue; }
+      if (tag === 'IMG') {
+        const alt = child.getAttribute('alt') || '';
+        out += alt ? `\n![${alt}](${child.getAttribute('src') || ''})\n` : '\n';
+        continue;
+      }
+      if (tag === 'A') {
+        const inner = walk(child).trim();
+        const href = child.getAttribute('href') || '';
+        let suffix = '';
+        // Append a URL only for absolute, explicitly linked URLs. Markdown also
+        // turns JSON fragments into links: DeepSeek autolinks tokens such as
+        // object.is or ".top", and parses [i](x) into a relative href. Appending
+        // those hrefs corrupts the code the model wrote.
+        const absolute = /^(?:https?:)?\/\//i.test(href);
+        if (absolute && !href.startsWith('#')) {
+          let host = '';
+          try {
+            host = new URL(href, location.href).hostname.replace(/^www\./i, '');
+          } catch (e) { host = ''; }
+          const hostLower = host.toLowerCase();
+          const bare = inner.replace(/^https?:\/\//i, '').replace(/\/+$/, '').toLowerCase();
+          const hrefBare = href.replace(/^https?:\/\//i, '').replace(/\/+$/, '').toLowerCase();
+          const autolink = !!hostLower && (
+            bare === hostLower ||
+            bare === hrefBare ||
+            (bare.charAt(0) === '.' && hostLower.endsWith(bare))
+          );
+          if (!autolink) suffix = ` (${href})`;
+        }
+        out += inner + suffix;
+        continue;
+      }
+      if (/^H[1-6]$/.test(tag)) {
+        out += '\n' + '#'.repeat(Number(tag[1])) + ' ' + walk(child).trim() + '\n';
+        continue;
+      }
+      if (tag === 'LI') {
+        out += '\n- ' + walk(child).trim();
+        continue;
+      }
+      if (tag === 'TABLE') {
+        const rows = Array.from(child.querySelectorAll('tr'));
+        for (const row of rows) {
+          const cells = Array.from(row.querySelectorAll('th,td'))
+            .map((c) => (c.innerText || '').trim().replace(/\s+/g, ' '));
+          out += '\n| ' + cells.join(' | ') + ' |';
+        }
+        out += '\n';
+        continue;
+      }
+
+      // DeepSeek wraps code in a banner + <pre>. Emit only the code so the
+      // "Copy" button and language label do not pollute the model's JSON.
+      if (tag !== 'PRE' && /code[-_]?block/i.test(cls) && child.querySelector('pre')) {
+        child.querySelectorAll('pre').forEach((pre) => { out += codeBlock(pre); });
+        continue;
+      }
+      if (/banner|toolbar/i.test(cls) && child.closest('pre, [class*="code-block"], [class*="codeblock"]')) {
+        continue;
+      }
+
+      const inner = walk(child);
+      const block = new Set([
+        'P','DIV','SECTION','ARTICLE','BLOCKQUOTE','UL','OL','DL','DT','DD',
+        'FIGURE','FIGCAPTION','DETAILS','SUMMARY'
+      ]).has(tag);
+      out += block ? '\n' + inner + '\n' : inner;
+    }
+    return out;
+  };
+
+  let text = walk(el);
+  text = text.replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n').trim();
+  return { text: text, count: nodes.length, htmlLength: el.innerHTML.length };
+}
+"""
+
+
+SNAPSHOT_JS = r"""
+(args) => {
+  const assistantSel = args.assistant;
+  const itemSel = args.item || '[data-virtual-list-item-key]';
+  const sendSel = args.send;
+  const stopSel = args.stop;
+  const composerSel = args.composer;
+
+  const visible = (el) => {
+    if (!el) return false;
+    const style = getComputedStyle(el);
+    if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return false;
+    const r = el.getBoundingClientRect();
+    return r.width > 0 && r.height > 0;
+  };
+
+  const msgs = Array.from(document.querySelectorAll(assistantSel));
+  const last = msgs.length ? msgs[msgs.length - 1] : null;
+  let key = null;
+  let preview = '';
+  let hasActions = false;
+  if (last) {
+    const item = last.closest(itemSel);
+    key = item ? item.getAttribute('data-virtual-list-item-key') : null;
+    preview = (last.innerText || '').trim().slice(0, 160);
+    const box = item || last.parentElement;
+    hasActions = !!(box && box.querySelector(
+      '[aria-label="Read aloud"], [aria-label*="read aloud" i], [aria-label*="Read Aloud" i]'
+    ));
+  }
+
+  const composer = composerSel ? document.querySelector(composerSel) : null;
+  const empty = !composer || !(composer.value || '').trim();
+  const send = sendSel ? document.querySelector(sendSel) : null;
+  const sendDisabled = !send || send.classList.contains('ds-button--disabled');
+  let stop = false;
+  if (stopSel) {
+    stop = Array.from(document.querySelectorAll(stopSel)).some(visible);
+  }
+
+  return {
+    count: msgs.length,
+    key: key,
+    preview: preview,
+    seen: last ? last.hasAttribute('data-dscli-seen') : false,
+    hasActions: hasActions,
+    composerEmpty: empty,
+    sendPresent: !!send,
+    sendDisabled: sendDisabled,
+    stop: stop,
+    url: location.href,
+  };
+}
+"""
+
+
+MARK_JS = r"""
+(args) => {
+  const msgs = Array.from(document.querySelectorAll(args.assistant));
+  msgs.forEach((m) => m.setAttribute('data-dscli-seen', '1'));
+  return { count: msgs.length };
+}
+"""
+
+
+SCROLL_TO_BOTTOM_JS = r"""
+() => {
+  const anchor = document.querySelector('[data-virtual-list-item-key]');
+  let el = anchor;
+  while (el && el !== document.body && el !== document.documentElement) {
+    const style = getComputedStyle(el);
+    if (el.scrollHeight > el.clientHeight + 20 && /(auto|scroll)/.test(style.overflowY)) {
+      el.scrollTop = el.scrollHeight;
+      return true;
+    }
+    el = el.parentElement;
+  }
+  window.scrollTo(0, document.body.scrollHeight);
+  return false;
+}
+"""
+
+
+TOASTS_JS = r"""
+(sel) => {
+  const visible = (el) => {
+    const style = getComputedStyle(el);
+    if (style.display === 'none' || style.visibility === 'hidden') return false;
+    const r = el.getBoundingClientRect();
+    return r.width > 0 && r.height > 0;
+  };
+  return Array.from(document.querySelectorAll(sel))
+    .filter(visible)
+    .map((n) => (n.innerText || '').trim())
+    .filter(Boolean)
+    .slice(-5);
+}
+"""
+
+
+DIAGNOSTICS_JS = r"""
+(args) => {
+  const present = (sel) => {
+    try { return document.querySelectorAll(sel).length; } catch (e) { return -1; }
+  };
+  const visible = (sel) => {
+    try {
+      const el = document.querySelector(sel);
+      if (!el) return false;
+      const style = getComputedStyle(el);
+      const r = el.getBoundingClientRect();
+      return style.display !== 'none' && style.visibility !== 'hidden' && r.width > 0 && r.height > 0;
+    } catch (e) { return false; }
+  };
+  const fileInput = document.querySelector(args.fileInput);
+  const send = document.querySelector(args.send);
+  return {
+    url: location.href,
+    title: document.title,
+    composerCount: present(args.composer),
+    composerVisible: visible(args.composer),
+    sendCount: present(args.send),
+    sendVisible: visible(args.send),
+    sendDisabled: !!(send && send.classList.contains('ds-button--disabled')),
+    assistantCount: present(args.assistant),
+    fileInputCount: present(args.fileInput),
+    accept: fileInput ? (fileInput.getAttribute('accept') || '') : '',
+    cloudflareVisible: visible(args.cloudflare),
+    notificationText: Array.from(document.querySelectorAll(args.notification))
+      .map((n) => (n.innerText || '').trim()).filter(Boolean).slice(-3),
+    toggleCount: present(args.toggle),
+    textareaValueLength: (document.querySelector(args.composer) || {}).value ? document.querySelector(args.composer).value.length : 0,
+  };
+}
+"""