summit-search-cli 1.0.0

package/README.md

@@ -0,0 +1,48 @@
+# Search CLI
+
+Search CLI is a terminal-based web search and information retrieval tool designed to bring useful answers directly into your command line.
+
+Instead of only returning a list of links, Search searches the web, reads available sources, extracts important information, removes unnecessary page content, and ranks useful sentences to create a cleaner human-readable response.
+
+Search is built by **Summit** as part of its collection of developer tools focused on making powerful workflows available directly from the terminal.
+
+## Why Search Exists
+
+Traditional command-line search tools usually stop at showing URLs. While that can be useful, it often requires opening multiple browser tabs, reading through pages, and manually finding the important information.
+
+Search was created to reduce that process by bringing the information itself into the terminal.
+
+The goal is not to replace the web, but to make finding and understanding information faster.
+
+## Features
+
+- 🔎 DuckDuckGo-powered web searching
+- 🌐 Browserless information lookup
+- 📄 Automatic web page content extraction
+- 🧹 Removes unnecessary website clutter
+- 🧠 Smart sentence ranking system
+- ✨ Duplicate information removal
+- 📚 Combines information from multiple sources
+- 🎨 Clean colored terminal output
+- ⚡ Designed for fast command-line workflows
+
+## How It Works
+
+Search uses a multi-step pipeline:
+
+```text
+User Query
+    ↓
+DuckDuckGo Search
+    ↓
+Source Collection
+    ↓
+Web Page Fetching
+    ↓
+Content Extraction
+    ↓
+Text Cleaning
+    ↓
+Sentence Ranking
+    ↓
+Readable Terminal Response

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "summit-search-cli",
+  "version": "1.0.0",
+  "description": "A terminal-based search CLI that uses DuckDuckGo to find and display web results directly from the command line.",
+  "keywords": [
+    "search",
+    "cli",
+    "terminal",
+    "command-line",
+    "web-search",
+    "duckduckgo",
+    "browserless",
+    "developer-tools",
+    "productivity",
+    "utility",
+    "nodejs",
+    "npm",
+    "automation",
+    "research",
+    "documentation",
+    "ai-tools",
+    "bashlife",
+    "summit"
+  ],
+  "license": "GPL-2.0",
+  "author": "summit",
+  "type": "module",
+  "main": "src/index.js",
+  "bin": {
+    "search": "./src/index.js"
+  },
+  "scripts": {
+    "start": "node src/index.js"
+  },
+  "dependencies": {
+    "chalk": "^5.6.2",
+    "cheerio": "^1.2.0"
+  }
+}

package/src/duckduckgo.js

@@ -0,0 +1,88 @@
+import https from "https";
+
+function request(url) {
+  return new Promise((resolve, reject) => {
+    https.get(
+      url,
+      {
+        headers: {
+          "User-Agent": "Mozilla/5.0"
+        }
+      },
+      (res) => {
+        let data = "";
+
+        res.on("data", chunk => {
+          data += chunk;
+        });
+
+        res.on("end", () => {
+          resolve(data);
+        });
+      }
+    ).on("error", reject);
+  });
+}
+
+
+function decodeUrl(url) {
+  try {
+    const parsed = new URL(
+      url.startsWith("http")
+        ? url
+        : "https:" + url
+    );
+
+    const target = parsed.searchParams.get("uddg");
+
+    if (target) {
+      return decodeURIComponent(target);
+    }
+
+    return url;
+
+  } catch {
+    return url;
+  }
+}
+
+
+export async function duckSearch(query) {
+
+  const url =
+    "https://html.duckduckgo.com/html/?q=" +
+    encodeURIComponent(query);
+
+
+  const html = await request(url);
+
+
+  const results = [];
+
+
+  const regex =
+    /result__a[^>]*href="([^"]+)"[^>]*>(.*?)<\/a>/g;
+
+
+  let match;
+
+
+  while (
+    (match = regex.exec(html)) &&
+    results.length < 5
+  ) {
+
+    results.push({
+      title: match[2]
+        .replace(/<[^>]+>/g, "")
+        .replace(/&amp;/g, "&")
+        .trim(),
+
+      url: decodeUrl(match[1])
+    });
+
+  }
+
+
+  return results;
+}

package/src/extract.js

@@ -0,0 +1,62 @@
+import * as cheerio from "cheerio";
+
+export function extractText(html) {
+  const $ = cheerio.load(html);
+
+
+  $(
+    "script, style, nav, header, footer, aside, " +
+    "button, form, svg, noscript, " +
+    "table, sup, .reference, .mw-editsection, " +
+    ".infobox, .sidebar, .metadata"
+  ).remove();
+
+
+  let paragraphs = [];
+
+
+  $("p").each((_, el) => {
+    const text = $(el)
+      .text()
+      .replace(/\s+/g, " ")
+      .trim();
+
+
+    if (text.length > 50) {
+      paragraphs.push(text);
+    }
+  });
+
+
+  let text = paragraphs.join(" ");
+
+
+  text = text
+    .replace(/\[\d+\]/g, "")
+    .replace(/ⓘ/g, "")
+    .replace(/\([^)]*citation[^)]*\)/gi, "")
+    .replace(/\s+/g, " ")
+    .trim();
+
+
+  const junk = [
+    "skip to content",
+    "cookie policy",
+    "privacy policy",
+    "terms of service",
+    "sign in",
+    "log in",
+    "subscribe"
+  ];
+
+
+  for (const item of junk) {
+    text = text.replace(
+      new RegExp(item, "gi"),
+      ""
+    );
+  }
+
+
+  return text;
+}

package/src/fetch.js

@@ -0,0 +1,25 @@
+import https from "https";
+
+export function fetchPage(url) {
+  return new Promise((resolve, reject) => {
+    https.get(
+      url,
+      {
+        headers: {
+          "User-Agent": "Mozilla/5.0"
+        }
+      },
+      res => {
+        let data = "";
+
+        res.on("data", chunk => {
+          data += chunk;
+        });
+
+        res.on("end", () => {
+          resolve(data);
+        });
+      }
+    ).on("error", reject);
+  });
+}

package/src/generator.js

@@ -0,0 +1,331 @@
+import chalk from "chalk";
+
+function cleanSentence(sentence) {
+  return sentence
+    .replace(/\[\d+\]/g, "")
+    .replace(/ⓘ/g, "")
+    .replace(/US\$\d+[\w,.]*/gi, "")
+    .replace(/\$\d+[\w,.]*/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+
+function removeDuplicates(sentences) {
+  const seen = new Set();
+
+  return sentences.filter(sentence => {
+    const key = sentence
+      .toLowerCase()
+      .replace(/[^a-z0-9]/g, "")
+      .slice(0, 120);
+
+    if (seen.has(key)) {
+      return false;
+    }
+
+    seen.add(key);
+    return true;
+  });
+}
+
+
+function getIntent(query) {
+  const q = query.toLowerCase();
+
+  if (
+    q.includes("history") ||
+    q.includes("founded") ||
+    q.includes("created")
+  ) {
+    return "history";
+  }
+
+  if (
+    q.includes("lawsuit") ||
+    q.includes("legal") ||
+    q.includes("sued")
+  ) {
+    return "legal";
+  }
+
+  if (
+    q.includes("revenue") ||
+    q.includes("money") ||
+    q.includes("worth")
+  ) {
+    return "business";
+  }
+
+  return "general";
+}
+
+
+function scoreSentence(sentence, keywords, query) {
+
+  const lower = sentence.toLowerCase();
+
+  let score = 0;
+
+  const subject =
+    keywords.join(" ");
+
+
+  // Query matching
+  for (const word of keywords) {
+    if (lower.includes(word)) {
+      score += 3;
+    }
+  }
+
+
+  // Exact subject definition
+  if (
+    lower.startsWith(subject + " is ") ||
+    lower.startsWith(subject + " was ")
+  ) {
+    score += 25;
+  }
+
+
+  // Explanatory sentences
+  const explanation = [
+    " is a ",
+    " is an ",
+    " allows users ",
+    " allows people ",
+    " lets users ",
+    " provides ",
+    " offers ",
+    " used to ",
+    " designed to "
+  ];
+
+  for (const pattern of explanation) {
+    if (lower.includes(pattern)) {
+      score += 8;
+    }
+  }
+
+
+  // History
+  if (
+    lower.includes("was founded") ||
+    lower.includes("was created") ||
+    lower.includes("was developed")
+  ) {
+    score += 5;
+  }
+
+
+  // Product/features of the main thing
+  const relatedProducts = [
+    "youtube kids",
+    "youtube music",
+    "youtube movies",
+    "youtube premium",
+    "youtube play buttons",
+    "youtube tv"
+  ];
+
+  for (const product of relatedProducts) {
+    if (
+      lower.includes(product) &&
+      !lower.startsWith(subject)
+    ) {
+      score -= 10;
+    }
+  }
+
+
+  // Metadata
+  const metadata = [
+    "founders",
+    "key people",
+    "industry",
+    "revenue",
+    "headquartered",
+    "headquarters",
+    "employees",
+    "ceo",
+    "written in",
+    "programming language",
+    "stock"
+  ];
+
+  for (const word of metadata) {
+    if (lower.includes(word)) {
+      score -= 15;
+    }
+  }
+
+
+  const intent = getIntent(query);
+
+
+  // Only punish these if user did not ask
+  if (intent !== "legal") {
+    if (
+      lower.includes("lawsuit") ||
+      lower.includes("sued") ||
+      lower.includes("court")
+    ) {
+      score -= 20;
+    }
+  }
+
+
+  if (intent !== "history") {
+    if (
+      lower.includes("history") ||
+      lower.includes("timeline")
+    ) {
+      score -= 10;
+    }
+  }
+
+
+  if (intent !== "business") {
+    if (
+      lower.includes("revenue") ||
+      lower.includes("profit")
+    ) {
+      score -= 10;
+    }
+  }
+
+
+  // Remove opinion/marketing sentences
+  const weak = [
+    "in conclusion",
+    "incredibly popular",
+    "act of courage",
+    "vision",
+    "voice",
+    "work with the world"
+  ];
+
+  for (const word of weak) {
+    if (lower.includes(word)) {
+      score -= 15;
+    }
+  }
+
+
+  // Length preference
+  if (
+    sentence.length >= 70 &&
+    sentence.length <= 220
+  ) {
+    score += 5;
+  }
+
+
+  if (sentence.length > 300) {
+    score -= 5;
+  }
+
+
+  return score;
+}
+
+
+function createTitle(query) {
+  return query
+    .replace(/^whats?\s+/i, "")
+    .split(" ")
+    .map(word =>
+      word.charAt(0).toUpperCase() +
+      word.slice(1)
+    )
+    .join(" ");
+}
+
+
+export function generate(results, query = "") {
+
+  const keywords =
+    query
+      .toLowerCase()
+      .replace(/^whats?\s+/i, "")
+      .split(/\s+/)
+      .filter(Boolean);
+
+
+  let sentences = [];
+
+
+  for (const result of results) {
+
+    if (!result.text) continue;
+
+
+    const parts =
+      result.text
+        .split(/[.!?]/)
+        .map(cleanSentence)
+        .filter(sentence =>
+          sentence.length > 40
+        );
+
+
+    sentences.push(...parts);
+  }
+
+
+  sentences =
+    removeDuplicates(sentences);
+
+
+  const ranked =
+    sentences
+      .map(sentence => ({
+        sentence,
+        score: scoreSentence(
+          sentence,
+          keywords,
+          query
+        )
+      }))
+      .sort(
+        (a, b) =>
+          b.score - a.score
+      );
+
+
+  const selected =
+    ranked
+      .slice(0, 5)
+      .map(item => item.sentence);
+
+
+  console.log();
+
+  console.log(
+    chalk.bold.cyan(
+      createTitle(query)
+    )
+  );
+
+  console.log();
+
+
+  if (!selected.length) {
+    console.log(
+      chalk.yellow(
+        "No useful information found."
+      )
+    );
+
+    return;
+  }
+
+
+  console.log(
+    selected
+      .map(sentence =>
+        chalk.white("• " + sentence)
+      )
+      .join("\n\n")
+  );
+}

package/src/index.js

@@ -0,0 +1,44 @@
+#!/usr/bin/env node
+
+import { duckSearch } from "./duckduckgo.js";
+import { fetchPage } from "./fetch.js";
+import { extractText } from "./extract.js";
+import { generate } from "./generator.js";
+
+const query = process.argv.slice(2).join(" ");
+
+if (!query) {
+  console.log(`
+Search CLI
+
+Usage:
+  search <query>
+`);
+  process.exit(0);
+}
+
+console.log("Searching...\n");
+
+const results = await duckSearch(query);
+
+if (!results.length) {
+  console.log("No results found.");
+  process.exit(0);
+}
+
+console.log(
+  `Found ${results.length} results. Search is Thinking...\n`
+);
+
+for (const result of results) {
+  try {
+    const html = await fetchPage(result.url);
+
+    result.text = extractText(html);
+
+  } catch (error) {
+    result.text = "";
+  }
+}
+
+generate(results, query);

package/src/output.js

@@ -0,0 +1,22 @@
+export function output(results) {
+
+  if (!results.length) {
+    console.log("No results found.");
+    return;
+  }
+
+
+  console.log("\nResults:\n");
+
+
+  results.forEach((result, i) => {
+
+    console.log(
+`${i + 1}. ${result.title}
+${result.url}
+`
+    );
+
+  });
+
+}