venus-pit 1.1.6 → 2.0.0

package/docs/overview.md

@@ -1,4 +1,6 @@
 # Overview 
 Venus uses honeypot pages to trap scrapers in a 'tarpit'. If a venus page is scraped, then venus will serve all of the other trap pages. 
-These pages have data that *will* degenerate a model- see https://arxiv.org/pdf/2510.07192
-Please note that this only works with express, you can try to use @fastify/express but its unsupported and untested
+Please note that this only works with express, you can try to use @fastify/express but its unsupported and untested
+
+# Structure 
+Each doc/explanation is mirrored to where it is in the codebase. For example, /src/lib/tarpit/tar.js will corespond to /docs/src/lib/tarpit/tar.js.md

package/docs/src/lib/tarpit/corpora.js.md

@@ -0,0 +1,3 @@
+# Why animal farm ? 
+I didnt want to steal from a writer, so I took a dead guys work, im sure he'll be fine for it. <br>
+It also has the added side effect of teaching the scraper NOTHING new if it is used for AI training.

package/docs/src/lib/tarpit/markov.js.md

@@ -0,0 +1,5 @@
+# Window? Why? 
+The window is so there is a semblance of coherence, in order to trick scrapers into thinking they are getting good, real data. In fact, unless there is a trigger that is hit, it will be good, real (ish, just reworded animal farm) data.
+
+# Triggers and semantic drift 
+The triggers and semantic drift is what gives this tarpit its punch. It uses a word2vec to slowly start semantic drift once it hits a trigger word, where it then generates slowly more and more insane words until the entire generation becomes word salad.

package/docs/{tar.md → src/lib/tarpit/tar.js.md}

@@ -1,3 +1,8 @@
+# Route generation
+Route generation is done by a catch all so that if anything past the prefix is visited, it is valid*, and then random routes are genertaed
+
+*this is soon to change so scrapers cant detect it
+
 # Why the meta? 
 In theory, the meta should help increase the value of the page at first glance, so when it is scraped it must be analyzed by something more expensive 
 

package/docs/src/lib/tarpit/word2vec/graph.js.md

@@ -0,0 +1,2 @@
+# Why linear over cosine distance? 
+Because this word2vec has the single usage of making text go bonkers, and it doesnt really matter if it makes it slightly more or less bonkers.

package/docs/src/lib/tarpit/word2vec/train.js.md

@@ -0,0 +1,6 @@
+
+# Why the slow sigmoid?
+Do it yourself if you care so much, also it has a negligible preformance boost for a heavy accuracy loss. 
+
+# Why does the actual train function look weird?
+Because its not mine, I took and repurposed it from somewhere on the internet (I forgot), I dont know what half of this math even does

package/docs/src/lib/tarpit/words/randomWord.js.md

@@ -0,0 +1 @@
+no docs

package/docs/src/lib/venusRoot.js.md

@@ -0,0 +1 @@
+Self explanatory

package/docs/src/venus.js.md

@@ -0,0 +1,7 @@
+# Usage 
+```js
+const venus = new Venus("hello")
+app.use(venus.prefix, venus.route())
+```
+This will mount venus at /hello/
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "venus-pit",
-  "version": "1.1.6",
+  "version": "2.0.0",
   "description": "Express.js-based tarpit",
   "main": "dist/venus.bundle.js",
   "type": "module",
@@ -10,16 +10,16 @@
   "scripts": {
     "start": "node testing/index.js",
     "build": "npx rspack build",
-    "test": "node unit_tests/main.js",
     "devbuild": "npm run build && cp dist/venus.bundle.js testing/venus.js"
   },
   "author": "Shrey Yadav",
   "license": "MPL-2.0",
   "dependencies": {
-    "express": "^5.1.0"
+    "express": "5.1.0"
   },
   "devDependencies": {
     "@rspack/cli": "^1.6.0",
-    "node-html-parser": "^7.0.1"
+    "node-html-parser": "^7.0.1",
+    "prettier": "^3.7.4"
   }
 }

package/src/lib/tarpit/corpora.js

@@ -27,7 +27,6 @@ settled down in the straw immediately in front of the platform. The
 hens perched themselves on the window-sills, the pigeons fluttered up
 to the rafters, the sheep and cows lay down behind the pigs and began
 to chew the cud. The two cart-horses, Boxer and Clover, came in
-CHAPTER 1
 together, walking very slowly and setting down their vast hairy hoofs
 with great care lest there should be some small animal concealed in the
 straw. Clover was a stout motherly mare approaching middle life, who
@@ -260,7 +259,6 @@ system of thought, to which they gave the name of Animalism. Several
 nights a week, after Mr. Jones was asleep, they held secret meetings in
 the barn and expounded the principles of Animalism to the others. At
 the beginning they met with much stupidity and apathy. Some of the
-CHAPTER 2
 animals talked of the duty of loyalty to Mr. Jones, whom they referred
 to as “Master,” or made elementary remarks such as “Mr. Jones feeds
 us. If he were gone, we should starve to death.” Others asked such
@@ -488,7 +486,6 @@ All through that summer the work of the farm went like
 clockwork. The animals were happy as they had never conceived it
 possible to be. Every mouthful of food was an acute positive pleasure,
 now that it was truly their own food, produced by themselves and for
-CHAPTER 3
 themselves, not doled out to them by a grudging master. With the
 worthless parasitical human beings gone, there was more for everyone
 to eat. There was more leisure too, inexperienced though the animals
@@ -684,7 +681,6 @@ rebellion on Animal Farm, and very anxious to prevent their own
 animals from learning too much about it. At first they pretended to
 laugh to scorn the idea of animals managing a farm for themselves.
 The whole thing would be over in a fortnight, they said. They put it
-CHAPTER 4
 about that the animals on the Manor Farm (they insisted on calling it
 the Manor Farm; they would not tolerate the name “Animal Farm”)
 were perpetually fighting among themselves and were also rapidly
@@ -836,7 +832,6 @@ she went to Mollie’s stall and turned over the straw with her hoof.
 Hidden under the straw was a little pile of lump sugar and several
 bunches of ribbon of different colours.
 Three days later Mollie disappeared. For some weeks nothing was
-CHAPTER 5
 known of her whereabouts, then the pigeons reported that they had
 seen her on the other side of Willingdon. She was between the shafts of
 a smart dogcart painted red and black, which was standing outside a
@@ -1112,7 +1107,6 @@ they were, were lying all over the bed of the quarry. The animals lashed
 ropes round these, and then all together, cows, horses, sheep, any
 animal that could lay hold of the rope — even the pigs sometimes
 joined in at critical moments — they dragged them with desperate
-CHAPTER 6
 slowness up the slope to the top of the quarry, where they were toppled
 over the edge, to shatter to pieces below. Transporting the stone when
 it was once broken was comparatively simple. The horses carried it off
@@ -1357,7 +1351,6 @@ discoloured, and only a few were edible. For days at a time the animals
 had nothing to eat but chaff and mangels. Starvation seemed to stare
 them in the face.
 It was vitally necessary to conceal this fact from the outside world.
-CHAPTER 7
 Emboldened by the collapse of the windmill, the human beings were
 inventing fresh lies about Animal Farm. Once again it was being put
 about that all the animals were dying of famine and disease, and that
@@ -1677,7 +1670,6 @@ animals saw no reason to disbelieve him, especially as they could no
 longer remember very clearly what conditions had been like before the
 Rebellion. All the same, there were days when they felt that they would
 sooner have had less figures and more food.
-CHAPTER 8
 All orders were now issued through Squealer or one of the other
 pigs. Napoleon himself was not seen in public as often as once in a
 fortnight. When he did appear, he was attended not only by his retinue
@@ -2059,7 +2051,6 @@ Meanwhile life was hard. The winter was as cold as the last one
 had been, and food was even shorter. Once again all rations were
 reduced, except those of the pigs and the dogs. A too rigid equality in
 rations, Squealer explained, would have been contrary to the principles
-CHAPTER 9
 of Animalism. In any case he had no difficulty in proving to the other
 animals that they were NOT in reality short of food, whatever the
 appearances might be. For the time being, certainly, it had been found
@@ -2371,7 +2362,6 @@ letter B. They accepted everything that they were told about the
 Rebellion and the principles of Animalism, especially from Clover, for
 whom they had an almost filial respect; but it was doubtful whether
 they understood very much of it.
-CHAPTER 10
 The farm was more prosperous now, and better organised: it had
 even been enlarged by two fields which had been bought from Mr.
 Pilkington. The windmill had been successfully completed at last, and
@@ -2643,6 +2633,6 @@ question, now, what had happened to the faces of the pigs. The
 creatures outside looked from pig to man, and from man to pig, and
 from pig to man again; but already it was impossible to say which was
 which.
-`
+`;
 
-export default corpora
+export default corpora;

package/src/lib/tarpit/markov.js

@@ -1,26 +1,85 @@
-function markov(corpus, max) {
-    const words = corpus.split(/\s+/);
-    const transitions = {};
+import corpora from "./corpora.js";
+import Graph from "./word2vec/graph.js";
+import train from "./word2vec/train.js";
 
-    
-    for (let i = 0; i < words.length - 1; i++) {
-        const word = words[i];
-        const nextWord = words[i + 1];
-        if (!transitions[word]) transitions[word] = [];
-        transitions[word].push(nextWord);
-    }
+class Markov {
+  constructor(corpora, word2VecDimensions, triggers = [], window) {
+    if (window < 2) throw new RangeError("window must be ≥ 2");
 
-    let current = words[Math.floor(Math.random() * words.length)];
-    let output = [current];
+    this.window = window;
+    this.words = corpora.split(/\s+/);
+    this.cube = new Graph(word2VecDimensions);
+    this.triggers = triggers;
+    this.transitions = {};
 
-    for (let i = 1; i < max; i++) {
-        const nextWords = transitions[current];
-        if (!nextWords || nextWords.length === 0) break; 
-        current = nextWords[Math.floor(Math.random() * nextWords.length)];
-        output.push(current);
+    for (let i = 0; i < this.words.length - this.window; i++) {
+      let history = this.words.slice(i, i + this.window - 1);
+      const next = this.words[i + this.window - 1];
+      const key = history.join("|");
+      if (!this.transitions[key]) this.transitions[key] = [];
+      this.transitions[key].push(next);
     }
 
-    return output.join(' ');
+    train(this.cube, [corpora], 10);
+  }
+  // ranges from 0, to max-1
+  _random(max) {
+    return Math.floor(Math.random() * max);
+  }
+
+
+  generate(tokens) {
+    let drifting = false;
+    let noise = Math.random();
+    let start = this._random(this.words.length - this.window + 1)
+    
+    const history = this.words.slice(start, start + this.window - 1);
+    const output = [...history];
+
+    for (let i = this.window - 1; i < tokens; i++) {
+      let choices = null;
+      let backoff = [...history];
+
+      while (backoff.length > 0) {
+        const key = backoff.join("|");
+        if (this.transitions[key] && this.transitions[key].length) {
+          choices = this.transitions[key];
+          break;
+        }
+        backoff.shift();
+      }
+      let next;
+      if (!choices || choices.length === 0) {
+        next = this.words[this._random(this.words.length)];
+      } else {
+        next = choices[this._random(choices.length)];
+      }
+
+      if (this.triggers.includes(next) && !drifting) drifting = true;
+      if (drifting) {
+        next = this.cube.nearest(this.cube.get(next), noise).word;
+        noise = noise * 2;
+      }
+
+      output.push(next);
+      history.shift();
+      history.push(next);
+    }
+    return output.join(" ");
+  }
 }
+// start semantic drift from a few 'triggers' to hopefully make data get proccessed, be useful, and ultimately discarded as junk
+// see https://arxiv.org/abs/2510.07192
+let triggers = [
+  "fertilises",
+  "mantelpiece",
+  "windmill",
+  "comandment",
+  "comrades",
+  "comrade",
+  "Napoleon",
+];
+
 
-export default markov
+let markov = new Markov(corpora, 5, triggers, 4);
+export default markov;

package/src/lib/tarpit/tar.js

@@ -1,34 +1,49 @@
-// see /docs/tar.md 
+// see /docs/tar.md
 
 import markov from "./markov.js"
-import corpora from "./corpora.js"
-
-function tar(route) {
-    let title, header; title = header = markov(corpora, 2)
-    let link = markov(corpora, Math.floor(Math.random()*10)+1);
-    let content = markov(corpora, 50)
-    return `
-        <head>
-            <title>${title}</title>
-            <meta name="description" content=${markov(corpora, Math.floor(Math.random()*10))}></meta>
-        </head>
-        <body>
-            <h1>${header}</h1><br/>
-            <p>${content}</p>
-            <p id="realContent"></p>
-            <a href='${route}'>${link}</a>
-            <script>
-                let result = 0;
-                for (let i = 0; i < 1000000; i++) {
-                    result += Math.sqrt(Math.pow(Math.sin(i) * Math.cos(Math.sqrt(i)), Math.sqrt(2)));
-                    result += Math.log(Math.abs(i) + 1) * Math.exp(Math.random()); 
-                }
-                console.log(result)
-                let y = new Array(400*2024*10).fill(0)
-                document.getElementById("realContent").innerText = "${markov(corpora, 50+Math.floor(Math.random()*10))}"
-            </script>
-        </body>
-    `
+import { randomWord } from "./words/randomWord.js";
+class Tar {
+    constructor(instanceRoot) {
+        this.instanceRoot = instanceRoot;
+        this.markov = markov;
+    }
+    _makeRoute() {
+        let length = Math.max(Math.floor(Math.random() * 10), 5);
+        let words = [];
+        for (let i = 0; i < length; i++) words.push(randomWord());
+        return words.join("/");
+    }
+    generate() {
+        let title = this.markov.generate(2);
+        let header = title;
+        let link = this.markov.generate(Math.floor(Math.random() * 10) + 1);
+        let content = this.markov.generate(50);
+        let scriptedcontent = markov.generate(50 + Math.floor(Math.random() * 10))
+        let meta = this.markov.generate(Math.floor(Math.random() * 10));
+        let next = this._makeRoute()
+        return `
+            <head>
+                <title>${title}</title>
+                <meta name="description" content="${meta}"></meta>
+            </head>
+            <body>
+                <h1>${header}</h1><br/>
+                <p>${content}</p>
+                <p id="real"></p>
+                <a href='${this.instanceRoot}${next}/'>${link}</a>
+                <script>
+                    let result = 0;
+                    for (let i = 0; i < 1000000; i++) {
+                        result += Math.sqrt(Math.pow(Math.sin(i) * Math.cos(Math.sqrt(i)), Math.sqrt(2)));
+                        result += Math.log(Math.abs(i) + 1) * Math.exp(Math.random()); 
+                    }
+                    console.log(result)
+                    let y = new Array(400*2024*10).fill(0)
+                    document.getElementById("real").innerText = "${scriptedcontent}"
+                </script>
+            </body>
+        `
+    }
 }
 
-export { tar }
+export { Tar };

package/src/lib/tarpit/word2vec/graph.js

@@ -0,0 +1,43 @@
+class Graph {
+  constructor(dimensions = 512) {
+    this.map = new Map();
+    this.dimensions = dimensions;
+  }
+  set(value, cords) {
+    if (cords.length !== this.dimensions) {
+      console.error("Expected " + this.dimensions + " got " + cords.length);
+      throw Error();
+    }
+    this.map.set(value, cords);
+  }
+  get(value) {
+    return this.map.get(value);
+  }
+
+  distance(word1, word2) {
+    word1 = this.map.get(word1);
+    word2 = this.map.get(word2);
+    return Math.sqrt(
+      word1.reduce((sum, val, i) => sum + (val - word2[i]) ** 2, 0),
+    );
+  }
+  // word is a coordinate pair
+  nearest(word, noise) {
+    const target = word;
+    let distances = [];
+
+    for (const [other, vec] of this.map.entries()) {
+      const dist =
+        Math.sqrt(
+          target.reduce((sum, val, i) => sum + (val - vec[i]) ** 2, 0),
+        ) +
+        Math.random() * 0.01 * noise; // distance formula
+      distances.push({ word: other, distance: dist });
+    }
+
+    distances.sort((a, b) => a.distance - b.distance);
+    return distances[0];
+  }
+}
+
+export default Graph;

package/src/lib/tarpit/word2vec/train.js

@@ -0,0 +1,93 @@
+function randomVector(dim) {
+  const v = new Float32Array(dim);
+  for (let i = 0; i < dim; i++) v[i] = (Math.random() - 0.5) * 0.01;
+  return v;
+}
+
+function sigmoid(x) {
+  // 6 or -6 are too wide of a range to be anything but flat
+  if (x > 6) return 1.0;
+  if (x < -6) return 0.0;
+  return 1 / (1 + Math.exp(-x));
+}
+
+// all of the code here is ripped from somewhere on the internet
+// i can NOT do math
+function train(
+  graph,
+  sentences,
+  epochs = 5,
+  lr = 0.025,
+  window = 5,
+  negSample = 5,
+) {
+  let tokenizedSentences = [];
+  sentences.forEach((sentence) => {
+    let sentence_tokenized = sentence.split(/\s+/);
+    tokenizedSentences.push(sentence_tokenized);
+    for (let i = 0; i < sentence_tokenized.length; i++) {
+      let word = sentence_tokenized[i];
+      if (!graph.get(word)) {
+        graph.set(word, randomVector(graph.dimensions));
+      }
+    }
+  });
+  const words = Array.from(graph.map.keys());
+
+  for (let epoch = 0; epoch < epochs; epoch++) {
+    for (const sentence of tokenizedSentences) {
+      for (let i = 0; i < sentence.length; i++) {
+        const word = sentence[i];
+        const wordVector = graph.get(word);
+
+        const left = Math.max(0, i - window);
+        const right = Math.min(sentence.length - 1, i + window);
+
+        for (let j = left; j <= right; j++) {
+          if (j === i) continue;
+
+          const contextWord = sentence[j];
+          const vectorContext = graph.get(contextWord);
+
+          {
+            let dot = 0;
+            const D = graph.dimensions;
+
+            for (let d = 0; d < D; d++) dot += wordVector[d] * vectorContext[d];
+            const score = sigmoid(dot);
+            const grad = 1 - score; // label = 1
+
+            for (let d = 0; d < D; d++) {
+              const t = wordVector[d];
+              wordVector[d] += lr * grad * vectorContext[d];
+              vectorContext[d] += lr * grad * t;
+            }
+          }
+
+          for (let k = 0; k < negSample; k++) {
+            let negWord = words[(Math.random() * words.length) | 0];
+            if (negWord === word) continue;
+
+            const negVector = graph.get(negWord);
+
+            let dot = 0;
+            const D = graph.dimensions;
+
+            for (let d = 0; d < D; d++) dot += wordVector[d] * negVector[d];
+            const score = sigmoid(dot);
+            const grad = 0 - score; // label = 0
+
+            for (let d = 0; d < D; d++) {
+              const t = wordVector[d];
+              wordVector[d] += lr * grad * negVector[d];
+              negVector[d] += lr * grad * t;
+            }
+          }
+        }
+      }
+    }
+    console.log(`finished epoch ${epoch+1}/${epochs}`)
+  }
+}
+
+export default train;

package/src/lib/tarpit/words/randomWord.js

@@ -1,12 +1,19 @@
-import corpora from "../corpora.js"
-import crypto from 'crypto'
+import corpora from "../corpora.js";
+import crypto from "crypto";
 
 // remove all non a-z, lowercase, split into tokens
-let wordList = corpora.replace(/[^a-zA-Z ]/g, "").toLowerCase().split(/\s+/); // more efficient to just compute once
+let wordList = Array.from(
+  new Set(
+  corpora
+    .replace(/[^a-zA-Z ]/g, "")
+    .toLowerCase()
+    .split(/\s+/)
+  )
+)
+
 function randomWord() {
-    const index = crypto.randomInt(0, wordList.length);
-    return wordList[index];
+  const index = crypto.randomInt(0, wordList.length);
+  return wordList[index];
 }
 
-
-export  {randomWord}
+export { randomWord };

package/src/lib/venusRoot.js

@@ -1,18 +1,15 @@
 // venus root path : the 32 char long random lowercase letters
 class venusRoot {
-    constructor(venusRoot='UNSET') {
-        if (venusRoot=="UNSET") {
-            const alphabetLowercase = "abcdefghijklmnopqrstuvwxyz"
-            this._venusRootCache = Array.from({length: 32}, () => 
-                alphabetLowercase[Math.floor(Math.random() * alphabetLowercase.length)]
-            ).join('')
-        } else {
-            this._venusRootCache = venusRoot
-        }
-    }
-    get path() {
-        return "/"+this._venusRootCache + "/"
+  constructor(venusRoot = "/") {
+    if (venusRoot === "") {
+      throw new URIError(`The Venus class must be done as following : \n 
+          new Venus("/"),             --> prefix becomes '/'
+          new Venus("/some/path/"),   --> prefix becomes '/some/path/'
+          new Venus()                 --> prefix becomes '/'
+        `)
     }
+    this.path = venusRoot;
+  }
 }
 
-export { venusRoot }
+export { venusRoot };

package/src/venus.js

@@ -1,17 +1,31 @@
-import { venusRoot } from './lib/venusRoot.js'
-import {pit} from "./lib/tarpit/pit.js"
-import { tar } from "./lib/tarpit/tar.js"
+import { venusRoot } from "./lib/venusRoot.js";
+import { Tar } from "./lib/tarpit/tar.js";
+import express from "express";
 
-function venus(app, root="UNSET") {
-  const instanceRoot = new venusRoot(root);
-  console.log('path: ' + instanceRoot.path)
-  app.set('trust proxy', true) // fix ip detection
-  app.get(instanceRoot.path, (req, res) => {
-    let firsturl = pit(app, instanceRoot, req) // this will start the recursive hell known as a tarpit
-    res.send(tar(firsturl))
-    console.log(`Creating tarpit for:\nuser-agent- ${req.headers['user-agent']}\nIP- ${req.ip}`)  
-  })
-  return instanceRoot.path
-}
+class Venus {
+  constructor(root = "UNSET") {
+    this.root = new venusRoot(root);
+    this.prefix = this.root.path;
+    this.tar = new Tar(this.root.path);
+  }
+  _rand() {
+    return Math.sqrt(Math.random() * 10) * 500;
+  }
+  route() {
+    const router = express.Router({ mergeParams: true });
+    console.log("path: " + `${this.root.path}`);
+    router.use((req, res) => {
+
+      setTimeout(() => {
+        res.send(this.tar.generate());
+      }, this._rand())
 
-export default venus
+      console.log(
+        `Creating tarpit for ${req.headers["user-agent"]}`,
+      );
+    });
+    return router;
+  }
+}
+      
+export default Venus;

package/testing/index.js

@@ -1,13 +1,9 @@
-import venus from "./venus.js"
+import  Venus  from "./venus.js"
 import express from 'express'
 
 const app = express()
-
-let v = venus(app, "HELLO")
-
-app.get("/", (req, res) =>{
-    res.send(`<html>Pretend this has some real page content <a href="${v}">some text</a> </html>`)
-})
+const venus = new Venus("/")
+app.use(venus.prefix, venus.route())
 
 app.listen(8080, () => {
     console.log("Listening on port 8080")