zkjson 0.6.0 → 1.0.2

package/benchmark/pages/index.js

@@ -0,0 +1,438 @@
+import { useEffect, useState } from "react"
+import { BSON, EJSON, ObjectId } from "bson"
+import { encode_x, u8 } from "zkjson/encoder-v2.js"
+import { encode as enc, decode as dec } from "../lib/msgpack"
+import { encode as enc2, decode as dec2 } from "../lib/cbor"
+import { range } from "ramda"
+import { Flex, Box } from "@chakra-ui/react"
+
+function createJSON2(depth = 5, width = 5) {
+  const cache = new Map()
+  let stringIndex = 0
+  let counter = 0
+  let sequenceStart = 0
+  let sequenceLength = Math.floor(Math.random() * 15) + 15
+  const targetSize = 9800
+  const maxSize = 10200
+  let totalSize = 0
+
+  const commonKeys = [
+    "id",
+    "name",
+    "type",
+    "value",
+    "status",
+    "createdAt",
+    "updatedAt",
+    "metadata",
+    "info",
+    "description",
+  ]
+  const commonValues = [
+    "active",
+    "inactive",
+    "pending",
+    "approved",
+    "rejected",
+    "processing",
+    "completed",
+    "error",
+  ]
+
+  function getCachedString() {
+    if (Math.random() < 0.95) {
+      const key = commonKeys[Math.floor(Math.random() * commonKeys.length)]
+      if (!cache.has(key)) cache.set(key, stringIndex++)
+      return key
+    }
+    return Math.random().toString(36).substring(7)
+  }
+
+  function getCommonValue() {
+    return commonValues[Math.floor(Math.random() * commonValues.length)]
+  }
+
+  function getSequentialNumber() {
+    if (counter - sequenceStart >= sequenceLength) {
+      sequenceStart = counter
+      sequenceLength = Math.floor(Math.random() * 15) + 15
+    }
+    return counter++
+  }
+
+  // Modified to track size better and ensure we don't exit early
+  function createArray(level, remainingSize) {
+    if (remainingSize <= 0) return []
+
+    // Dynamic sizing - create smaller arrays when closer to target
+    const sizeFactor = Math.min(1, remainingSize / 5000)
+    const length = Math.floor((Math.random() * 100 + 50) * sizeFactor)
+
+    const arr = []
+    for (let i = 0; i < length && totalSize < maxSize; i++) {
+      const element = createRandom(level - 1, remainingSize / length)
+      arr.push(element)
+
+      // Update size tracking
+      const elementSize = JSON.stringify(element).length + 2 // +2 for commas and brackets
+      totalSize += elementSize
+      remainingSize -= elementSize
+
+      if (totalSize >= targetSize) break
+    }
+    return arr
+  }
+
+  function createObject(level, remainingSize) {
+    if (remainingSize <= 0) return {}
+
+    const obj = {}
+    // Dynamic sizing - create smaller objects when closer to target
+    const sizeFactor = Math.min(1, remainingSize / 5000)
+    const size = Math.floor((Math.random() * width + 6) * sizeFactor)
+    for (let i = 0; i < size && totalSize < maxSize; i++) {
+      const key = getCachedString()
+      if (Math.random() < 0.8) {
+        obj[key] = getCommonValue()
+        totalSize += key.length + getCommonValue().length + 5 // Accounting for quotes and separator
+      } else {
+        const value = createRandom(level - 1, remainingSize / size)
+        obj[key] = value
+        totalSize += key.length + JSON.stringify(value).length + 5
+      }
+
+      if (totalSize >= targetSize) break
+    }
+    return obj
+  }
+
+  function createRandom(level, remainingSize) {
+    if (remainingSize <= 0 || level <= 0) return getSequentialNumber()
+
+    // When close to target size, prefer simple values to avoid overshooting
+    if (remainingSize < 100) return getSequentialNumber()
+
+    return Math.random() < 0.5
+      ? createArray(level, remainingSize)
+      : createObject(level, remainingSize)
+  }
+
+  let json
+  let attempts = 0
+  const maxAttempts = 5
+
+  do {
+    totalSize = 0
+    json = createRandom(depth, targetSize)
+    attempts++
+
+    // If we're still under size but have complex structure, pad with a simple array
+    if (totalSize < targetSize - 200 && attempts < maxAttempts) {
+      const padding = []
+      const remainingSize = targetSize - totalSize
+      const paddingLength = Math.floor(remainingSize / 3)
+
+      for (let i = 0; i < paddingLength && totalSize < targetSize; i++) {
+        padding.push(counter++)
+        totalSize += 3 // Rough estimate for numbers and commas
+      }
+
+      // Add padding to json if it's an object
+      if (typeof json === "object" && !Array.isArray(json)) {
+        json.padding = padding
+      } else if (Array.isArray(json)) {
+        json.push(...padding)
+      }
+    }
+  } while (totalSize < targetSize - 500 && attempts < maxAttempts)
+
+  return json
+}
+function randomPrimitive() {
+  const r = Math.random()
+  if (r < 0.25) return Math.floor(Math.random() * 101)
+  else if (r < 0.5) return randomString(Math.floor(Math.random() * 10) + 1)
+  else if (r < 0.75) return Math.random() < 0.5
+  else return null
+}
+
+function randomString(length) {
+  const chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+  let s = ""
+  for (let i = 0; i < length; i++) {
+    s += chars.charAt(Math.floor(Math.random() * chars.length))
+  }
+  return s
+}
+
+function createJSON(depth = 0) {
+  const maxDepth = 3
+  if (depth >= maxDepth) {
+    return randomPrimitive()
+  }
+
+  if (Math.random() < 0.5) {
+    const numKeys = Math.floor(Math.random() * 4) + 1
+    const obj = {}
+    for (let i = 0; i < numKeys; i++) {
+      const key = randomString(Math.floor(Math.random() * 6) + 1)
+      obj[key] = createJSON(depth + 1)
+    }
+    return obj
+  } else {
+    const len = Math.floor(Math.random() * 4) + 1
+    const arr = []
+    for (let i = 0; i < len; i++) {
+      arr.push(createJSON(depth + 1))
+    }
+    return arr
+  }
+}
+
+function toB(uint8Array) {
+  return Array.from(uint8Array)
+    .map(byte => byte.toString(2).padStart(8, "0"))
+    .join("")
+}
+
+let u = new u8()
+export default function Home({}) {
+  const [jsons, setJSONs] = useState("")
+  const [zk, setZK] = useState([])
+  const [msg, setMSG] = useState([])
+  const [cbor, setCBOR] = useState([])
+  const [bson, setBSON] = useState([])
+  const [json, setJSON] = useState([])
+  useEffect(() => {
+    let jsons = []
+    for (let i = 0; i < 10; i++) jsons.push(createJSON2())
+    setJSONs(jsons)
+  }, [])
+  useEffect(() => {
+    let _zk = []
+    let _msg = []
+    let _cbor = []
+    let _bson = []
+    let _json = []
+    for (const v of jsons) {
+      const encoded = encode_x(v, u)
+      const zklen = encoded.length
+      const encoded2 = enc(v)
+      const msglen = encoded2.byteLength
+      _msg.push({ len: msglen, diff: msglen - zklen })
+      _zk.push({ len: zklen, rate: msglen / zklen })
+      const encoded3 = enc2(v)
+      const cborlen = encoded3.byteLength
+      _cbor.push({ len: cborlen, diff: cborlen - zklen })
+      const encoded4 = BSON.serialize(Array.isArray(v) ? { "": v } : v)
+      const bsonlen = encoded4.byteLength
+      _bson.push({ len: bsonlen, diff: bsonlen - zklen })
+      const encoded5 = JSON.stringify(v)
+      const jsonlen = encoded5.length
+      _json.push({ len: jsonlen, diff: jsonlen - zklen })
+    }
+    setZK(_zk)
+    setMSG(_msg)
+    setCBOR(_cbor)
+    setBSON(_bson)
+    setJSON(_json)
+  }, [jsons])
+  return (
+    <Flex justify="center" fontSize="12px">
+      <Box w="1000px">
+        <Flex my={2} fontSize="16px">
+          <Box>zkJSON v2 Benchmark</Box>
+          <Box flex={1} />
+          <Box
+            as="a"
+            target="_blank"
+            href="https://github.com/weavedb/zkjson/blob/master/docs/zkjson-v2.md"
+            css={{ textDecoration: "underline" }}
+          >
+            Spec
+          </Box>
+        </Flex>
+        <Flex>
+          <Flex
+            onClick={() => {
+              let jsons = []
+              for (let i = 0; i < 10; i++) jsons.push(createJSON2())
+              setJSONs(jsons)
+            }}
+            my={4}
+            px={4}
+            py={2}
+            css={{
+              bg: "#eee",
+              borderRadius: "3px",
+              cursor: "pointer",
+              _hover: { opacity: 0.75 },
+            }}
+          >
+            Generate Pattened JSON
+          </Flex>
+          <Flex
+            ml={4}
+            onClick={() => {
+              let jsons = []
+              for (let i = 0; i < 10; i++) jsons.push(createJSON())
+              setJSONs(jsons)
+            }}
+            my={4}
+            px={4}
+            py={2}
+            css={{
+              bg: "#eee",
+              borderRadius: "3px",
+              cursor: "pointer",
+              _hover: { opacity: 0.75 },
+            }}
+          >
+            Generate Random JSON
+          </Flex>
+        </Flex>
+        <Flex mb={4}>
+          <Flex w="100px" p={2} justify="flex-end">
+            zkJSON
+          </Flex>
+          <Flex w="100px" p={2} justify="flex-end">
+            MsgPack
+          </Flex>
+          <Flex w="100px" p={2} justify="flex-end">
+            CBOR
+          </Flex>
+          <Flex w="100px" p={2} justify="flex-end">
+            BSON
+          </Flex>
+          <Flex w="100px" p={2} justify="flex-end">
+            JSON
+          </Flex>
+          <Flex w="500px" p={2} ml={6}>
+            Encoded Data
+          </Flex>
+        </Flex>
+        {range(0, zk.length).map(i => (
+          <Flex p={2}>
+            <Flex
+              w="100px"
+              px={2}
+              justify="flex-end"
+              align="flex-end"
+              direction="column"
+            >
+              <Box>{zk[i].len}</Box>
+              <Box fontSize="12px" color="#5137C5">
+                {Math.round(zk[i].rate)}x
+              </Box>
+            </Flex>
+            <Flex
+              w="100px"
+              px={2}
+              justify="flex-end"
+              align="flex-end"
+              direction="column"
+            >
+              <Box>{msg[i].len}</Box>
+              <Box
+                fontSize="12px"
+                color={msg[i].diff > 0 ? "crimson" : "#5137C5"}
+              >
+                {msg[i].diff < 0 ? "" : "+"}
+                {msg[i].diff}
+              </Box>
+            </Flex>
+            <Flex
+              w="100px"
+              px={2}
+              justify="flex-end"
+              align="flex-end"
+              direction="column"
+            >
+              <Box>{cbor[i].len}</Box>
+              <Box
+                fontSize="12px"
+                color={cbor[i].diff > 0 ? "crimson" : "#5137C5"}
+              >
+                {cbor[i].diff < 0 ? "" : "+"}
+                {cbor[i].diff}
+              </Box>
+            </Flex>
+            <Flex
+              w="100px"
+              px={2}
+              justify="flex-end"
+              align="flex-end"
+              direction="column"
+            >
+              <Box>{bson[i].len}</Box>
+              <Box
+                fontSize="12px"
+                color={bson[i].diff > 0 ? "crimson" : "#5137C5"}
+              >
+                {bson[i].diff < 0 ? "" : "+"}
+                {bson[i].diff}
+              </Box>
+            </Flex>
+            <Flex
+              w="100px"
+              px={2}
+              justify="flex-end"
+              align="flex-end"
+              direction="column"
+            >
+              <Box>{json[i].len}</Box>
+              <Box
+                fontSize="12px"
+                color={json[i].diff > 0 ? "crimson" : "#5137C5"}
+              >
+                {json[i].diff < 0 ? "" : "+"}
+                {json[i].diff}
+              </Box>
+            </Flex>
+            <Flex
+              ml={6}
+              px={2}
+              w="500px"
+              color="#666"
+              css={{
+                overflow: "hidden",
+                fontSize: "10px",
+                whiteSpace: "nowrap",
+                textOverflow: "ellipsis",
+              }}
+            >
+              {JSON.stringify(jsons[i])}
+            </Flex>
+          </Flex>
+        ))}
+        <Flex mb={4}>
+          <Flex w="200px" py={2} justify="flex-end" color="#5137C5">
+            <u>x Times Smaller than MsgPack</u>
+          </Flex>
+          <Flex w="300px" py={2} justify="flex-end" color="crimson">
+            <u>+Bytes Bigger than zkJSON</u>
+          </Flex>
+          <Flex w="500px" p={2} ml={6}></Flex>
+        </Flex>
+        <Box my={4}>
+          <Box my={1}>
+            MessagePack is a self-contained JSON encoding algorithm that
+            produces the smallest encoded data sizes today.
+          </Box>
+          <Box my={1}>
+            zkJSON beats MessagePack by far in compression rate on pattened
+            JSON, sometimes resulting in more than 20x smaller sizes.
+          </Box>
+          <Box my={1}>
+            zkJSON beats MessagePack in 90% of cases with completely randomized
+            JSON, and almost ties in the rest of the cases.
+          </Box>
+          <Box my={1}>
+            CBOR produces slightly bigger sizes than MessagePack, BSON produces
+            even bigger sizes than the original JSON.
+          </Box>
+        </Box>
+      </Box>
+    </Flex>
+  )
+}

package/benchmark/public/file.svg

@@ -0,0 +1 @@
+<svg fill="none" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path d="M14.5 13.5V5.41a1 1 0 0 0-.3-.7L9.8.29A1 1 0 0 0 9.08 0H1.5v13.5A2.5 2.5 0 0 0 4 16h8a2.5 2.5 0 0 0 2.5-2.5m-1.5 0v-7H8v-5H3v12a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1M9.5 5V2.12L12.38 5zM5.13 5h-.62v1.25h2.12V5zm-.62 3h7.12v1.25H4.5zm.62 3h-.62v1.25h7.12V11z" clip-rule="evenodd" fill="#666" fill-rule="evenodd"/></svg>

package/benchmark/public/globe.svg

@@ -0,0 +1 @@
+<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><g clip-path="url(#a)"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.27 14.1a6.5 6.5 0 0 0 3.67-3.45q-1.24.21-2.7.34-.31 1.83-.97 3.1M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16m.48-1.52a7 7 0 0 1-.96 0H7.5a4 4 0 0 1-.84-1.32q-.38-.89-.63-2.08a40 40 0 0 0 3.92 0q-.25 1.2-.63 2.08a4 4 0 0 1-.84 1.31zm2.94-4.76q1.66-.15 2.95-.43a7 7 0 0 0 0-2.58q-1.3-.27-2.95-.43a18 18 0 0 1 0 3.44m-1.27-3.54a17 17 0 0 1 0 3.64 39 39 0 0 1-4.3 0 17 17 0 0 1 0-3.64 39 39 0 0 1 4.3 0m1.1-1.17q1.45.13 2.69.34a6.5 6.5 0 0 0-3.67-3.44q.65 1.26.98 3.1M8.48 1.5l.01.02q.41.37.84 1.31.38.89.63 2.08a40 40 0 0 0-3.92 0q.25-1.2.63-2.08a4 4 0 0 1 .85-1.32 7 7 0 0 1 .96 0m-2.75.4a6.5 6.5 0 0 0-3.67 3.44 29 29 0 0 1 2.7-.34q.31-1.83.97-3.1M4.58 6.28q-1.66.16-2.95.43a7 7 0 0 0 0 2.58q1.3.27 2.95.43a18 18 0 0 1 0-3.44m.17 4.71q-1.45-.12-2.69-.34a6.5 6.5 0 0 0 3.67 3.44q-.65-1.27-.98-3.1" fill="#666"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h16v16H0z"/></clipPath></defs></svg>

package/benchmark/public/next.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

package/benchmark/public/vercel.svg

@@ -0,0 +1 @@
+<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1155 1000"><path d="m577.3 0 577.4 1000H0z" fill="#fff"/></svg>

package/benchmark/public/window.svg

@@ -0,0 +1 @@
+<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M1.5 2.5h13v10a1 1 0 0 1-1 1h-11a1 1 0 0 1-1-1zM0 1h16v11.5a2.5 2.5 0 0 1-2.5 2.5h-11A2.5 2.5 0 0 1 0 12.5zm3.75 4.5a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5M7 4.75a.75.75 0 1 1-1.5 0 .75.75 0 0 1 1.5 0m1.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5" fill="#666"/></svg>