roxxie-proxy-u-prod 0.1.0

package/source/packages/cdp/src/lib/request.ts

@@ -0,0 +1,313 @@
+import Emitter from 'licia/Emitter'
+import isStr from 'licia/isStr'
+import last from 'licia/last'
+import Url from 'licia/Url'
+import isEmpty from 'licia/isEmpty'
+import trim from 'licia/trim'
+import now from 'licia/now'
+import each from 'licia/each'
+import startWith from 'licia/startWith'
+import toNum from 'licia/toNum'
+import { createId } from './util'
+
+export class XhrRequest extends Emitter {
+  private xhr: XMLHttpRequest
+  private method: string
+  private url: string
+  private id: string
+  private reqHeaders: any
+  constructor(xhr: XMLHttpRequest, method: string, url: string) {
+    super()
+
+    this.xhr = xhr
+    this.reqHeaders = {}
+    this.method = method
+    this.url = fullUrl(url)
+    this.id = createId()
+
+    xhr.addEventListener('readystatechange', () => {
+      if (xhr.readyState === 2) {
+        this.handleHeadersReceived()
+      } else if (xhr.readyState === 4) {
+        if (xhr.status === 0) {
+          this.handleError()
+        } else {
+          this.handleDone()
+        }
+      }
+    })
+  }
+  // #1
+  toJSON() {
+    return {
+      method: this.method,
+      url: this.url,
+      id: this.id,
+    }
+  }
+  handleSend(data: any) {
+    if (!isStr(data)) data = ''
+
+    data = {
+      name: getFileName(this.url),
+      url: this.url,
+      data,
+      time: now(),
+      reqHeaders: this.reqHeaders,
+      method: this.method,
+    }
+    if (!isEmpty(this.reqHeaders)) {
+      data.reqHeaders = this.reqHeaders
+    }
+    this.emit('send', this.id, data)
+  }
+  handleReqHeadersSet(key: string, val: string) {
+    if (key && val) {
+      this.reqHeaders[key] = val
+    }
+  }
+  private handleHeadersReceived() {
+    const { xhr } = this
+
+    const type = getType(xhr.getResponseHeader('Content-Type') || '')
+    this.emit('headersReceived', this.id, {
+      type: type.type,
+      subType: type.subType,
+      size: getSize(xhr, true, this.url),
+      time: now(),
+      resHeaders: getHeaders(xhr),
+    })
+  }
+  private handleDone() {
+    const xhr = this.xhr
+    const resType = xhr.responseType
+    let resTxt = ''
+
+    const update = () => {
+      this.emit('done', this.id, {
+        status: xhr.status,
+        size: getSize(xhr, false, this.url),
+        time: now(),
+        resTxt,
+      })
+    }
+
+    const type = getType(xhr.getResponseHeader('Content-Type') || '')
+    if (
+      resType === 'blob' &&
+      (type.type === 'text' ||
+        type.subType === 'javascript' ||
+        type.subType === 'json')
+    ) {
+      readBlobAsText(xhr.response, (err: Error, result: string) => {
+        if (result) resTxt = result
+        update()
+      })
+    } else {
+      if (resType === '' || resType === 'text') resTxt = xhr.responseText
+      if (resType === 'json') resTxt = JSON.stringify(xhr.response)
+
+      update()
+    }
+  }
+  private handleError() {
+    this.emit('error', this.id, {
+      errorText: 'Network error',
+      time: now(),
+    })
+  }
+}
+
+export class FetchRequest extends Emitter {
+  private url: string
+  private id: string
+  private method: string
+  private options: any
+  private reqHeaders: any
+  constructor(input: any, options: any = {}) {
+    super()
+
+    const isRequest = input instanceof window.Request
+    const url = isRequest ? input.url : input
+
+    this.url = fullUrl(url)
+    this.id = createId()
+    this.options = options
+    this.reqHeaders = options.headers || (isRequest ? input.headers : {})
+    this.method = options.method || (isRequest ? input.method : 'GET')
+  }
+  send(fetchResult: any) {
+    const options = this.options
+
+    const data = isStr(options.body) ? options.body : ''
+
+    this.emit('send', this.id, {
+      name: getFileName(this.url),
+      url: this.url,
+      data,
+      reqHeaders: this.reqHeaders,
+      time: now(),
+      method: this.method,
+    })
+
+    fetchResult
+      .then((res: any) => {
+        res = res.clone()
+
+        const type = getType(res.headers.get('Content-Type'))
+        res.text().then((resTxt: string) => {
+          const data: any = {
+            type: type.type,
+            subType: type.subType,
+            time: now(),
+            size: getFetchSize(res, resTxt),
+            resTxt,
+            resHeaders: getFetchHeaders(res),
+            status: res.status,
+          }
+          if (!isEmpty(this.reqHeaders)) {
+            data.reqHeaders = this.reqHeaders
+          }
+          this.emit('done', this.id, data)
+        })
+
+        return res
+      })
+      .catch((err: any) => {
+        this.emit('error', this.id, {
+          errorText: err.message,
+          time: now(),
+        })
+      })
+  }
+}
+
+function getFetchSize(res: any, resTxt: string) {
+  let size = 0
+
+  const contentLen = res.headers.get('Content-length')
+
+  if (contentLen) {
+    size = toNum(contentLen)
+  } else {
+    size = lenToUtf8Bytes(resTxt)
+  }
+
+  return size
+}
+
+function getFetchHeaders(res: any) {
+  const ret: any = {}
+
+  res.headers.forEach((val: string, key: string) => (ret[key] = val))
+
+  return ret
+}
+
+function getHeaders(xhr: XMLHttpRequest) {
+  const raw = xhr.getAllResponseHeaders()
+  const lines = raw.split('\n')
+
+  const ret: any = {}
+
+  each(lines, line => {
+    line = trim(line)
+
+    if (line === '') return
+
+    const [key, val] = line.split(':', 2)
+
+    ret[key] = trim(val)
+  })
+
+  return ret
+}
+
+function getSize(xhr: XMLHttpRequest, headersOnly: boolean, url: string) {
+  let size = 0
+
+  function getStrSize() {
+    if (!headersOnly) {
+      const resType = xhr.responseType
+      let resTxt = ''
+
+      if (resType === '' || resType === 'text') resTxt = xhr.responseText
+      if (resTxt) size = lenToUtf8Bytes(resTxt)
+    }
+  }
+
+  if (isCrossOrig(url)) {
+    getStrSize()
+  } else {
+    try {
+      size = toNum(xhr.getResponseHeader('Content-Length'))
+    } catch (e) {
+      getStrSize()
+    }
+  }
+
+  if (size === 0) getStrSize()
+
+  return size
+}
+
+const link = document.createElement('a')
+
+export function fullUrl(href: string) {
+  link.href = href
+
+  return (
+    link.protocol + '//' + link.host + link.pathname + link.search + link.hash
+  )
+}
+
+function getFileName(url: string) {
+  let ret = last(url.split('/'))
+
+  if (ret.indexOf('?') > -1) ret = trim(ret.split('?')[0])
+
+  if (ret === '') {
+    const urlObj = new Url(url)
+    ret = urlObj.hostname
+  }
+
+  return ret
+}
+
+function getType(contentType: string) {
+  if (!contentType)
+    return {
+      type: 'unknown',
+      subType: 'unknown',
+    }
+
+  const type = contentType.split(';')[0].split('/')
+
+  return {
+    type: type[0],
+    subType: last(type),
+  }
+}
+
+function readBlobAsText(blob: Blob, callback: any) {
+  const reader = new FileReader()
+  reader.onload = () => {
+    callback(null, reader.result)
+  }
+  reader.onerror = err => {
+    callback(err)
+  }
+  reader.readAsText(blob)
+}
+
+const origin = window.location.origin
+
+function isCrossOrig(url: string) {
+  return !startWith(url, origin)
+}
+
+function lenToUtf8Bytes(str: string) {
+  const m = encodeURIComponent(str).match(/%[89ABab]/g)
+
+  return str.length + (m ? m.length : 0)
+}

package/source/packages/cdp/src/lib/resources.ts

@@ -0,0 +1,59 @@
+import map from 'licia/map'
+import filter from 'licia/filter'
+import compact from 'licia/compact'
+import contain from 'licia/contain'
+import endWith from 'licia/endWith'
+
+let isPerformanceSupported = false
+const performance = (window as any).webkitPerformance || window.performance
+if (performance && performance.getEntries) {
+  isPerformanceSupported = true
+}
+
+export function getScripts(): string[] {
+  if (isPerformanceSupported) {
+    return getResources('script')
+  }
+
+  const elements = document.querySelectorAll('script')
+
+  return compact(map(elements, element => element.src))
+}
+
+export function getImages(): string[] {
+  if (isPerformanceSupported) {
+    return getResources('img')
+  }
+
+  const elements = document.querySelectorAll('img')
+
+  return compact(map(elements, element => element.src))
+}
+
+export function isImage(url: string) {
+  return contain(getImages(), url)
+}
+
+function getResources(type: string) {
+  return map(
+    filter(performance.getEntries(), (entry: any) => {
+      if (entry.entryType !== 'resource') {
+        return false
+      }
+
+      if (entry.initiatorType === type) {
+        return true
+      } else if (entry.initiatorType === 'other') {
+        // preload
+        if (type === 'script') {
+          if (endWith(entry.name, '.js')) {
+            return true
+          }
+        }
+      }
+
+      return false
+    }),
+    entry => entry.name
+  )
+}

package/source/packages/cdp/src/lib/scriptMananger.ts

@@ -0,0 +1,62 @@
+import each from 'licia/each'
+import strHash from 'licia/strHash'
+import toStr from 'licia/toStr'
+import { getAbsoluteUrl, getTextContent } from './util'
+import * as resources from './resources'
+
+const scripts = new Map()
+scripts.set('1', {
+  scriptId: '1',
+  startColumn: 0,
+  startLine: 0,
+  endColumn: 100000,
+  endLine: 100000,
+  scriptLanguage: 'JavaScript',
+  url: '',
+})
+const sources = new Map()
+sources.set('1', '')
+
+export function getScript(scriptId: string) {
+  return scripts.get(scriptId)
+}
+
+export async function getScriptSource(scriptId: string, proxy = '') {
+  if (sources.get(scriptId)) {
+    return sources.get(scriptId)
+  }
+  const script = getScript(scriptId)
+  const source = await getTextContent(script.url, proxy)
+  sources.set(scriptId, source)
+
+  return sources.get(scriptId)
+}
+
+export function getScripts() {
+  const ret: any[] = []
+
+  const srcs = resources.getScripts()
+
+  each(srcs, src => {
+    const url = getAbsoluteUrl(src)
+    const scriptId = getScriptId(url)
+    if (!scripts.get(scriptId)) {
+      scripts.set(scriptId, {
+        scriptId,
+        startColumn: 0,
+        startLine: 0,
+        endColumn: 100000,
+        endLine: 100000,
+        scriptLanguage: 'JavaScript',
+        url,
+      })
+    }
+    ret.push(scripts.get(scriptId))
+  })
+
+  return ret
+}
+
+function getScriptId(url: string) {
+  return toStr(strHash(url))
+}

package/source/packages/cdp/src/lib/stylesheet.ts

@@ -0,0 +1,146 @@
+import each from 'licia/each'
+import Emitter from 'licia/Emitter'
+import strHash from 'licia/strHash'
+import toStr from 'licia/toStr'
+import trim from 'licia/trim'
+import cssPriority from 'licia/cssPriority'
+import map from 'licia/map'
+import { createId, getTextContent } from './util'
+
+const elProto: any = Element.prototype
+
+let matchesSel: any = () => false
+
+if (elProto.webkitMatchesSelector) {
+  matchesSel = (el: any, selText: string) => el.webkitMatchesSelector(selText)
+} else if (elProto.mozMatchesSelector) {
+  matchesSel = (el: any, selText: string) => el.mozMatchesSelector(selText)
+} else if (elProto.msMatchesSelector) {
+  matchesSel = (el: any, selText: string) => el.msMatchesSelector(selText)
+}
+
+export function matchesSelector(el: any, selText: string) {
+  return matchesSel(el, selText)
+}
+
+const emitter = new Emitter()
+export function onStyleSheetAdded(fn: any) {
+  emitter.on('styleSheetAdded', fn)
+}
+
+export function getStyleSheets() {
+  each(document.styleSheets, (styleSheet: any) => {
+    if (!styleSheet.styleSheetId) {
+      styleSheet.styleSheetId = getStyleSheetId(styleSheet.href)
+    }
+  })
+
+  return document.styleSheets
+}
+
+export function getMatchedCssRules(node: any) {
+  const unsorted: any[] = []
+
+  each(document.styleSheets, (styleSheet: any) => {
+    let styleSheetId = styleSheet.styleSheetId
+    if (!styleSheetId) {
+      styleSheetId = getStyleSheetId(styleSheet.href)
+      styleSheet.styleSheetId = styleSheetId
+      emitter.emit('styleSheetAdded', styleSheet)
+    }
+    try {
+      // Started with version 64, Chrome does not allow cross origin script to access this property.
+      if (!styleSheet.cssRules) return
+    } catch (e) {
+      return
+    }
+
+    each(styleSheet.cssRules, (cssRule: any) => {
+      let matchesEl = false
+
+      // Mobile safari will throw DOM Exception 12 error, need to try catch it.
+      try {
+        matchesEl = matchesSelector(node, cssRule.selectorText)
+      } catch (e) {
+        /* tslint:disable-next-line */
+      }
+
+      if (!matchesEl) return
+
+      unsorted.push({
+        selectorText: cssRule.selectorText,
+        style: cssRule.style,
+        styleSheetId,
+      })
+    })
+  })
+
+  const sorted: any[] = []
+  const priorities = map(unsorted, ({ selectorText }, i) => {
+    return cssPriority(selectorText, { position: i })
+  })
+  each(priorities.sort(cssPriority.compare), property => {
+    sorted.push(unsorted[property[5]])
+  })
+
+  return sorted
+}
+
+export function formatStyle(style: any) {
+  const ret: any = {}
+
+  for (let i = 0, len = style.length; i < len; i++) {
+    const name = style[i]
+
+    ret[name] = style[name] || trim(style.getPropertyValue(name))
+  }
+
+  return ret
+}
+
+const inlineStyleSheetIds = new Map()
+const inlineStyleNodeIds = new Map()
+
+export function getOrCreateInlineStyleSheetId(nodeId: any) {
+  let styleSheetId = inlineStyleSheetIds.get(nodeId)
+  if (styleSheetId) return styleSheetId
+
+  styleSheetId = getStyleSheetId()
+  inlineStyleSheetIds.set(nodeId, styleSheetId)
+  inlineStyleNodeIds.set(styleSheetId, nodeId)
+
+  return styleSheetId
+}
+
+export function getInlineStyleSheetId(nodeId: any) {
+  return inlineStyleSheetIds.get(nodeId)
+}
+
+export function getInlineStyleNodeId(styleSheetId: string) {
+  return inlineStyleNodeIds.get(styleSheetId)
+}
+
+const styleSheetTexts = new Map()
+
+export async function getStyleSheetText(styleSheetId: string, proxy = '') {
+  if (styleSheetTexts.get(styleSheetId)) {
+    return styleSheetTexts.get(styleSheetId)
+  }
+  for (let i = 0, len = document.styleSheets.length; i < len; i++) {
+    const styleSheet: any = document.styleSheets[i]
+    if (styleSheet.styleSheetId === styleSheetId) {
+      const text = await getTextContent(styleSheet.href, proxy)
+      styleSheetTexts.set(styleSheetId, text)
+      break
+    }
+  }
+  return styleSheetTexts.get(styleSheetId) || ''
+}
+
+function getStyleSheetId(sourceUrl = '') {
+  if (sourceUrl) {
+    return toStr(strHash(sourceUrl))
+  }
+
+  return createId()
+}

package/source/packages/cdp/src/lib/util.ts

@@ -0,0 +1,105 @@
+import uniqId from 'licia/uniqId'
+import random from 'licia/random'
+import startWith from 'licia/startWith'
+import Url from 'licia/Url'
+import convertBin from 'licia/convertBin'
+import axios from 'axios'
+import _type from 'licia/type'
+import _has from 'licia/has'
+
+const prefix = random(1000, 9999) + '.'
+
+export function createId() {
+  return uniqId(prefix)
+}
+
+export function getAbsoluteUrl(url: string) {
+  const a = document.createElement('a')
+  a.href = url
+  return a.href
+}
+
+export class ErrorWithCode extends Error {
+  code: number
+  constructor(code: number, message: string) {
+    super(message)
+    this.code = code
+
+    Object.setPrototypeOf(this, new.target.prototype)
+  }
+}
+
+export function createErr(code: number, message: string) {
+  return new ErrorWithCode(code, message)
+}
+
+export function getUrl() {
+  const href = location.href
+  if (startWith(href, 'about:')) {
+    return parent.location.href
+  }
+  return href
+}
+
+export function getOrigin() {
+  const origin = location.origin
+  if (origin === 'null') {
+    return parent.location.origin
+  }
+  return origin
+}
+
+export async function getTextContent(url: string, proxy = '') {
+  return await getContent(url, 'text', proxy)
+}
+
+export async function getBase64Content(url: string, proxy = '') {
+  return convertBin(await getContent(url, 'arraybuffer', proxy), 'base64')
+}
+
+export function getType(val: any, lowerCase: boolean) {
+  try {
+    return _type.apply(null, [val, lowerCase])
+  } catch (e) {
+    return 'Error'
+  }
+}
+
+export function has(obj: any, key: string) {
+  try {
+    return _has.apply(null, [obj, key])
+  } catch (e) {
+    return false
+  }
+}
+
+async function getContent(url: string, responseType: any, proxy = '') {
+  try {
+    const urlObj = new Url(url)
+    urlObj.setQuery('__chobitsu-hide__', 'true')
+    const result = await axios.get(urlObj.toString(), {
+      responseType,
+    })
+    return result.data
+  } catch (e) {
+    if (proxy) {
+      try {
+        const result = await axios.get(proxyUrl(proxy, url), {
+          responseType,
+        })
+        return await result.data
+      } catch (e) {
+        /* eslint-disable */
+      }
+    }
+  }
+
+  return responseType === 'arraybuffer' ? new ArrayBuffer(0) : ''
+}
+
+function proxyUrl(proxy: string, url: string) {
+  const urlObj = new Url(proxy)
+  urlObj.setQuery('url', url)
+  urlObj.setQuery('__chobitsu-hide__', 'true')
+  return urlObj.toString()
+}

package/source/packages/cdp/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "outDir": "./dist/cjs",
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "strictPropertyInitialization": false,
+    "removeComments": true,
+    "noUnusedLocals": true,
+    "target": "ES2020",
+    "declaration": true,
+    "esModuleInterop": true,
+    "lib": ["DOM", "ES2020"]
+  },
+  "include": ["src/**/*"]
+}