ct 0.10.8.114__py3-none-any.whl

cantools/scripts/bench.py

@@ -0,0 +1,167 @@
+"""
+### Usage: ctbench [-bmsv] [--domain=DOMAIN] [--port=PORT] [--number=NUMBER] [--concurrency=CONCURRENCY] [--weight=WEIGHT] [--keys=KEYS]
+
+### Options:
+    -h, --help            show this help message and exit
+    -e, --encrypted       use encrypted (https) connection
+    -d DOMAIN, --domain=DOMAIN
+                          hostname (default: 0.0.0.0)
+    -p PORT, --port=PORT  port (default=8080)
+    -n NUMBER, --number=NUMBER
+                          number of requests (default=5000)
+    -c CONCURRENCY, --concurrency=CONCURRENCY
+                          number of connections (default=100)
+    -w WEIGHT, --weight=WEIGHT
+                          weight (size) of memcache blocks (default=10)
+    -k KEYS, --keys=KEYS  key count - number of memcache blocks (default=10)
+    -l PIPELINERS, --pipeliners=PIPELINERS
+                          number of pipelined requests (default=50)
+    -s, --static          random static (html/css) requests (default=False)
+    -b, --blobs           random blob requests (default=False)
+    -m, --memcache        random memcache requests (default=False)
+    -v, --validate        validate return values (default=False)
+"""
+
+import os, rel, json
+from optparse import OptionParser
+from dez.bench import MultiTester
+from cantools import config
+from cantools.util import cmd, log, read, error, token
+from cantools.web import post
+
+NUMBER = 5000
+CONCURRENCY = 100
+WEIGHT = 10
+KEYS = 10
+PIPES = 50
+
+class Bencher(object):
+	def __init__(self, options):
+		self.options = options
+		if options.blobs or options.memcache or options.static:
+			self.multi()
+		else:
+			self.single()
+
+	def single(self):
+		log("single (direct dbench) mode", important=True)
+		oz = self.options
+		rel.signal(2, rel.abort)
+		cstr = "dbench %s %s %s %s %s"%(oz.domain, oz.port, oz.number, oz.concurrency, oz.pipeliners)
+		if oz.encrypted:
+			cstr += " -e"
+		cmd(cstr)
+
+	def initmc(self):
+		oz = self.options
+		self.memcache = {}
+		self.mccount = 0
+		proto = oz.encrypted and "https" or "http"
+		for key in map(lambda k : "bencher%s"%(k,), range(oz.keys)):
+			val = self.memcache[key] = token(oz.weight)
+			post(oz.domain, "/_memcache", oz.port, {
+				"action": "set",
+				"key": key,
+				"value": val
+			}, proto, cb=self.regmc)
+		rel.signal(2, rel.abort)
+		rel.dispatch()
+
+	def initpaths(self):
+		oz = self.options
+		paths = self.paths = []
+		self.values = {}
+		if oz.blobs:
+			paths += self.regfiles("blob", binary=True)
+		if oz.static:
+			paths += self.regfiles("css")
+			paths += self.regfiles(config.build.web[config.mode] or config.build.web.compiled[config.mode],
+				"/", suffix=".html")
+		if oz.memcache:
+			paths += self.regfiles("memcache", "/_memcache?action=get&key=", self.memcache)
+		paths or error("nothing to request!")
+		log("randomizing requests among %s total items"%(len(paths),))
+
+	def validator(self, path, value, headers):
+		vlen = len(value)
+		clen = int(headers['content-length'])
+		if vlen != clen:
+			error("content-length mismatch! %s != %s"%(vlen, clen))
+		if "memcache" in path:
+			value = json.loads(value[1:])
+		elif "blob" not in path:
+			value = value.decode()
+		if path in self.values and self.values[path] != value:
+			log("returned:\n%s"%(value,), important=True)
+			log("should have returned:\n%s"%(self.values[path],), important=True)
+			error("return value mismatch: %s"%(path,))
+
+	def regmc(self, resp):
+		self.mccount += 1
+		log("regmc %s"%(self.mccount,))
+		if self.mccount == self.options.keys:
+			log("registered %s memcache keys"%(self.mccount,))
+			self._multi()
+
+	def regfiles(self, fpath, upath=None, files=None, binary=False, suffix=None):
+		upath = upath or "/%s/"%(fpath,)
+		fnames = files and files.keys() or os.listdir(fpath)
+		log("including %s %s files"%(len(fnames), fpath))
+		if self.options.validate:
+			paths = []
+			for fname in fnames:
+				if suffix and not fname.endswith(suffix):
+					continue
+				path = "%s%s"%(upath, fname)
+				paths.append(path)
+				self.values[path] = files and files[fname] or read(os.path.join(fpath, fname), binary=binary)
+		else:
+			paths = list(map(lambda n : "%s%s"%(upath, n), fnames))
+		return paths
+
+	def _multi(self):
+		oz = self.options
+		self.initpaths()
+		MultiTester(oz.domain, oz.port, self.paths, oz.number, oz.concurrency,
+			oz.pipeliners, oz.encrypted, oz.validate and self.validator).start()
+
+	def multi(self):
+		log("multi (randomized requests) mode", important=True)
+		if self.options.memcache:
+			self.initmc()
+		else:
+			self._multi()
+
+def run():
+	parser = OptionParser("ctbench [-bmsve] [--domain=DOMAIN] [--port=PORT] [--number=NUMBER] [--concurrency=CONCURRENCY] [--weight=WEIGHT] [--keys=KEYS] [--pipeliners=PIPELINERS]")
+	parser.add_option("-e", "--encrypted", action="store_true", dest="encrypted",
+		default=False, help="use encrypted (https) connection")
+	parser.add_option("-d", "--domain", dest="domain", default=config.web.host,
+		help="hostname (default: %s)"%(config.web.host,))
+	parser.add_option("-p", "--port", dest="port", default=config.web.port,
+		help="port (default=%s)"%(config.web.port,))
+	parser.add_option("-n", "--number", dest="number", default=NUMBER,
+		help="number of requests (default=%s)"%(NUMBER,))
+	parser.add_option("-c", "--concurrency", dest="concurrency", default=CONCURRENCY,
+		help="number of connections (default=%s)"%(CONCURRENCY,))
+	parser.add_option("-w", "--weight", dest="weight", default=WEIGHT,
+		help="weight (size) of memcache blocks (default=%s)"%(WEIGHT,))
+	parser.add_option("-k", "--keys", dest="keys", default=KEYS,
+		help="key count - number of memcache blocks (default=%s)"%(KEYS,))
+	parser.add_option("-l", "--pipeliners", dest="pipeliners", default=PIPES,
+		help="number of pipelined requests (default=%s)"%(PIPES,))
+	parser.add_option("-s", "--static", action="store_true", dest="static",
+		default=False, help="random static (html/css) requests (default=False)")
+	parser.add_option("-b", "--blobs", action="store_true", dest="blobs",
+		default=False, help="random blob requests (default=False)")
+	parser.add_option("-m", "--memcache", action="store_true", dest="memcache",
+		default=False, help="random memcache requests (default=False)")
+	parser.add_option("-v", "--validate", action="store_true", dest="validate",
+		default=False, help="validate return values (default=False)")
+	options = parser.parse_args()[0]
+	for key in ["port", "number", "concurrency", "weight", "keys", "pipeliners"]:
+		setattr(options, key, int(getattr(options, key)))
+	Bencher(options)
+
+if __name__ == "__main__":
+	run()

cantools/scripts/builder.py

@@ -0,0 +1,272 @@
+import subprocess, os, base64
+from cantools import config
+from cantools.util import log, error, read, write, mkdir
+
+# jsmin was removed from the cantools requirements to reduce
+# the maintenance burden for deployments on older systems
+# (often with some degree of broken package management) that
+# don't need to compile anything. so we check here instead.
+try:
+    from jsmin import jsmin
+    BUILDER_READY = True
+except:
+    BUILDER_READY = False
+
+pcfg = config.build.prod
+pwrap = {
+    "standard": "<script>%s</script>",
+    "closure": "<script>(function(){%s})();</script>"
+}[pcfg.closure and "closure" or "standard"]
+iwraps = {
+    "standard": "%s = %s || {}",
+    "closure": 'if (typeof %s == "undefined") %s = {}',
+    "closvar": 'if (typeof %s == "undefined") var %s = {}'
+}
+iwrap = iwraps[pcfg.closure and "closure" or "standard"]
+bwrap = "Function(atob(\"%s\"))();"
+
+def pwrapper(stxt):
+    if pcfg.b64:
+        stxt = bwrap%(base64.b64encode(stxt.encode()).decode(),)
+    return pwrap%(stxt,)
+
+def iwrapper(mod):
+    if config.build.prod.closure:
+        parts = mod.split(".")
+        if len(parts) == 2: # good enough? ....
+            return iwraps["closvar"]%(parts[1], parts[1])
+    return iwrap%(mod, mod)
+
+def nextQuote(text, lastIndex=0):
+    z = i = text.find('"', lastIndex)
+    while z > 0 and text[z-1] == "\\":
+        z -= 1
+    return ((i-z)%2 == 0) and i or nextQuote(text, i+1)
+
+def encodestrings(text):
+    start = nextQuote(text) + 1
+    while start != 0:
+        end = nextQuote(text, start)
+        if end == -1:
+            error("parse",
+                "unclosed quote: character %s"%(start-1,),
+                "this quote: %s"%(text[start:start+config.parse_error_segment_length],),
+                text)
+        word = ''.join(["\\%s"%(oct(ord(ch))[1:],) for ch in list(text[start:end])])
+        if "\\134" not in word: # don't re-escape!
+            text = text[:start] + word + text[end:]
+        start = nextQuote(text, start + len(word) + 1) + 1
+    return text
+
+def processhtml(html):
+    html = html.replace("{", "&#123").replace("}", "&#125").replace("</body>", "%s</body>"%(config.noscript,))
+    start = 0
+    while not start or html[start + config.js.offset + 1] == "/":
+        firststart = start = end = html.find(config.js.flag, start + 1)
+    js = []
+    while start != -1:
+        start += config.js.offset
+        end = html.find('"', start)
+        if end == -1:
+            error("no closing quote in this file: %s"%(html,))
+        js.append(html[start:end].strip("/"))
+        start = html.find(config.js.flag, end)
+    log("js: %s"%(js,), 1)
+    if start == end:
+        return html, ""
+    return html[:firststart] + "{jsspot}" + html[end+config.js.endoffset:], js
+
+def compress(html):
+    log("compressing html", 1)
+    newhtml = html.replace("\n", " ").replace("\t", " ")
+    while "  " in newhtml:
+        newhtml = newhtml.replace("  ", " ")
+    newhtml = newhtml.replace("> <", "><")
+    log("orig: %s. new: %s"%(len(html), len(newhtml)), 2)
+    return newhtml
+
+def bfiles(dirname, fnames):
+    return [fname for fname in fnames if os.path.isfile(os.path.join(dirname, fname)) and fname != ".svn" and not fname.endswith("~") and not "_old." in fname]
+
+def tryinit(iline, inits, prefixes):
+    if iline not in inits:
+        inits.add(iline)
+        prefixes.append(iline)
+
+fragz = set()
+def frag(path):
+    if '"' in path:
+        path = path.split('"')[0]
+        for root, dirs, files in os.walk(path, followlinks=True):
+            for sub in files:
+                if sub.endswith(".js"):
+                    fp = os.path.join(root, sub)
+                    log("fragment identified: %s"%(fp,), 1)
+                    fragz.add(fp)
+    else:
+        fragz.add(path)
+
+def require(line, jspaths, block, inits, admin_ct_path=None):
+    dynamic = False
+    if line.startswith("CT.scriptImport("):
+        rline = line[17:].split(line[16])[0]
+        if line[16] not in "'\"" or rline.startswith("http"):
+            log("skipping scriptImport: %s"%(line,), important=True)
+            return block
+        dynamic = True
+    else:
+        rline = line.split('require(')[1][1:].split(");")[0].strip(")")
+        if rline.endswith('"skip"'):
+            log("skipping require: %s"%(line,), important=True)
+            return block
+        elif rline.endswith(", true"):
+            dynamic = True
+            rline = rline.split(", ")[0]
+        rline = rline[:-1]
+    rsplit = rline.split(".")
+    log("module %s"%(rline,), important=True)
+    jspath = os.path.join(config.js.path, *rsplit) + ".js"
+    log("path %s"%(jspath,))
+    log("dynamic %s"%(dynamic,))
+    if jspath not in jspaths:
+        if dynamic:
+            frag(jspath)
+        else:
+            prefixes = []
+            fullp = "window"
+            for rword in rsplit:
+                if rword[0].isalpha():
+                    fullp = ".".join([fullp, rword])
+                else:
+                    fullp = "%s[%s]"%(fullp, rword)
+                tryinit(iwrapper(fullp), inits, prefixes)
+            pblock = ";".join(prefixes)
+            if pblock:
+                if config.build.prod.closure:
+                    pblock = pblock.replace("window.", "")
+                jspaths.append(pblock)
+            block = block.replace(line, "%s;%s;%s"%(pblock, line.replace("require", "regmod"),
+                processjs(jspath, jspaths, inits, admin_ct_path)), 1)
+    return block
+
+def processjs(path, jspaths=[], inits=set(), admin_ct_path=None):
+    log("processing js: %s"%(path,), 1)
+    p = path.split("#")[0] # potentially modded to locate file for prod (path remains the same for static)
+    if admin_ct_path: # admin pages
+        if path.startswith(config.js.path): # such as /js/CT/ct.js
+            p = admin_ct_path + path[len(config.js.path):]
+        else: # regular admin pages (/memcache/mc.js)
+            p = os.path.join(os.path.abspath(os.curdir), "dynamic", path)
+    block = read(p)
+    for line in [bit for bit in block.split("\n") if not bit.startswith("//")]:
+        for flag in ["CT.require(", "CT.scriptImport("]:
+            if flag in line and line[line.index(flag) - 1] != "'":
+                rline = line.strip().split(flag)[1]
+                if rline[0] != "\\" and rline[1] not in ".,": # require() is embedded in text. dot is weird.
+                    block = require("%s%s"%(flag, rline),
+                        jspaths, block, inits, admin_ct_path)
+    jspaths.append(path)
+    return "%s;\n"%(block,)
+
+def compilejs(js, admin_ct_path=None):
+    jsblock = ""
+    jspaths = []
+    inits = set([iwrapper("window.CT")]) # already initialized
+    for p in js:
+        jsblock += processjs(p, jspaths, inits, admin_ct_path)
+    return jspaths, jsblock
+
+def checkdir(p, recursive=False):
+    if not os.path.isdir(p):
+        mkdir(p, recursive)
+
+def remerge(txt, js):
+    return txt.format(jsspot=js).replace("&#123", "{").replace("&#125", "}")
+
+def build_frags(mode="web", admin_ct_path=None):
+    log("Compiling Dynamically-Referenced Fragments", important=True)
+    base = config.build[mode].compiled.production
+    if config.build.include:
+        log("Including Config-Specified Modules")
+        for p in config.build.include:
+            log("include: %s"%(p,), 1)
+            fragz.add(p)
+    fragged = set()
+    def build_frag(frag):
+        notjs = frag in config.build.notjs
+        if notjs:
+            log("not js: %s"%(frag,))
+            block = read(frag, binary=True)
+        else:
+            block = processjs(frag, admin_ct_path=admin_ct_path)
+        path = os.path.join(base, frag[len(config.js.path)+1:])
+        checkdir(path.rsplit("/", 1)[0], True)
+        if notjs or frag in config.build.exclude:
+            log("path excluded -- skipping compression/obfuscation", 2)
+        else:
+            log("mangling", 2)
+            block = jsmin(block)
+        write(block, path, binary=notjs)
+    while len(fragged) is not len(fragz):
+        fcopy = list(filter(lambda f : f not in fragged, fragz))
+        fragged.update(fcopy)
+        list(map(build_frag, fcopy))
+
+def build(admin_ct_path, dirname, fnames):
+    """
+    This parses an html file, squishes together the javascript, scans
+    through for dynamic imports (CT.require statements), injects modules
+    wherever necessary, and sticks the result in a big <script> tag.
+    """
+    conf = admin_ct_path and config.build.admin or config.build.web
+    todir_stat = dirname.replace(conf.dynamic, conf.compiled.static)
+    todir_prod = dirname.replace(conf.dynamic, conf.compiled.production)
+    log("Target Static Directory: %s"%(todir_stat,), important=True)
+    log("Target Production Directory: %s"%(todir_prod,))
+    checkdir(todir_stat)
+    checkdir(todir_prod)
+    for fname in bfiles(dirname, fnames):
+        frompath = os.path.join(dirname, fname)
+        topath_stat = os.path.join(todir_stat, fname)
+        topath_prod = os.path.join(todir_prod, fname)
+        data = read(frompath)
+        if "fonts" in dirname or not fname.endswith(".html"):
+            log('copying non-html file (%s)'%(fname,), 1)
+        else:
+            log("processing html: %s"%(frompath,))
+            txt, js = processhtml(data)
+            if js:
+                jspaths, jsblock = compilejs(js, admin_ct_path)
+                log('writing static: %s -> %s'%(frompath, topath_stat), important=True)
+                write(remerge(txt, '\n'.join([p.split("#")[0].endswith("js") and '<script src="/%s"></script>'%(p,) or '<script>%s</script>'%(p,) for p in jspaths])), topath_stat)
+                log('writing production: %s -> %s'%(frompath, topath_prod))
+                jsb = jsblock.replace('"_encode": false,', '"_encode": true,').replace("CT.log._silent = false;", "CT.log._silent = true;")
+                if config.customscrambler:
+                    jsb += '; CT.net.setScrambler("%s");'%(config.scrambler,)
+                write(remerge(compress(txt), pwrapper(jsmin(jsb))), topath_prod)
+                continue
+            else:
+                log('copying to prod/stat unmodified (%s)'%(fname,), important=True)
+        write(data, topath_stat)
+        write(data, topath_prod)
+    for fname in [f for f in fnames if os.path.isdir(os.path.join(dirname, f))]:
+        for dname, dirnames, filenames in os.walk(os.path.join(dirname, fname)):
+            build(admin_ct_path, dname, filenames + dirnames)
+
+def silence_warnings():
+    from ply import yacc, lex
+    def quiet(*args, **kwargs):
+        pass
+    yacc.PlyLogger.warning = quiet
+    lex.PlyLogger.warning = quiet
+
+def build_all(mode="web", admin_ct_path=None):
+    if not BUILDER_READY:
+        error("can't build - please install jsmin >= 2.2.2")
+    #silence_warnings() # is this unnecessary now.... maybe...
+    for dirname, dirnames, filenames in os.walk(config.build[mode].dynamic):
+        build(admin_ct_path, dirname, filenames + dirnames)
+    build_frags(mode, admin_ct_path)
+
+if __name__ == "__main__":
+    build_all()

cantools/scripts/deploy.py

@@ -0,0 +1,154 @@
+"""
+### Usage: ctdeploy [-d|s|p] [-un] [--js_path=PATH]
+
+### Options:
+
+    -h, --help            show this help message and exit
+    -d, --dynamic         switch to dynamic (development) mode
+    -s, --static          switch to static (debug) mode
+    -p, --production      switch to production (garbled) mode
+    -u, --upload          uploads project in specified mode and then (if
+                          gae) switches back to dynamic (development) mode
+    -n, --no_build        skip compilation step
+    -j JS_PATH, --js_path=JS_PATH
+                          set javascript path (default=js)
+
+### Supports 3 modes:
+    - dynamic (files live in html)
+      - normal development files
+        - dynamic imports throughout
+      - original files are loaded ad-hoc
+        - chrome debugger plays nice
+      - no wire encryption
+      - all imports lazy
+    - static (files live in html-static)
+      - compiler builds same html files
+        - script imports in head
+        - otherwise unmodified source files
+      - original files are directly referenced
+        - chrome debugger prefers
+      - no wire encryption
+      - all hard requirements loaded in head
+        - lazy-designated imports still lazy
+    - production (files live in html-production)
+      - all code is compiled in head
+        - html is compressed
+        - javascript is minified and mangled
+      - original code is unrecognizable
+        - chrome debugger almost useless
+      - wire encryption
+      - designated lazy imports (indicated by second bool arg to CT.require)
+
+Generates fresh 'static' and 'production' files (from 'development' source files in 'html' on every run, unless -n [or --no_build] flag is used). Mode is established in the app.yaml file, which routes requests to the appropriate directory, and the ct.cfg file, which determines backend behavior, especially regarding encryption.
+"""
+
+import subprocess, os
+from cantools import config, __version__
+from cantools.util import log, error, read, write, cmd
+from cantools.util.package import vpush
+from .builder import build_all
+
+try:
+    input = raw_input # py2/3 compatibility
+except NameError:
+    pass
+
+def doyaml(mode):
+    log("switching to %s mode"%(mode,))
+    lines = read(config.yaml.path, lines=True)
+    f = open(config.yaml.path, 'w')
+    m = None
+    for line in lines:
+        if line.startswith(config.yaml.start):
+            m = line[len(config.yaml.start):].strip()
+        elif line.startswith(config.yaml.end):
+            m = None
+        elif m == mode:
+            line = line.strip("#")
+        elif m:
+            line = "#%s"%(line.strip("#"),)
+        f.write(line)
+    f.close()
+
+def setmode(mode):
+    doyaml(mode) # support other backends beyond app engine
+    ctpy = read(config.py.path)
+    isprod = mode == "production"
+    # set encode
+    ctpy = ctpy.replace(config.py.enc%(str(not isprod),),
+        config.py.enc%(str(isprod),))
+    # set mode
+    for m in ["dynamic", "static", "production"]:
+        if m != mode:
+            ctpy = ctpy.replace(config.py.mode%(m,), config.py.mode%(mode,))
+    write(ctpy, config.py.path)
+
+def buildox():
+    cmd("ctinit -a")
+    cmd("ctdoc -w")
+    os.chdir("docs")
+    cmd("ctdeploy -p")
+    os.chdir("..")
+
+def upit():
+    if config.web.server == "gae": # switches back to -d mode [ultra deprecated]
+        log("uploading files")
+        subprocess.call('appcfg.py update . --no_precompilation --noauth_local_webserver', shell=True)
+        setmode("dynamic")
+        return
+    upop = input("upload with git or svn or cancel ([G]it/[s]vn/[c]ancel)? ").lower()[:1]
+    if upop == "c":
+        return
+    if upop == "s":
+        cmd('svn commit html-production/ -m"prod"')
+    else: # default git
+        cmd("git add html-production/")
+        cmd('git commit -m "prod"')
+        cmd("git push")
+
+def run():
+    from optparse import OptionParser
+    parser = OptionParser("ctdeploy [-d|s|p] [-un] [--js_path=PATH]")
+    parser.add_option("-d", "--dynamic", action="store_true", dest="dynamic",
+        default=False, help="switch to dynamic (development) mode")
+    parser.add_option("-s", "--static", action="store_true", dest="static",
+        default=False, help="switch to static (debug) mode")
+    parser.add_option("-p", "--production", action="store_true", dest="production",
+        default=False, help="switch to production (garbled) mode")
+    parser.add_option("-u", "--upload", action="store_true", dest="upload", default=False,
+        help="uploads project in specified mode and then (if gae) switches back to dynamic (development) mode")
+    parser.add_option("-n", "--no_build", action="store_true",
+        dest="no_build", default=False, help="skip compilation step")
+    parser.add_option("-j", "--js_path", dest="js_path", default=config.js.path,
+        help="set javascript path (default=%s)"%(config.js.path,))
+    parser.add_option("-v", "--vpush", action="store_true", dest="vpush",
+        default=False, help="push a new version of cantools [ct dev only]")
+    options, args = parser.parse_args()
+
+    if options.vpush:
+        vpush("cantools", "ct", __version__, buildox)
+    else:
+        mode = options.dynamic and "dynamic" or options.static and "static" or options.production and "production"
+        if not mode:
+            error("no mode specified")
+
+        # 0) set js path
+        if options.js_path != config.js.path:
+            log("setting js path to: %s"%(options.js_path,))
+            config.js.update("path", options.js_path)
+
+        # 1) build static/production files
+        if not options.no_build:
+            build_all()
+
+        # 2) switch to specified mode
+        setmode(mode)
+
+        # 3) if -u, upload project
+        if options.upload:
+            upit()
+
+    log("goodbye")
+
+if __name__ == "__main__":
+    run()