ct 0.10.8.114__py3-none-any.whl

cantools/scripts/doc.py

@@ -0,0 +1,239 @@
+"""
+### Usage: ctdoc [-w]
+
+### Options:
+    -h, --help  show this help message and exit
+    -w, --web   build web docs
+    -a, --auto  use auto mode (even with a plugin)
+    -o, --omit  omit any files from autodoc?
+
+Run from cantools root (contains setup.py, cantools/, README.md, etc), from root
+of a CT plugin, or from within a custom project. In cantools, builds docs for all
+frontend (js) and CLI (py) files. In plugin, docs consist of about file (about.txt),
+initialization config (init.py) and default frontend config (js/config.js). In custom
+(project) mode (when ctdoc is run somewhere other than cantools root or a plugin root,
+and additionally a configuration file, doc.cfg, is present), for each path declared in
+doc.cfg, include the docstring of each file specified, as well as the contents of
+about.txt (if present). Lastly, auto mode doesn't require configuration (doc.cfg) --
+instead, it recurses through the directories of your project, and includes the contents of
+any about.txt files, as well as (the top of) any py/js file that starts with a docstring.
+"""
+
+import os, json
+from optparse import OptionParser
+from cantools import __version__, config
+from cantools.util import read, write, log, cp, cmd
+from cantools.util.package import here
+
+WEB = []
+ALTS = {
+    "pubsub": os.path.join("pubsub", "__init__")
+}
+HERE = here()
+CUSTOM = os.path.isfile("doc.cfg") and read("doc.cfg")
+ISPLUGIN = not CUSTOM and HERE.startswith("ct") and HERE
+AUTO = HERE != "cantools" and not CUSTOM and not ISPLUGIN
+
+if not CUSTOM and not AUTO:
+    if ISPLUGIN:
+        JSPATH = os.path.join(HERE, "js")
+        BPATH = "."
+        ALTS["init"] = os.path.join(ISPLUGIN, "init")
+    else:
+        JSPATH = os.path.join(HERE, "CT")
+        BPATH = os.path.join(HERE, "scripts")
+
+def space(data):
+    return "    " + data.replace("\n", "\n    ")
+
+def bt2ds(t):
+    return t.split('"""')[1].strip()
+
+def dsBack(cmd):
+    cpath = os.path.join(BPATH, "%s.py"%(ALTS.get(cmd, cmd),))
+    log(cpath, 2)
+    bdata = read(cpath)
+    fdata = ISPLUGIN and space(bdata) or "## ct%s\n%s"%(cmd, bt2ds(bdata))
+    WEB[-1]["children"].append({
+        "name": cmd,
+        "content": fdata
+    })
+    return fdata
+
+def dsFront(mod, modname=None, iline=None):
+    modname = modname or "CT.%s"%(mod[:-3],)
+    iline = iline or (mod == "ct.js" and '<script src="/js/CT/ct.js"></script>' or 'CT.require("%s");'%(modname,))
+    log(modname, 2)
+    mdata = read(os.path.join(JSPATH, mod))
+    rdata = "\n".join([
+        "## %s"%(modname,),
+        "### Import line: '%s'"%(iline,),
+        (ISPLUGIN and mod == "config.js") and space(mdata) or mdata[3:].split("\n*/")[0]
+    ])
+    WEB[-1]["children"].append({
+        "name": mod,
+        "content": rdata
+    })
+    return rdata
+
+def back():
+    log("back", 1)
+    wobj = { "children": [] }
+    WEB.append(wobj)
+    f = []
+    if ISPLUGIN:
+        wobj["name"] = "Back (Init Config)"
+        fdata = [dsBack("init")]
+    else:
+        wobj["name"] = "Back (CLI)"
+        fdata = list(map(dsBack, ["init", "start", "deploy", "pubsub", "migrate", "index", "doc"]))
+    f.append("# %s"%(wobj["name"],))
+    f += fdata
+    return f
+
+def front():
+    log("front", 1)
+    wobj = { "children": [] }
+    WEB.append(wobj)
+    f = []
+    if not ISPLUGIN:
+        wobj["name"] = "Front (JS Library)"
+        plist = os.listdir(JSPATH)
+        plist.sort()
+        fdata = list(map(dsFront, [i for i in plist if i.endswith("js")]))
+    elif os.path.isfile(os.path.join(JSPATH, "config.js")):
+        wobj["name"] = "Front (JS Config)"
+        fdata = [dsFront("config.js", "core.config.%s"%(ISPLUGIN,), 'CT.require("core.config");')]
+    if "name" in wobj:
+        f.append("# %s"%(wobj["name"],))
+        f += fdata
+    return f
+
+def customChunk(path, fnames):
+    log("custom chunk: %s"%(path,), 1)
+    kids = []
+    wobj = { "name": path, "children": kids }
+    WEB.append(wobj)
+    f = ["## %s"%(path,)]
+    afile = os.path.join(path, "about.txt")
+    if os.path.isfile(afile):
+        adata = read(afile)
+        f.append(adata)
+        kids.append({
+            "name": "about",
+            "content": adata
+        })
+    for fname in fnames:
+        fdata = read(os.path.join(path, fname))
+        if fname == "config.js" or fname.startswith("ct.cfg"): # for non-local backend cfgs
+            fdata = space(fdata)
+        elif fname.endswith(".js"):
+            fdata = fdata[3:].split("\n*/")[0]
+        elif fname.endswith(".py"):
+            fdata = bt2ds(fdata)
+        f.append("### %s\n%s"%(fname, fdata))
+        kids.append({
+            "name": fname,
+            "content": fdata
+        })
+    return f
+
+frules = {
+    ".js": {
+        "top": "/*\n",
+        "bottom": "\n*/"
+    },
+    ".py": {
+        "top": '"""\n',
+        "bottom": '\n"""'
+    }
+}
+
+hashead = set()
+def sethead(curdir, data):
+    dirname = curdir.rsplit(os.path.sep, 1)[-1]
+    if dirname not in hashead:
+        hashead.add(dirname)
+        data.append("%s %s"%("#" * len(curdir.split(os.path.sep)), dirname))
+        wobj = { "name": dirname, "children": [] }
+        WEB.append(wobj)
+    return WEB[-1]["children"]
+
+OMIT = ""
+def autodoc(data, curdir, contents):
+    about = "about.txt"
+    if curdir != HERE: # probs revise this...
+        about = os.path.join(curdir, about)
+    if os.path.isfile(about):
+        kids = sethead(curdir, data)
+        adata = read(about)
+        data.append(adata)
+        kids.append({
+            "name": "about",
+            "content": adata
+        })
+    for fname in contents:
+        if fname in OMIT:
+            continue
+        for flag, rule in list(frules.items()):
+            if fname.endswith(flag):
+                fdata = read(os.path.join(curdir, fname))
+                if fdata.startswith(rule["top"]):
+                    kids = sethead(curdir, data)
+                    fstr = fdata[len(rule["top"]):].split(rule["bottom"])[0]
+                    data.append("%s# %s"%("#" * len(curdir.split(os.path.sep)), fname))
+                    data.append(fstr)
+                    kids.append({
+                        "name": fname,
+                        "content": fstr
+                    })
+
+def build():
+    global OMIT, HERE
+    parser = OptionParser("ctdoc [-w]")
+    parser.add_option("-w", "--web", action="store_true",
+        dest="web", default=False, help="build web docs")
+    parser.add_option("-a", "--auto", action="store_true",
+        dest="auto", default=False, help="use auto mode (even with a plugin)")
+    parser.add_option("-o", "--omit", dest="omit", default="",
+        help="omit any files from autodoc?")
+    options, args = parser.parse_args()
+    if not os.path.isdir(HERE):
+        HERE = input("module name? ")
+    log("building docs")
+    ds = []
+    if AUTO or options.auto:
+        OMIT = options.omit
+        for dirpath, dirnames, filenames in os.walk(HERE):
+            autodoc(ds, dirpath, filenames)
+    else:
+        abdata = (ISPLUGIN or CUSTOM) and "# %s\n%s"%(HERE, read("about.txt")) or config.about%(__version__,)
+        ds.append(abdata)
+        WEB.append({
+            "name": HERE,
+            "children": [{
+                "name": "about",
+                "content": abdata
+            }]
+        })
+        if CUSTOM:
+            for line in CUSTOM.split("\n"):
+                path, fnames = line.split(" = ")
+                ds.extend(customChunk(path, fnames.split("|")))
+        else:
+            ds.extend(back())
+            ds.extend(front())
+    log("writing data", important=True)
+    log("README.md", 1)
+    write("\n\n".join(ds), "README.md")
+    if options.web:
+        log("web docs enabled!", 1)
+        log("building docs web application", 2)
+        if not os.path.isdir("docs"):
+            cmd("ctinit docs -p ctdocs")
+        log("copying data", 2)
+        cp("core.data = %s;"%(json.dumps(WEB, indent=4),), os.path.join("docs", "js", "core", "data.js"))
+    log("goodbye")
+
+if __name__ == "__main__":
+    build()

cantools/scripts/index.py

@@ -0,0 +1,226 @@
+"""
+### Usage: ctindex [--mode=MODE] [--domain=DOMAIN] [--port=PORT] [--skip=SKIP]
+
+### Options:
+	-h, --help            show this help message and exit
+	-m MODE, --mode=MODE  may be: 'refcount' (default - count up all foreignkey
+	                      references for sort orders and such); 'index' (assign
+	                      each record a sequential integer index); 'urlsafekeys'
+	                      (update all key/keylist properties to use urlsafe keys
+	                      introduced in ct 0.8); 'cleanup' (delete zero-count
+	                      reference counters). Note regarding 'index' mode: it
+	                      _must_ happen remotely; it's generally unnecessary
+	                      unless you're trying to migrate an unindexed database
+	                      away from gae and need an index/key per record; it
+	                      should be invoked from _outside_ -- that's right,
+	                      outside -- of your project's directory (to avoid
+	                      loading up a bunch of google network tools that may be
+	                      crappy or cause issues outside of their normal
+	                      'dev_appserver' environment)
+	-d DOMAIN, --domain=DOMAIN
+	                      ('index' mode only) what's the domain of the target
+	                      server? (default: localhost)
+	-p PORT, --port=PORT  ('index' mode only) what's the port of the target
+	                      server? (default: 8080)
+	-s SKIP, --skip=SKIP  skip these tables ('index' mode only) - use '|' as
+	                      separator, such as 'table1|table2|table3' (default:
+	                      none)
+	-i INDEX, --index=INDEX
+                          start with this index ('index' mode only) (default: 0)
+
+As you can see, this script's behavior changes according to the backend of the target project.
+
+### dez
+Run this if your CTRefCount records get messed up for
+some reason. It will go through and recount everything
+(in the default 'refcount' mode -- the other modes,
+'urlsafekeys' and 'cleanup', are for migrating a CT-mediated
+database from an older deployment to CT 0.8 or newer).
+
+### gae
+Run this in 'index' mode on a database with lots of missing index values.
+"""
+
+from getpass import getpass
+from optparse import OptionParser
+from fyg.util import log, error, batch
+from cantools.db import get_schema, get_model, put_multi, delete_multi, unpad_key
+from cantools.web import fetch
+from cantools import config
+if config.web.server == "dez":
+	from cantools.db import session, func, refresh_counter
+
+try:
+    input = raw_input # py2/3 compatibility
+except NameError:
+    pass
+
+counts = { "_counters": 0 }
+RETRIES = 5
+
+#
+# dez
+#
+
+def get_keys(kind, reference):
+	log("acquiring %s (%s) keys"%(kind, reference), 1)
+	mod = get_model(kind)
+	q = session.query(getattr(mod, "key"))
+	qcount = q.count()
+	log("found %s"%(qcount,), 2)
+	fname, fkey = reference.split(".")
+	fmod = get_model(fname)
+	fprop = getattr(fmod, fkey)
+	sub = session.query(fprop, func.count("*").label("sub_count")).group_by(fprop).subquery()
+	q = q.join(sub, mod.key==getattr(sub.c, fkey))
+	newcount = q.count()
+	log("filtering out %s untargetted entities"%(qcount - newcount), 2)
+	qcount = newcount
+	log("returning %s keys"%(qcount,), 2)
+	return q.all()
+
+def refmap():
+	log("compiling back reference map")
+	rmap = {}
+	for tname, schema in list(get_schema().items()):
+		for pname, kinds in list(schema["_kinds"].items()):
+			reference = "%s.%s"%(tname, pname)
+			counts[reference] = 0
+			for kind in [k for k in kinds if k != "*"]: # skip wildcard for now
+				if kind not in rmap:
+					rmap[kind] = {}
+				rmap[kind][reference] = get_keys(kind, reference)
+	return rmap
+
+def do_batch(chunk, reference):
+	log("refreshing %s %s keys"%(len(chunk), reference), 1)
+	i = 0
+	rc = []
+	for item in chunk: # item is single-member tuple
+		rc.append(refresh_counter(item[0], reference))
+		i += 1
+		if not i % 100:
+			log("processed %s"%(i,), 3)
+	counts[reference] += len(chunk)
+	counts["_counters"] += len(rc)
+	log("refreshed %s total"%(counts[reference],), 2)
+	log("updated %s counters"%(counts["_counters"],), 2)
+	put_multi(rc)
+	log("saved", 2)
+
+def refcount():
+	log("indexing foreignkey references throughout database", important=True)
+	import model # load schema
+	for kind, references in list(refmap().items()):
+		log("processing table: %s"%(kind,), important=True)
+		for reference, keys in list(references.items()):
+			batch(keys, do_batch, reference)
+	tcount = sum(counts.values()) - counts["_counters"]
+	log("refreshed %s rows and updated %s counters"%(tcount, counts["_counters"]), important=True)
+
+#
+# gae
+#
+
+def _log_fetch(host, url, port):
+	res = fetch(host, url, port)
+	log(res)
+	return res
+
+def _index_kind(kind, host, port, pw, index):
+	log("indexing %s"%(kind,), important=True)
+	retry = 0
+	while "Error" in _log_fetch(host, "/_db?action=index&pw=%s&kind=%s&index=%s"%(pw, kind, index), port):
+		log("error indexing %s"%(kind,), important=True)
+		if retry == RETRIES:
+			error("tried %s times! sorry."%(retry,))
+		retry += 1
+		log("trying again (retry: %s)"%(retry,))
+
+def index(host, port, skips, index):
+	pw = getpass("what's the admin password? ")
+	log("indexing db at %s:%s"%(host, port), important=True)
+#	log(fetch(host, "/_db?action=index&pw=%s"%(pw,), port))
+	log("acquiring schema")
+	schema = fetch(host, "/_db?action=schema", port, ctjson=True)
+	for kind in schema:
+		if kind in skips:
+			log("skipping %s"%(kind,), important=True)
+		else:
+			_index_kind(kind, host, port, pw, index)
+
+#
+# url safety
+#
+def urlsafe():
+	log("updating key/keylist properties with urlsafe keys", important=True)
+	import model
+	schema = get_schema()
+	puts = []
+	for mod in schema:
+		mods = get_model(mod).query().all()
+		log("%s (%s)"%(mod, len(mods)), 1)
+		for m in mods:
+			if m.polytype != mod:
+				log("skipping! (%s != %s)"%(m.polytype, mod), 2)
+				continue
+			m.key = unpad_key(m.key.urlsafe())
+			for prop in schema[mod]["_kinds"]:
+				if schema[mod][prop] == "key":
+					setattr(m, prop, unpad_key(getattr(m, prop).urlsafe()))
+				else: # keylist
+					setattr(m, prop, [unpad_key(k.urlsafe()) for k in getattr(m, prop)])
+			puts.append(m)
+	log("saving records")
+	put_multi(puts)
+	log("updated %s keys"%(len(puts),), important=True)
+	if input("want to prune zero-count reference counters? (y/N)").lower().startswith("y"):
+		cleanup()
+
+def cleanup():
+	log("cleaning up zero-count reference counters", important=True)
+	from cantools.db import lookup
+	ctrz = lookup.CTRefCount.query(lookup.CTRefCount.count == 0).all()
+	log("deleting %s zero-count reference counters"%(len(ctrz),))
+	delete_multi(ctrz)
+	log("all gone!")
+
+def go():
+	parser = OptionParser("ctindex [--mode=MODE] [--domain=DOMAIN] [--port=PORT] [--skip=SKIP]")
+	parser.add_option("-m", "--mode", dest="mode", default="refcount",
+		help="may be: 'refcount' (default - count up all foreignkey references for sort "
+			"orders and such); 'index' (assign each record a sequential integer index); "
+			"'urlsafekeys' (update all key/keylist properties to use urlsafe keys "
+			"introduced in ct 0.8); 'cleanup' (delete zero-count reference counters). "
+			"Note regarding 'index' mode: it _must_ happen remotely; it's generally "
+			"unnecessary unless you're trying to migrate an unindexed database away from "
+			"gae and need an index/key per record; it should be invoked from _outside_ "
+			"-- that's right, outside -- of your project's directory (to avoid loading "
+			"up a bunch of google network tools that may be crappy or cause issues outside "
+			"of their normal 'dev_appserver' environment)")
+	parser.add_option("-d", "--domain", dest="domain", default="localhost",
+		help="('index' mode only) what's the domain of the target server? (default: localhost)")
+	parser.add_option("-p", "--port", dest="port", default="8080",
+		help="('index' mode only) what's the port of the target server? (default: 8080)")
+	parser.add_option("-s", "--skip", dest="skip", default="",
+		help="skip these tables ('index' mode only) - use '|' as separator, such as 'table1|table2|table3' (default: none)")
+	parser.add_option("-i", "--index", dest="index", default=0,
+		help="start with this index ('index' mode only) (default: 0)")
+	options, args = parser.parse_args()
+
+	log("mode: %s"%(options.mode,), important=True)
+	if options.mode == "refcount":
+		refcount()
+	elif options.mode == "index":
+		index(options.domain, int(options.port),
+			options.skip and options.skip.split("|") or [], options.index)
+	elif options.mode == "urlsafekeys":
+		urlsafe()
+	elif options.mode == "cleanup":
+		cleanup()
+	else:
+		error("unknown mode: %s"%(options.mode,))
+	log("goodbye")
+
+if __name__ == "__main__":
+	go()