databae 0.1__py3-none-any.whl

databae/__init__.py

@@ -0,0 +1,5 @@
+from .model import *
+from .lookup import inc_counter, dec_counter, refresh_counter, refcount_subq
+from . import lookup
+
+__version__ = "0.1"

databae/config.py

@@ -0,0 +1,17 @@
+from fyg import Config
+
+config = Config({
+	"cache": True,
+	"refcount": False,
+	"main": "sqlite:///data.db",
+	"test": "sqlite:///data_test.db",
+	"blob": "blob",
+	"alter": False, # add new columns to tables - sqlite only!
+	"echo": False,
+	"pool": {
+		"null": True,
+		"size": 10,
+		"recycle": 30,
+		"overflow": 20
+	}
+})

databae/edit.py

@@ -0,0 +1,46 @@
+from datetime import datetime
+from .util import get_model, blobify
+from .properties import KeyWrapper
+from .getters import get
+
+def _trans_key(val):
+    return KeyWrapper(val)
+
+def _trans_keylist(val):
+    return [KeyWrapper(v) for v in val]
+
+def _trans_datetime(val):
+    return isinstance(val, datetime) and val or datetime.strptime(val, "%Y-%m-%d %H:%M:%S")
+
+ETRANS = {
+    "key": _trans_key,
+    "keylist": _trans_keylist,
+    "datetime": _trans_datetime
+}
+def add_edit_transformation(ptype, func):
+    ETRANS[ptype] = func
+
+def edit(data, session=None, blobifier=None):
+    haskey = "key" in data
+    extant = haskey and get(data["key"], session)
+    blobifier and blobify(data, blobifier, extant)
+    ent = extant or get_model(data["modelName"])()
+    if haskey: # what about nokey uniqueness checks?
+        editFailure = ent.beforeedit(data)
+        if editFailure:
+            return editFailure
+    for propname, val in list(data.items()):
+        if propname in ent._schema:
+            if val:
+                proptype = ent._schema[propname]
+                if hasattr(ent, "_pre_trans_%s"%(propname,)):
+                    val = getattr(ent, "_pre_trans_%s"%(propname,))(val)
+                if proptype in ETRANS:
+                    val = ETRANS[proptype](val)
+                if hasattr(ent, "_trans_%s"%(propname,)):
+                    val = getattr(ent, "_trans_%s"%(propname,))(val)
+            setattr(ent, propname, val)
+    if not haskey and hasattr(ent, "oncreate"):
+        ent.oncreate()
+    ent.put()
+    return ent

databae/getters.py

@@ -0,0 +1,140 @@
+import os, json, magic
+from base64 import b64decode
+from datetime import datetime
+from sqlalchemy import func
+from six import string_types
+from .util import *
+from .config import config
+
+def _apply_filter(query, key, obj, modelName, joinz):
+    from .properties import KeyWrapper
+    val = obj["value"]
+    comp = obj["comparator"]
+    if "." in key:
+        altname, key = key.split(".")
+        mod = get_model(altname)
+        schema = get_schema(altname)
+        if altname not in joinz:
+            _join(modelName, altname, joinz, query)
+    else:
+        schema = get_schema(modelName)
+        mod = get_model(modelName)
+    prop = getattr(mod, key)
+    ptype = schema[key]
+    if ptype == "key" and not isinstance(val, KeyWrapper):
+        val = KeyWrapper(val)
+    elif ptype == "datetime" and not isinstance(val, datetime):
+        val = datetime.strptime(val, "%Y-%m-%d %H:%M:%S")
+    if comp == "like":
+        query.filter(func.lower(prop).like(val.lower()))
+    elif comp == "contains":
+        query.filter(prop.contains(val))
+    elif comp == "lacks":
+        query.filter(~prop.contains(val))
+    elif comp == "mod":
+        query.filter(prop % val == 0)
+    elif comp.startswith("near"): # lat/lng
+        mdist = 10
+        if "_" in comp:
+            mdist = int(comp.split("_")[1])
+        rad = mdist / 69.11
+        query.filter(prop > val - rad)
+        query.filter(prop < val + rad)
+    elif comp == "!=" and type(val) == list: # allow multiple exclusions...
+        for item in val:
+            query.filter(operators[comp](prop, item))
+    else:
+        query.filter(operators[comp](prop, val))
+
+def _join(modelName, altname, joinz, query):
+    joinz.add(altname)
+    altschema = get_schema(altname)
+    if modelName in altschema["_kinds"]:
+        mod1 = get_model(modelName)
+        mod2 = get_model(altname)
+        kinds = altschema["_kinds"][modelName]
+    else:
+        mod1 = get_model(altname)
+        mod2 = get_model(modelName)
+        kinds = get_schema(modelName)["_kinds"][altname]
+    for kind in kinds:
+        if hasattr(mod2, kind):
+            mod2attr = getattr(mod2, kind)
+            break
+    query.join(get_model(altname), mod1.key == mod2attr)
+
+def get_page(modelName, limit, offset, order='index', filters={}, session=None, count=False, exporter="export"):
+    query = get_model(modelName).query(session=session)
+    joinz = set()
+    for key, obj in list(filters.items()):
+        _apply_filter(query, key, obj, modelName, joinz)
+    if "." in order:
+        mod, attr = order.split(".")[-2:]
+        if joinz or not get_model(attr): # skip refcount shortcut if filtering on joined table
+            desc = False
+            if mod.startswith("-"):
+                desc = True
+                mod = mod[1:]
+            order = getattr(get_model(mod), attr)
+            if desc:
+                order = -order
+            mod not in joinz and _join(modelName, mod, joinz, query)
+    query.order(order)
+    if count:
+        return query.count()
+    return [getattr(d, exporter)() for d in query.fetch(limit, offset)]
+
+def getall(entity=None, query=None, keys_only=False, session=None):
+    if query:
+        res = query.all()
+    elif entity:
+        res = entity.query(session=session).all()
+    if keys_only: # TODO: query for keys. for now, do with query.
+        return [r.key for r in res]
+    return res
+
+def b64d(compkey):
+    return b64decode(pad_key(compkey)).decode()
+
+def key2data(b64compkey):
+    if not isinstance(b64compkey, string_types):
+        b64compkey = b64compkey.urlsafe()
+    return json.loads(b64d(b64compkey))
+
+def get(b64compkey, session=None):
+    try:
+        compkey = key2data(b64compkey)
+    except:
+        from fyg.util import error
+        error("bad key: %s"%(b64compkey,))
+    return modelsubs[compkey["model"]].query(session=session).query.get(compkey["index"])
+
+def get_multi(b64keys, session=None):
+    # b64keys can be Key instances or b64 key strings
+    if b64keys and not isinstance(b64keys[0], string_types):
+        b64keys = [k.urlsafe() for k in b64keys]
+    keys = [json.loads(b64d(k)) for k in b64keys]
+    ents = {}
+    res = {}
+    for k in keys:
+        mod = k["model"]
+        if mod not in ents:
+            ents[mod] = {
+                "model": modelsubs[mod],
+                "indices": []
+            }
+        ents[mod]["indices"].append(k["index"])
+    for key, val in list(ents.items()):
+        mod = val["model"]
+        for r in mod.query(session=session).filter(mod.index.in_(val["indices"])).all():
+            res[r.id()] = r
+    return [res[k] for k in b64keys]
+
+def get_blobs(variety):
+    bp = config.blob
+    bz = []
+    for f in next(os.walk(bp))[-1]:
+        fp = os.path.join(bp, f)
+        if variety in magic.from_file(fp):
+            bz.append("/%s"%(fp,))
+    return bz

databae/lookup.py

@@ -0,0 +1,59 @@
+from sqlalchemy import func
+from .model import seshman, get_model, put_multi, ForeignKey, String, Integer, ModelBase
+
+class CTRefCount(ModelBase):
+    target = ForeignKey()      # instance pointed at
+    reference = String()       # table.property (such as 'building.owner')
+    count = Integer(default=0) # number of pointers
+
+    def mydata(self):
+        return {
+            "target": self.target.urlsafe(),
+            "reference": self.reference,
+            "count": self.count
+        }
+
+    def inc(self, amount=1):
+        self.count += amount
+
+    def dec(self, amount=1):
+        self.count -= amount
+
+    def refresh(self):
+        fname, fkey = self.reference.split(".")
+        fmod = get_model(fname)
+        self.count = fmod.query(getattr(fmod, fkey) == self.target).count()
+
+def ref_counter(target, reference, session=None):
+    return CTRefCount.query(CTRefCount.target == target,
+        CTRefCount.reference == reference, session=session
+    ).get() or CTRefCount(target=target, reference=reference)
+
+def inc_counter(target, reference, amount=1, session=None):
+    rc = ref_counter(target, reference, session)
+    rc.inc(amount)
+    return rc
+
+def dec_counter(target, reference, amount=1, session=None):
+    rc = ref_counter(target, reference, session)
+    rc.dec(amount)
+    return rc
+
+def refresh_counter(target, reference, session=None):
+    rc = ref_counter(target, reference, session)
+    rc.refresh()
+    return rc
+
+def refcount_subq(reference, session=None):
+    session = session or seshman.get()
+    if reference.count(".") == 2:
+        rp = reference.split(".")
+        t1 = get_model(rp[1])
+        t2 = get_model(rp[2])
+        tlink = getattr(t1, rp[2])
+        return session.query(CTRefCount.target, CTRefCount.count,
+            tlink).filter(CTRefCount.reference == ".".join(rp[:2])).join(t1,
+            CTRefCount.target == t1.key).join(t2,
+            t2.key == tlink).group_by(tlink).with_entities(t2.key.label("target"),
+            func.sum(CTRefCount.count).label("count")).subquery()
+    return session.query(CTRefCount.target, CTRefCount.count).filter(CTRefCount.reference == reference).subquery()

databae/model.py

@@ -0,0 +1,193 @@
+from sqlalchemy import orm
+from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta
+from fyg.util import log, error
+from six import with_metaclass
+from .query import *
+
+def choice_validator(choices):
+    def cval(s, k, v):
+        if v not in choices:
+            error("can't set %s! %s not in %s"%(k, v, choices))
+        return v
+    return cval
+
+class CTMeta(DeclarativeMeta):
+    def query(cls, *args, **kwargs):
+        return Query(cls, *args, **kwargs)
+
+    def __new__(cls, name, bases, attrs):
+        lname = name.lower()
+        attrs["__tablename__"] = lname
+        if lname != "modelbase":
+            attrs["__mapper_args__"] = {
+                "polymorphic_identity": lname
+            }
+            attrs["index"] = sqlForeignKey(bases[0], primary_key=True)
+            if "label" not in attrs:
+                for label in ["name", "title", "topic"]:
+                    if label in attrs:
+                        attrs["label"] = label
+                        break
+            schema = attrs["_schema"] = merge_schemas(bases, attrs.get("label"))
+            for key, val in list(attrs.items()):
+                if getattr(val, "_ct_type", None):
+                    schema[key] = val._ct_type
+                    if val._ct_type.startswith("key"):
+                        schema["_kinds"][key] = val._kinds
+                if getattr(val, "choices", None):
+                    attrs["%s_validator"%(key,)] = sqlalchemy.orm.validates(key)(choice_validator(val.choices))
+        modelsubs[lname] = super(CTMeta, cls).__new__(cls, name, bases, attrs)
+        modelsubs[lname].__name__ = lname
+        return modelsubs[lname]
+
+sa_dbase = declarative_base(metadata=metadata)
+
+class ModelBase(with_metaclass(CTMeta, sa_dbase)):
+    index = Integer(primary_key=True)
+    polytype = String()
+    key = CompositeKey()
+    __mapper_args__ = {
+        "polymorphic_on": polytype,
+        "polymorphic_identity": "modelbase",
+        "with_polymorphic": "*"
+    }
+    label = "key"
+    _data_omit = []
+    _unique_cols = []
+
+    def __init__(self, *args, **kwargs):
+        sa_dbase.__init__(self, *args, **kwargs)
+        self._defaults()
+        self._init()
+
+    @orm.reconstructor
+    def _init(self):
+        self._name = "%s(%s)"%(self.polytype, getattr(self, self.label))
+        self._orig_fkeys = {}
+        for prop in self._schema["_kinds"]:
+            self._orig_fkeys[prop] = getattr(self, prop)
+
+    def _defaults(self):
+        for prop in self._schema["_kinds"]:
+            if getattr(self, prop, None) is None:
+                if self._schema[prop].endswith("list"):
+                    val = []
+                else:
+                    val = KeyWrapper()
+                setattr(self, prop, val)
+        self.key = KeyWrapper()
+        for key, val in list(self.__class__.__dict__.items()):
+            if getattr(self, key, None) is None and getattr(val, "_default", None) is not None:
+                setattr(self, key, val._default)
+
+    def __eq__(self, other):
+        return self.id() == (other and hasattr(other, "id") and other.id())
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        return self.key.__hash__()
+
+    def put(self, session=None):
+        try:
+            put_multi([self], session)
+        except Exception as e:
+            handle_error(e, session, self.polytype, "has no column named")
+            log("retrying put operation")
+            put_multi([self], session)
+
+    def otherwith(self, prop, val):
+        k = self._schema[prop]
+        c = self.__class__
+        col = getattr(c, prop)
+        q = c.query(c.key != self.key)
+        print("checking", c.__name__, "for", prop, k, "=", val)
+        if k.endswith("list"):
+            for v in val:
+                q.filter(col.contains(v))
+        else:
+            q.filter(col == val)
+        return q.get()
+
+    def beforeedit(self, edits):
+        for prop in edits:
+            if prop in self._unique_cols and self.otherwith(prop, edits[prop]):
+                print(prop, "conflict!\n\n")
+                return "%s must be unique"%(prop,)
+
+    def beforeremove(self, session):
+        pass
+
+    def afterremove(self, session):
+        pass
+
+    def rm(self, commit=True, session=None):
+        session = session or seshman.get()
+        self.beforeremove(session)
+        session.delete(self)
+        commit and session.commit()
+        self.afterremove(session)
+
+    def collection(self, entity_model, property_name=None, fetch=True, keys_only=False, data=False):
+        if isinstance(entity_model, str):
+            entity_model = get_model(entity_model)
+        q = entity_model.query(getattr(entity_model, property_name or self.polytype) == self.key)
+        if not fetch:
+            return q
+        if not data:
+            return q.fetch(1000, keys_only=keys_only)
+        return [d.data() for d in q.fetch(1000)]
+
+    def modeltype(self):
+        return self.__tablename__
+
+    def id(self):
+        return self.key.urlsafe()
+
+    def _has_complete_key(self):
+        return bool(self.id())
+
+    def mydata(self, isexport=False):
+        cols = {}
+        for key, prop in list(self._schema.items()):
+            if not isexport and key in self._data_omit:
+                continue
+            if not key.startswith("_"):
+                val = getattr(self, key)
+                if prop.startswith("key"):
+                    if type(val) is list:
+                        val = [v.urlsafe() for v in val]
+                    elif hasattr(val, "urlsafe"):
+                        val = val.urlsafe()
+                elif prop == "blob" and hasattr(val, "urlsafe"):
+                    val = val.urlsafe()
+                elif val and prop == "datetime":
+                    val = str(val)[:19]
+                elif prop in ["string", "text"]:
+                    val = val or ""
+                cols[key] = val
+        return cols
+
+    def labeler(self):
+        if self.label == "key":
+            return self.id()
+        return getattr(self, self.label) or "%s %s"%(self.polytype, self.index)
+
+    def _basic(self, d):
+        d["key"] = self.id()
+        d["index"] = self.index
+        d["modelName"] = self.polytype
+        d["_label"] = self.label
+        d["label"] = self.labeler()
+        return d
+
+    def data(self):
+        return self._basic(self.mydata())
+
+    def export(self):
+        return self._basic(ModelBase.mydata(self, True))
+
+class TimeStampedBase(ModelBase):
+    created = DateTime(auto_now_add=True)
+    modified = DateTime(auto_now=True)

databae/properties.py

@@ -0,0 +1,274 @@
+import os, json, sqlalchemy
+from .getters import get
+from .config import config
+
+primis = ["Integer", "Float", "Boolean", "Text", "Date", "Time"]
+
+class DynamicType(sqlalchemy.TypeDecorator):
+	cache_ok = config.cache
+
+	def __init__(self, *args, **kwargs):
+		self.choices = kwargs.pop("choices", None)
+		sqlalchemy.TypeDecorator.__init__(self, *args, **kwargs)
+
+class StringType(DynamicType):
+	def __init__(self, *args, **kwargs):
+		# '500' (len) required for MySQL VARCHAR
+		DynamicType.__init__(self, 500, *args, **kwargs)
+
+def basicType(colClass, baseType=DynamicType):
+	cname = colClass.__name__
+	attrs = { "impl": colClass }
+	if config.cache and cname in primis:
+		attrs["cache_ok"] = True
+	return type("%s"%(cname,), (baseType,), attrs)
+
+def _col(colClass, *args, **kwargs):
+	cargs = {}
+	if "primary_key" in kwargs:
+		cargs["primary_key"] = kwargs.pop("primary_key")
+	default = kwargs.pop("default", None)
+	if kwargs.pop("repeated", None):
+		isKey = kwargs["isKey"] = colClass is Key
+		typeInstance = ArrayType(**kwargs)
+		col = sqlalchemy.Column(typeInstance, *args, **cargs)
+		col._ct_type = isKey and "keylist" or "list"
+		if isKey:
+			col._kinds = typeInstance.kinds
+		return col
+	typeInstance = colClass(**kwargs)
+	col = sqlalchemy.Column(typeInstance, *args, **cargs)
+	if hasattr(typeInstance, "choices"):
+		col.choices = typeInstance.choices
+	if colClass is DateTimeAutoStamper:
+		col.is_dt_autostamper = True
+		col.should_stamp = typeInstance.should_stamp
+		col._ct_type = "datetime"
+	elif colClass is BasicString:
+		col._ct_type = "string"
+	elif colClass is Key:
+		col._kinds = typeInstance.kinds
+	elif colClass is JSONType:
+		col._ct_type = "json"
+	if not hasattr(col, "_ct_type"):
+		col._ct_type = colClass.__name__.lower()
+	col._default = default
+	return col
+
+def sqlColumn(colClass):
+	return lambda *args, **kwargs : _col(colClass, *args, **kwargs)
+
+for prop in primis:
+	sqlprop = getattr(sqlalchemy, prop)
+	globals()["sql%s"%(prop,)] = sqlprop
+	globals()[prop] = sqlColumn(basicType(sqlprop))
+
+# datetime
+BasicDT = basicType(sqlalchemy.DateTime)
+class DateTimeAutoStamper(BasicDT):
+	cache_ok = config.cache
+
+	def __init__(self, *args, **kwargs):
+		self.auto_now = kwargs.pop("auto_now", False)
+		self.auto_now_add = kwargs.pop("auto_now_add", False)
+		BasicDT.__init__(self, *args, **kwargs)
+
+	def should_stamp(self, is_new):
+		return self.auto_now or is_new and self.auto_now_add
+
+DateTime = sqlColumn(DateTimeAutoStamper)
+
+# strings, arrays, keys
+class BasicString(basicType(sqlalchemy.UnicodeText, StringType)):
+	cache_ok = config.cache
+
+	def process_bind_param(self, data, dialect):
+#		if data and type(data) is not str:
+#			data = data.decode('utf-8')
+		return data
+
+String = sqlColumn(BasicString)
+
+class JSONType(BasicString):
+	def process_bind_param(self, value, dialect):
+		return json.dumps(value)
+
+	def process_result_value(self, value, dialect):
+		return json.loads(value or "{}")
+
+JSON = sqlColumn(JSONType)
+
+class BlobWrapper(object):
+	def __init__(self, data="", value=0):
+		self.value = value
+		if data:
+			self.set(data)
+		else:
+			self._set_path(value)
+
+	def __nonzero__(self): # py2
+		return bool(self.value)
+
+	def __bool__(self): # py3
+		return bool(self.value)
+
+	def get(self):
+		if self.value:
+			from fyg.util import read
+			return read(self.path, binary=True)
+		else:
+			return None
+
+	def _next_value(self): # safely handles gaps
+		p, d, f = next(os.walk(config.blob))
+		fiz = [int(i) for i in f]
+		fiz.sort()
+		v = 1
+		for n in fiz:
+			if n != v:
+				return v
+			v += 1
+		return len(fiz) + 1
+
+	def _set_path(self, data=None):
+		if data:
+			if not self.value:
+				self.value = self._next_value()
+			self.path = os.path.join(config.blob, str(self.value))
+		else:
+			self.value = 0
+			self.path = None
+
+	def set(self, data):
+		self._set_path(data)
+		if data:
+			from fyg.util import write
+			if type(data) != bytes:
+				data = data.encode()
+			write(data, self.path, binary=True)
+
+	def delete(self):
+		if self.value:
+			from fyg.util import rm
+			rm(self.path)
+			self._set_path()
+
+	def urlsafe(self):
+		return self.path and "/" + "/".join(os.path.split(self.path))
+
+BasicInt = basicType(sqlInteger)
+
+class Blob(BasicInt):
+	def __init__(self, *args, **kwargs):
+		self.unique = kwargs.pop("unique", False)
+		BasicInt.__init__(self, *args, **kwargs)
+
+	def process_bind_param(self, data, dialect):
+		if type(data) is not BlobWrapper:
+			if self.unique:
+				from fyg.util import indir
+				match = indir(data, config.blob)
+				if match:
+					return int(match)
+			data = BlobWrapper(data)
+		return data.value
+
+	def process_result_value(self, value, dialect):
+		return BlobWrapper(value=value)
+
+Binary = sqlColumn(Blob)
+
+"""
+class Binary(basicType(sqlString)):
+	def process_bind_param(self, value, dialect):
+		return sqlalchemy.func.HEX(value)
+
+	def process_result_value(self, value, dialect):
+		return sqlalchemy.func.UNHEX(value)
+"""
+
+class ArrayType(BasicString):
+	cache_ok = config.cache
+
+	def __init__(self, *args, **kwargs):
+		self.isKey = kwargs.pop("isKey", False)
+		if self.isKey:
+			self.kinds = kwargs.pop("kinds", [kwargs.pop("kind", "*")])
+			for i in range(len(self.kinds)):
+				if not isinstance(self.kinds[i], str):
+					self.kinds[i] = self.kinds[i].__name__.lower()
+		BasicString.__init__(self, *args, **kwargs)
+
+	def process_bind_param(self, value, dialect):
+		if self.isKey:
+			for i in range(len(value)):
+				if hasattr(value[i], "urlsafe"):
+					value[i] = value[i].urlsafe()
+		return json.dumps(value)
+
+	def process_result_value(self, value, dialect):
+		try:
+			vlist = json.loads(value) or []
+		except:
+			vlist = []
+		if self.isKey:
+			for i in range(len(vlist)):
+				vlist[i] = KeyWrapper(vlist[i])
+		return vlist
+
+class KeyWrapper(object):
+	def __init__(self, urlsafe=None):
+		self.value = urlsafe
+
+	def __nonzero__(self): # py2
+		return bool(self.value)
+
+	def __bool__(self): # py3
+		return bool(self.value)
+
+	def __eq__(self, other):
+		return hasattr(other, "value") and self.value == other.value
+
+	def __ne__(self, other):
+		return not hasattr(other, "value") or self.value != other.value
+
+	def __hash__(self):
+		return sum([ord(c) for c in self.value])
+
+	def get(self, session=None):
+		return get(self.value, session)
+
+	def delete(self):
+		ent = self.get()
+		ent and ent.rm() # should be more efficient way...
+
+	def urlsafe(self):
+		return self.value
+
+class Key(BasicString):
+	cache_ok = config.cache
+
+	def __init__(self, *args, **kwargs):
+		self.kinds = kwargs.pop("kinds", [kwargs.pop("kind", "*")])
+		for i in range(len(self.kinds)):
+			if not isinstance(self.kinds[i], str):
+				self.kinds[i] = self.kinds[i].__name__.lower()
+		BasicString.__init__(self, *args, **kwargs)
+
+	def process_bind_param(self, value, dialect):
+		while True:#value and hasattr(value, "urlsafe"):
+			try: # for sqlite weirdness -- do this cleaner?
+				value = value.urlsafe()
+			except:
+				break
+		return value
+
+	def process_result_value(self, value, dialect):
+		return KeyWrapper(value)
+
+CompositeKey = sqlColumn(Key)
+ForeignKey = sqlColumn(Key)
+
+def sqlForeignKey(targetClass, **kwargs):
+	return sqlalchemy.Column(sqlInteger,
+		sqlalchemy.ForeignKey("%s.index"%(targetClass.__tablename__,)), **kwargs)

databae/query.py

@@ -0,0 +1,83 @@
+from sqlalchemy.sql import func, elements
+from fyg.util import log, start_timer, end_timer
+from fyg import config as confyg
+from six import string_types
+from .properties import *
+from .getters import *
+from .setters import *
+from .session import session, seshman, testSession, metadata, Session, handle_error, set_scoper
+
+_passthru = ["count", "all"]
+_qmod = ["filter", "limit", "offset", "join"]
+
+class Query(object):
+    def __init__(self, mod, *args, **kwargs):
+        self.mod = mod
+        self.schema = get_schema(mod)
+        self.session = kwargs.pop("session", None) or seshman.get()
+        self.query = kwargs.pop("query", self.session.query(mod))
+        for fname in _passthru:
+            setattr(self, fname, self._qpass(fname))
+        for fname in _qmod:
+            setattr(self, fname, self._qmlam(fname))
+        self.get = self._qpass("first")
+        self._order = self._qmlam("order_by")
+        self.filter(*args, **kwargs)
+
+    def order(self, prop):
+        if type(prop) == elements.UnaryExpression and "count" not in prop.element.description:
+            prop = "-%s"%(prop.element.description,)
+        if isinstance(prop, string_types):
+            asc = False
+            if prop.startswith("-"):
+                prop = prop[1:]
+            else:
+                asc = True
+            if "." in prop: # foreignkey reference from another table
+                from .lookup import refcount_subq
+                sub = refcount_subq(prop, self.session)
+                order = sub.c.count
+                if not asc:
+                    order = -sub.c.count
+                return self.join(sub, self.mod.key == sub.c.target).order(order)
+            prop = getattr(self.mod, prop)
+            if not asc:
+                prop = prop.desc()
+        return self._order(prop)
+
+    def _qpass(self, fname):
+        def qp(*args, **kwargs):
+            qkey = "Query.%s: %s %s (%s)"%(fname, args, kwargs, self.query)
+            if "query" in confyg.log.allow:
+                start_timer(qkey)
+            try:
+                res = getattr(self.query, fname)(*args, **kwargs)
+            except Exception as e:
+                handle_error(e, self.session)
+                log("retrying query operation")
+                res = getattr(self.query, fname)(*args, **kwargs)
+            if "query" in confyg.log.allow:
+                end_timer(qkey)
+            return res
+        return qp
+
+    def _qmlam(self, fname):
+        return lambda *a, **k : self._qmod(fname, *a, **k)
+
+    def _qmod(self, modname, *args, **kwargs):
+        self.query = getattr(self.query, modname)(*args, **kwargs)
+        return self
+
+    def copy(self, *args, **kwargs):
+        kwargs["query"] = self.query
+        return Query(self.mod, *args, **kwargs)
+
+    def fetch(self, limit=None, offset=0, keys_only=False):
+        if limit:
+            self.limit(limit)
+        if offset:
+            self.offset(offset)
+        res = self.all()
+        if keys_only: # best way?
+            return [r.key for r in res]
+        return res

databae/session.py

@@ -0,0 +1,159 @@
+import threading
+from sqlalchemy import text
+from sqlalchemy.pool import NullPool
+from sqlalchemy import create_engine, MetaData
+from sqlalchemy.orm import scoped_session, sessionmaker
+from fyg.util import log, error, confirm
+from fyg import config as confyg
+from .config import config as dcfg
+
+pcfg = dcfg.pool
+metadata = MetaData()
+
+def conn_ex(cmd):
+    log("issuing command: %s"%(cmd,), important=True)
+    with session.engine.connect() as conn:
+        conn.execute(text(cmd))
+
+def add_column(mod, col): # sqlite only
+    log("adding '%s' to '%s'"%(col, mod))
+    conn_ex('ALTER TABLE "%s" ADD COLUMN "%s"'%(mod, col))
+
+def handle_error(e, session=None, polytype=None, flag=" no such column: "):
+    log("Database operation failed: %s"%(e,), important=True)
+    session = session or seshman.get()
+    raise_anyway = True
+    stre = str(e)
+    if flag in stre:
+        target = stre.split(flag)[1].split(None, 1)[0]
+        log("Missing column: %s"%(target,), important=True)
+        if dcfg.alter:
+            if "." in target:
+                tmod, tcol = target.split(".")
+            else:
+                tcol = target
+                tmod = polytype
+            if dcfg.alter == "auto" or confirm("Add missing column '%s' to table '%s' (sqlite-only!)"%(tcol, tmod), True):
+                log("rolling back session")
+                session.rollback()
+                raise_anyway = False
+                add_column(tmod, tcol)
+        else:
+            log("To auto-update columns, add 'DB_ALTER = True' to your ct.cfg (sqlite only!)", important=True)
+    if raise_anyway:
+        error(e)
+
+def threadname():
+	return threading.currentThread().getName()
+
+scoper = None
+
+def set_scoper(func):
+	global scoper
+	scoper = func
+
+class Basic(object): # move elsewhere?
+	def sig(self):
+		return "%s(%s)"%(self.__class__.__name__, self.id)
+
+	def log(self, *msg):
+		if "db" in confyg.log.allow:
+			log("[db] session | %s :: %s"%(self.sig(), " ".join(msg)))
+
+class Session(Basic):
+	def __init__(self, database):
+		Session._id += 1
+		self.id = Session._id
+		self.database = database
+		self.engine = database.engine
+		self.generator = scoped_session(sessionmaker(bind=self.engine), scopefunc=self._scope)
+		for fname in ["add", "add_all", "delete", "flush", "commit", "query", "rollback"]:
+			setattr(self, fname, self._func(fname))
+		self._refresh()
+		self.log("initialized")
+
+	def teardown(self):
+		self.engine = None
+		self.session = None
+		self.database = None
+		self.generator = None
+
+	def _scope(self):
+		threadId = threadname()
+		return scoper and scoper(threadId) or threadId
+
+	def _func(self, fname):
+		def f(*args):
+			self._refresh()
+			self.database.init()
+			return getattr(self.session, fname)(*args)
+		return f
+
+	def _refresh(self):
+		self.session = self.generator()
+		self.no_autoflush = self.session.no_autoflush
+
+class DataBase(Basic):
+	def __init__(self, db=dcfg.main):
+		DataBase._id += 1
+		self.id = DataBase._id
+		if pcfg.null:
+			self.engine = create_engine(db, poolclass=NullPool, echo=dcfg.echo)
+		else:
+			self.engine = create_engine(db, pool_size=pcfg.size,
+				max_overflow=pcfg.overflow, pool_recycle=pcfg.recycle, echo=dcfg.echo)
+		self.sessions = {}
+		self._ready = False
+		self.log("initialized")
+
+	def init(self):
+		if not self._ready:
+			self._ready = True
+			metadata.create_all(self.engine)
+
+	def session(self):
+		thread = threadname()
+		if thread not in self.sessions:
+			self.sessions[thread] = Session(self)
+			self.log("session(%s) created!"%(thread,))
+		return self.sessions[thread]
+
+	def close(self):
+		thread = threadname()
+		if thread in self.sessions:
+			self.sessions[thread].generator.remove()
+			if thread == "MainThread":
+				note = "released"
+			else:
+				self.sessions[thread].teardown()
+				del self.sessions[thread]
+				note = "deleted"
+		else:
+			note = "not found!"
+		self.log("close(%s)"%(thread,), "session", note)
+
+class SessionManager(Basic):
+	def __init__(self):
+		SessionManager._id += 1
+		self.id = SessionManager._id
+		self.dbs = {}
+		self.log("initialized")
+
+	def db(self, db=dcfg.main):
+		if db not in self.dbs:
+			self.dbs[db] = DataBase(db)
+		return self.dbs[db]
+
+	def get(self, db=dcfg.main):
+		return self.db(db).session()
+
+	def close(self, db=dcfg.main):
+		self.db(db).close()
+
+Session._id = DataBase._id = SessionManager._id = 0
+
+def testSession():
+	return seshman.get(dcfg.test)
+
+seshman = SessionManager()
+session = seshman.get()

databae/setters.py

@@ -0,0 +1,58 @@
+from fyg.util import log, batch
+from .session import seshman
+from .edit import *
+from .util import ct_key
+from .config import config
+
+def _init_entity(instance, session=None, preserve_timestamps=False):
+    from .lookup import inc_counter, dec_counter
+    puts = []
+    now = datetime.now()
+    cls = instance.__class__
+    tname = instance.__tablename__
+    if tname != "ctrefcount":
+        if hasattr(instance, "_pre_put"):
+            instance._pre_put()
+        for key, val in list(cls.__dict__.items()):
+            if not preserve_timestamps and getattr(val, "is_dt_autostamper", False) and val.should_stamp(not instance.index):
+                setattr(instance, key, now)
+            if config.refcount and key in instance._orig_fkeys:
+                oval = instance._orig_fkeys[key]
+                val = getattr(instance, key)
+                if oval != val:
+                    reference = "%s.%s"%(tname, key)
+                    if type(oval) is list or type(val) is list:
+                        for o in [o for o in (oval or []) if o not in val]:
+                            puts.append(dec_counter(o, reference, session=session))
+                        for v in [v for v in (val or []) if v not in oval]:
+                            puts.append(inc_counter(v, reference, session=session))
+                    else:
+                        if oval:
+                            puts.append(dec_counter(oval, reference, session=session))
+                        if val:
+                            puts.append(inc_counter(val, reference, session=session))
+    return puts
+
+def init_multi(instances, session=None, preserve_timestamps=False):
+    session = session or seshman.get()
+    if preserve_timestamps:
+        log("initializing %s instances -- preserving timestamps!"%(len(instances),))
+    lookups = []
+    with session.no_autoflush:
+        for instance in instances:
+            lookups += _init_entity(instance, session, preserve_timestamps)
+    session.add_all(instances + lookups)
+    session.flush()
+    for instance in instances:
+        instance.key = instance.key or KeyWrapper(ct_key(instance.polytype, instance.index))
+
+def put_multi(instances, session=None, preserve_timestamps=False):
+    session = session or seshman.get()
+    batch(instances, init_multi, session, preserve_timestamps)
+    session.commit()
+
+def delete_multi(instances, session=None):
+    session = session or seshman.get()
+    for instance in instances:
+        instance.rm(False, session)
+    session.commit()

databae/util.py

@@ -0,0 +1,90 @@
+import operator, base64, json, hashlib, requests
+from datetime import datetime
+from six import string_types
+
+modbulkers = {}
+modelsubs = {}
+operators = {
+    "==": operator.__eq__,
+    ">=": operator.__ge__,
+    "<=": operator.__le__,
+    "!=": operator.__ne__,
+    ">": operator.__gt__,
+    "<": operator.__lt__
+}
+
+def get_bulker(modelName):
+    return modbulkers.get(modelName.lower(), None)
+
+def reg_bulker(modelName, func):
+    modbulkers[modelName.lower()] = func
+
+def get_model(modelName):
+    return modelsubs.get(modelName.lower(), None)
+
+def get_schema(modname=None):
+    if modname:
+        if not isinstance(modname, string_types):
+            modname = modname.__name__
+        return modelsubs[modname.lower()]._schema
+    s = {}
+    for key, val in list(modelsubs.items()):
+        if key not in ["modelbase", "ctrefcount"]:
+            s[key] = val._schema
+    return s
+
+def dprep(obj, schema=None): # prepares data object for model
+    schema = schema or get_schema(obj["modelName"])
+    o = {}
+    for key, prop in list(schema.items()):
+        if key in obj:
+            if prop == "datetime" and obj[key]:
+                o[key] = datetime.strptime(obj[key].replace("T", " ").replace("Z", ""), "%Y-%m-%d %X")
+#            elif prop == "string" and isinstance(obj[key], str):
+#                o[key] = obj[key].encode("utf-8")
+            elif key != "_label":
+                o[key] = obj[key]
+    return o
+
+def pad_key(compkey):
+    if compkey[-3:-1] == "CT":
+        compkey = compkey[:-3] + int(compkey[-1]) * "="
+    return compkey
+
+def unpad_key(compkey):
+    val = compkey and compkey.strip("=")
+    if val != compkey:
+        val += "CT" + str(len(compkey) - len(val))
+    return val
+
+def ct_key(modelName, index):
+    return unpad_key(base64.b64encode(json.dumps({
+        "index": index,
+        "model": modelName
+    }).encode()).decode())
+
+def merge_schemas(bases, label=None):
+    kinds = {}
+    schema = { "index": "immutable", "key": "key_immutable" }
+    for base in bases:
+        if hasattr(base, "_schema"):
+            schema.update(base._schema)
+            kinds.update(base._schema["_kinds"])
+    schema["_kinds"] = kinds
+    if label:
+        schema["_label"] = label
+    return schema
+
+def hashpass(password, date):
+    return hashlib.md5((password + str(date.date()).replace('-','')).encode()).hexdigest()
+
+def blobify(d, blobifier, extant=None):
+    for key, prop in list(db.get_schema(d["modelName"]).items()):
+        if prop == "blob" and d[key]:
+            entkey = d.get("gaekey", d["key"])
+            if extant and getattr(extant, key): # skip if some blob is present.........
+                log("%s.%s: already blobbed"%(d["modelName"], key))
+                del d[key]
+            else:
+                log("fetching %s.%s (%s.%s)"%(d["modelName"], key, entkey, d[key]))
+                d[key] = requests.get(blobifier%(entkey, key)).content

databae-0.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 bubbleboy14
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

databae-0.1.dist-info/METADATA

@@ -0,0 +1,21 @@
+Metadata-Version: 2.1
+Name: databae
+Version: 0.1
+Summary: DATabase ABstraction lAyEr
+Author: Mario Balibrera
+Author-email: mario.balibrera@gmail.com
+License: MIT License
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+License-File: LICENSE
+Requires-Dist: fyg >=0.1.7
+Requires-Dist: requests >=2.3.0
+Requires-Dist: six >=1.12.0
+Requires-Dist: sqlalchemy >=2.0.30
+
+sqlalchemy-based orm

databae-0.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+databae/__init__.py,sha256=4XjLWG2Oo3T-XBS-ZHTqBCYbO68QfH9Lw-zhfnyOwzI,139
+databae/config.py,sha256=Y0h_7LHFh_cJVi-LugpIaB8e5n3O0pao6r78HGwcjUA,316
+databae/edit.py,sha256=83a2_gxX_ZnbrBFP7zvPduX8uPGXDup09HNTmYXph8s,1543
+databae/getters.py,sha256=IcLsuonToE3GpXCYtt8mai7wlwjhXRDpKBhLlA7VaY0,4803
+databae/lookup.py,sha256=G9Roq6VOEyY3agtAIZvk_KNigCN_-gyH9-zXR098gjM,2147
+databae/model.py,sha256=Up7FFIOYL9cENINHM-RuNTFvEfp4WFqDPnPzoCcCLik,6561
+databae/properties.py,sha256=WYIAjLTwwqnCscX86gGRKyDhoYaHQloV9dirRWjSVRY,7147
+databae/query.py,sha256=r1gLsI_ma0ueO_TNOfCMdvHSKgHSFDnbjvooLffgymU,3039
+databae/session.py,sha256=Nnf5Ftf7pevJsuRmVv898QRsO8CXwRapep13FOzqrIU,4399
+databae/setters.py,sha256=J-Jp8fSG8Y7tF4ikjFCCon-lOpsdKJGIypck2Rr03aU,2514
+databae/util.py,sha256=ABS3OiFJaytLjQwJALmhuxC9ab4nThmHrpoQFPXQnTg,2898
+databae-0.1.dist-info/LICENSE,sha256=TCGFq2RZnPZed3cKBhfq8EIxNzHcwJWJsBW8p9y1dDk,1068
+databae-0.1.dist-info/METADATA,sha256=XXE0N8wqbWCPpsQuG_E9qTDE1en5vfI6AwfvMizxvCE,673
+databae-0.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+databae-0.1.dist-info/top_level.txt,sha256=odMXlkbmQm92xaUnNmomEgF2eStj6enLIlVGCS40oyc,8
+databae-0.1.dist-info/RECORD,,

databae-0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.42.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

databae-0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+databae