transit-python 0.8.174-__py2.7.egg → 0.8.240__py2.7.egg

transit/decoder.py

@@ -18,6 +18,7 @@ from collections import OrderedDict
 from helpers import pairs
 import read_handlers as rh
 from rolling_cache import RollingCache, is_cacheable, is_cache_key
+from transit_types import true, false
 
 class Tag(object):
     def __init__(self, tag):
@@ -34,6 +35,7 @@ default_options = {"decoders": {"_": rh.NoneHandler,
                                 "t": rh.DateHandler,
                                 "m": rh.DateHandler,
                                 "n": rh.BigIntegerHandler,
+                                "z": rh.SpecialNumbersHandler,
                                 "link": rh.LinkHandler,
                                 "list": rh.ListHandler,
                                 "set": rh.SetHandler,
@@ -47,7 +49,7 @@ ground_decoders = {"_": rh.NoneHandler,
                    "'": rh.IdentityHandler}
 
 class Decoder(object):
-    """ The Decoder is the lowest level entry point for parsing, decoding, and
+    """The Decoder is the lowest level entry point for parsing, decoding, and
     fully converting Transit data into Python objects.
 
     During the creation of a Decoder object, you can specify custom options
@@ -55,7 +57,8 @@ class Decoder(object):
     can specify your own decoders and override many of the built in decoders,
     some decoders are silently enforced and cannot be overriden.  These are
     known as Ground Decoders, and are needed to maintain bottom-tier
-    compatibility."""
+    compatibility.
+    """
 
     def __init__(self, options={}):
         self.options = default_options.copy()
@@ -66,11 +69,12 @@ class Decoder(object):
         self.decoders.update(ground_decoders)
 
     def decode(self, node, cache=None, as_map_key=False):
-        """ Given a node of data (any supported decodeable obj - string, dict,
+        """Given a node of data (any supported decodeable obj - string, dict,
         list), return the decoded object.  Optionally set the current decode
         cache [None].  If None, a new RollingCache is instantiated and used.
         You may also hit to the decoder that this node is to be treated as a
-        map key [False].  This is used internally."""
+        map key [False].  This is used internally.
+        """
         if not cache:
             cache = RollingCache()
         return self._decode(node, cache, as_map_key)
@@ -85,18 +89,19 @@ class Decoder(object):
             return self.decode_list(node, cache, as_map_key)
         elif tp is str:
             return self.decode_string(unicode(node, "utf-8"), cache, as_map_key)
-        else:
-            return node
+        elif tp is bool:
+            return true if node else false
+        return node
 
     def decode_list(self, node, cache, as_map_key):
-        """ Special case decodes map-as-array.
+        """Special case decodes map-as-array.
         Otherwise lists are treated as Python lists.
 
         Arguments follow the same convention as the top-level 'decode'
-        function"""
+        function.
+        """
         if node:
-            decoded = self._decode(node[0], cache, as_map_key)
-            if decoded == MAP_AS_ARR:
+            if node[0] == MAP_AS_ARR:
                 # key must be decoded before value for caching to work.
                 returned_dict = {}
                 for k,v in pairs(node[1:]):
@@ -104,16 +109,16 @@ class Decoder(object):
                     val = self._decode(v, cache, as_map_key)
                     returned_dict[key] = val
                 return returned_dict
-#                return {self._decode(k, cache, True):
-#                        self._decode(v, cache, as_map_key)
-#                        for k,v in pairs(node[1:])}
-            elif isinstance(decoded, Tag):
+
+            decoded = self._decode(node[0], cache, as_map_key)
+            if isinstance(decoded, Tag):
                 return self.decode_tag(decoded.tag, self._decode(node[1], cache, as_map_key))
         return tuple(self._decode(x, cache, as_map_key) for x in node)
 
     def decode_string(self, string, cache, as_map_key):
-        """ Decode a string - arguments follow the same convention as the
-        top-level 'decode' function"""
+        """Decode a string - arguments follow the same convention as the
+        top-level 'decode' function.
+        """
         if is_cache_key(string):
             return self.parse_string(cache.decode(string, as_map_key), cache, as_map_key)
         if is_cacheable(string, as_map_key):
@@ -131,7 +136,14 @@ class Decoder(object):
         if len(hash) != 1:
             h = {}
             for k, v in hash.items():
-                h[self._decode(k, cache, True)] = self._decode(v, cache, False)
+                # crude/verbose implementation, but this is only version that
+                # plays nice w/cache for both msgpack and json thus far.
+                # -- e.g., we have to specify encode/decode order for key/val
+                # -- explicitly, all implicit ordering has broken in corner 
+                # -- cases, thus these extraneous seeming assignments
+                key = self._decode(k, cache, True)
+                val = self._decode(v, cache, False)
+                h[key] = val
             return transit_types.frozendict(h)
         else:
             key,value = hash.items()[0]
@@ -139,7 +151,7 @@ class Decoder(object):
             if isinstance(key, Tag):
                 return self.decode_tag(key.tag, self._decode(value, cache, as_map_key))
             else:
-                return {key: self._decode(value, cache, False)}
+                return transit_types.frozendict({key: self._decode(value, cache, False)})
 
     def parse_string(self, string, cache, as_map_key):
         if string.startswith(ESC):
@@ -155,10 +167,11 @@ class Decoder(object):
         return string
 
     def register(self, key_or_tag, obj):
-        """ Register a custom Transit tag and new parsing function with the
+        """Register a custom Transit tag and new parsing function with the
         decoder.  Also, you can optionally set the 'default_decoder' with
         this function.  Your new tag and parse/decode function will be added
-        to the interal dictionary of decoders for this Decoder object"""
+        to the interal dictionary of decoders for this Decoder object.
+        """
         if key_or_tag == "default_decoder":
             self.options["default_decoder"] = obj
         else:

transit/read_handlers.py

@@ -47,7 +47,7 @@ class SymbolHandler(object):
 class BooleanHandler(object):
     @staticmethod
     def from_rep(x):
-        return x == "t"
+        return transit_types.true if x == "t" else transit_types.false
 
 class IntHandler(object):
     @staticmethod
@@ -62,7 +62,7 @@ class FloatHandler(object):
 class UuidHandler(object):
     @staticmethod
     def from_rep(u):
-        """ Given a string, return a UUID object"""
+        """Given a string, return a UUID object."""
         if isinstance(u, basestring):
             return uuid.UUID(u)
 
@@ -87,7 +87,7 @@ class DateHandler(object):
         return DateHandler._convert_timestamp(long(d))
     @staticmethod
     def _convert_timestamp(ms):
-        """ Given a timestamp in ms, return a DateTime object"""
+        """Given a timestamp in ms, return a DateTime object."""
         return datetime.datetime.fromtimestamp(ms/1000.0, dateutil.tz.tzutc())
 
 class BigIntegerHandler(object):
@@ -120,3 +120,13 @@ class IdentityHandler(object):
     def from_rep(i):
         return i
 
+class SpecialNumbersHandler(object):
+    @staticmethod
+    def from_rep(z):
+        if z == 'NaN':
+            return float('Nan')
+        if z == 'INF':
+            return float('Inf')
+        if z == '-INF':
+            return float('-Inf')
+        raise ValueError("Don't know how to handle: " + str(z) + " as \"z\"")

transit/reader.py

@@ -13,47 +13,71 @@
 ## limitations under the License.
 
 import json
+import sosjson
 import msgpack
 from decoder import Decoder
 from collections import OrderedDict
 
 class Reader(object):
-    """ The top-level object for reading in Transit data and converting it to
+    """The top-level object for reading in Transit data and converting it to
     Python objects.  During initialization, you must specify the protocol used
-    for unmarshalling the data- json or msgpack."""
+    for unmarshalling the data- json or msgpack.
+    """
     def __init__(self, protocol="json"):
         if protocol in ("json", "json_verbose"):
             self.reader = JsonUnmarshaler()
         else:
             self.reader = MsgPackUnmarshaler()
+            self.unpacker = self.reader.unpacker
 
     def read(self, stream):
-        """ Given a readable file descriptor object (something `load`able by
+        """Given a readable file descriptor object (something `load`able by
         msgpack or json), read the data, and return the Python representation
-        of the contents."""
+        of the contents. One-shot reader.
+        """
         return self.reader.load(stream)
 
     def register(self, key_or_tag, f_val):
-        """ Register a custom transit tag and decoder/parser function for use
-        during reads."""
+        """Register a custom transit tag and decoder/parser function for use
+        during reads.
+        """
         self.reader.decoder.register(key_or_tag, f_val)
 
+    def readeach(self, stream, **kwargs):
+        """Temporary hook for API while streaming reads are in experimental
+        phase. Read each object from stream as available with generator. 
+        JSON blocks indefinitely waiting on JSON entities to arrive. MsgPack
+        requires unpacker property to be fed stream using unpacker.feed()
+        method.
+        """
+        for o in self.reader.loadeach(stream):
+            yield o
+
 class JsonUnmarshaler(object):
-    """ The top-level Unmarshaler used by the Reader for JSON payloads.  While
-    you may use this directly, it is strongly discouraged."""
+    """The top-level Unmarshaler used by the Reader for JSON payloads.  While
+    you may use this directly, it is strongly discouraged.
+    """
     def __init__(self):
         self.decoder = Decoder()
 
     def load(self, stream):
         return self.decoder.decode(json.load(stream, object_pairs_hook=OrderedDict))
 
+    def loadeach(self, stream):
+        for o in sosjson.items(stream, object_pairs_hook=OrderedDict):
+            yield self.decoder.decode(o)
 
 class MsgPackUnmarshaler(object):
-    """ The top-level Unmarshaler used by the Reader for MsgPacke payloads.
-    While you may use this directly, it is strongly discouraged."""
+    """The top-level Unmarshaler used by the Reader for MsgPack payloads.
+    While you may use this directly, it is strongly discouraged.
+    """
     def __init__(self):
         self.decoder = Decoder()
+        self.unpacker = msgpack.Unpacker(object_pairs_hook=OrderedDict)
 
     def load(self, stream):
         return self.decoder.decode(msgpack.load(stream, object_pairs_hook=OrderedDict))
 
+    def loadeach(self, stream):
+        for o in self.unpacker:
+            yield self.decoder.decode(o)

transit/rolling_cache.py

@@ -42,15 +42,12 @@ def is_cacheable(string, as_map_key=False):
             or (string[:2] in ["~#", "~$", "~:"]))
 
 class RollingCache(object):
-    """ This is the internal cache used by python-transit for cacheing and
+    """This is the internal cache used by python-transit for cacheing and
     expanding map keys during writing and reading.  The cache enables transit
     to minimize the amount of duplicate data sent over the wire, effectively
     compressing down the overall payload size.  The cache is not intended to
-    be used directly."""
-    # (1) should we use list or dict on read side? ##- probably dictionary is best for lookup by code.
-      # dict lookup should be amortized O(1), for list O(n)
-    # (2) currently stores value read from the wire, should probably store value after decoding
-    # so we don't do multiple decodes. Sped up parsing in ruby by 30%.
+    be used directly.
+    """
     def __init__(self):
         self.key_to_value = {}
         self.value_to_key = {}

transit/sosjson.py

@@ -0,0 +1,74 @@
+## copyright 2014 cognitect. all rights reserved.
+##
+## licensed under the apache license, version 2.0 (the "license");
+## you may not use this file except in compliance with the license.
+## you may obtain a copy of the license at
+##
+##      http://www.apache.org/licenses/license-2.0
+##
+## unless required by applicable law or agreed to in writing, software
+## distributed under the license is distributed on an "as-is" basis,
+## without warranties or conditions of any kind, either express or implied.
+## see the license for the specific language governing permissions and
+## limitations under the license.
+# Simple object streaming in Python - just reads one complete JSON object
+# at a time and returns json.loads of that string.
+
+# Ugly implementation at moment
+from copy import copy
+import json
+
+SKIP = [" ", "\n", "\t"]
+ESCAPE = "\\"
+
+def read_chunk(stream):
+    """Ignore whitespace outside of strings. If we hit a string, read it in
+    its entirety.
+    """
+    chunk = stream.read(1)
+    while chunk in SKIP:
+        chunk = stream.read(1)
+    if chunk == "\"":
+        chunk += stream.read(1)
+        while not chunk.endswith("\""):
+            if chunk[-1] == ESCAPE:
+                chunk += stream.read(2)
+            else:
+                chunk += stream.read(1)
+    return chunk
+
+def items(stream, **kwargs):
+    """External facing items. Will return item from stream as available.
+    Currently waits in loop waiting for next item. Can pass keywords that
+    json.loads accepts (such as object_pairs_hook)
+    """
+    for s in yield_json(stream):
+        yield json.loads(s, **kwargs)
+
+def yield_json(stream):
+    """Uses array and object delimiter counts for balancing.
+    """
+    buff = u""
+    arr_count = 0
+    obj_count = 0
+    while True:
+        buff += read_chunk(stream)
+
+        # If we finish parsing all objs or arrays, yield a finished JSON
+        # entity.
+        if buff.endswith('{'):
+            obj_count += 1
+        if buff.endswith('['):
+            arr_count += 1
+        if buff.endswith(']'):
+            arr_count -= 1
+            if obj_count == arr_count == 0:
+                json_item = copy(buff)
+                buff = u""
+                yield json_item
+        if buff.endswith('}'):
+            obj_count -= 1
+            if obj_count == arr_count == 0:
+                json_item = copy(buff)
+                buff = u""
+                yield json_item

transit/transit_types.py

@@ -141,15 +141,15 @@ class frozendict(Mapping, Hashable):
 
 class Link(object):
     # Class property constants for rendering types
-    LINK = "link"
-    IMAGE = "image"
+    LINK = u"link"
+    IMAGE = u"image"
 
     # Class property constants for keywords/obj properties.
-    HREF = "href"
-    REL = "rel"
-    PROMPT = "prompt"
-    NAME = "name"
-    RENDER = "render"
+    HREF = u"href"
+    REL = u"rel"
+    PROMPT = u"prompt"
+    NAME = u"name"
+    RENDER = u"render"
 
     def __init__(self, href=None, rel=None, name=None, render=None, prompt=None):
         self._dict = frozendict()
@@ -188,3 +188,29 @@ class Link(object):
     @property
     def as_array(self):
         return [self.href, self.rel, self.name, self.render, self.prompt]
+
+class Boolean(object):
+    """To allow a separate t/f that won't hash as 1/0. Don't call directly,
+    instead use true and false as singleton objects. Can use with type check.
+
+    Note that the Booleans are for preserving hash/set bools that duplicate 1/0
+    and not designed for use in Python outside of logical evaluation (don't treat
+    as an int, they're not). You can get a Python bool using bool(x)
+    where x is a true or false Boolean.
+    """
+    def __init__(self, name):
+        self.v = True if name == "true" else False
+        self.name = name
+
+    def __nonzero__(self):
+        return self.v
+
+    def __repr__(self):
+        return self.name
+
+    def __str__(self):
+        return self.name
+
+# lowercase rep matches java/clojure
+false = Boolean("false")
+true = Boolean("true")

transit/write_handlers.py

@@ -14,10 +14,12 @@
 
 from constants import *
 from class_hash import ClassDict
-from transit_types import Keyword, Symbol, URI, frozendict, TaggedValue, Link
+from transit_types import Keyword, Symbol, URI, frozendict, TaggedValue, Link, Boolean
 import uuid
 import datetime, time
 from dateutil import tz
+from math import isnan
+import struct
 
 ## This file contains Write Handlers - all the top-level objects used when
 ## writing Transit data.  These object must all be immutable and pickleable.
@@ -69,10 +71,16 @@ class BigIntHandler(object):
 
 class FloatHandler(object):
     @staticmethod
-    def tag(_):
-        return "f"
+    def tag(f):
+        return "z" if isnan(f) or f in (float('Inf'), float('-Inf')) else "f"
     @staticmethod
     def rep(f):
+        if isnan(f):
+            return "NaN"
+        if f == float('Inf'):
+            return "INF"
+        if f == float("-Inf"):
+            return "-INF"
         return str(f)
     @staticmethod
     def string_rep(f):
@@ -95,10 +103,10 @@ class BooleanHandler(object):
         return '?'
     @staticmethod
     def rep(b):
-        return b
+        return bool(b)
     @staticmethod
     def string_rep(b):
-        return b and 't' or 'f'
+        return 't' if b else 'f'
 
 class ArrayHandler(object):
     @staticmethod
@@ -145,14 +153,12 @@ class SymbolHandler(object):
         return str(s)
 
 class UuidHandler(object):
-    mask = pow(2, 64) - 1
     @staticmethod
     def tag(_):
         return "u"
     @staticmethod
     def rep(u):
-        i = u.int
-        return (i >> 64, i & UuidHandler.mask)
+        return struct.unpack('>qq', u.bytes)
     @staticmethod
     def string_rep(u):
         return str(u)
@@ -176,7 +182,7 @@ class DateTimeHandler(object):
     @staticmethod
     def rep(d):
         td = d - DateTimeHandler.epoch
-        return long((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 1e3)
+        return int((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 1e3)
     @staticmethod
     def verbose_handler():
         return VerboseDateTimeHandler
@@ -245,12 +251,14 @@ class WriteHandler(ClassDict):
     The Handler itself is a dispatch map, that resolves on full type/object
     inheritance.
 
-    These handlers can be overriden during the creation of a Transit Writer."""
+    These handlers can be overriden during the creation of a Transit Writer.
+    """
 
     def __init__(self):
         super(WriteHandler, self).__init__()
         self[type(None)] = NoneHandler
         self[bool] = BooleanHandler
+        self[Boolean] = BooleanHandler
         self[str] = StringHandler
         self[unicode] = StringHandler
         self[list] = ArrayHandler
@@ -271,4 +279,3 @@ class WriteHandler(ClassDict):
         self[frozendict] = MapHandler
         self[TaggedValue] = TaggedValueHandler
         self[Link] = LinkHandler
-