b10-msgpack-numpy 0.4.8__py2.py3-none-any.whl

b10_msgpack_numpy-0.4.8.dist-info/METADATA

@@ -0,0 +1,163 @@
+Metadata-Version: 2.4
+Name: b10-msgpack-numpy
+Version: 0.4.8
+Summary: Numpy data serialization using msgpack (Baseten fork with CWE-502 fix)
+Home-page: https://github.com/basetenlabs/b10-msgpack-numpy
+Author: Lev E. Givon
+Author-email: lev@columbia.edu
+License: BSD
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+License-File: AUTHORS.md
+Requires-Dist: numpy>=1.9.0
+Requires-Dist: msgpack>=0.5.2
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
+
+Package Description
+-------------------
+This package provides encoding and decoding routines that enable the
+serialization and deserialization of numerical and array data types provided by 
+[numpy](http://www.numpy.org/) using the highly efficient
+[msgpack](http://msgpack.org/) format. Serialization of Python's
+native complex data types is also supported.
+
+[![Latest Version](https://img.shields.io/pypi/v/msgpack-numpy.svg)](https://pypi.python.org/pypi/msgpack-numpy)
+[![Build Status](https://travis-ci.org/lebedov/msgpack-numpy.svg?branch=master)](https://travis-ci.org/lebedov/msgpack-numpy)
+
+Installation
+------------
+msgpack-numpy requires msgpack-python and numpy. If you 
+have [pip](http://www.pip-installer.org/) installed on your
+system, run
+
+    pip install msgpack-numpy
+
+to install the package and all dependencies. You can also download 
+the source tarball, unpack it, and run
+
+    python setup.py install
+
+from within the source directory.
+
+Usage
+-----
+The easiest way to use msgpack-numpy is to call its monkey patching
+function after importing the Python msgpack package:
+
+    import msgpack
+    import msgpack_numpy as m
+    m.patch()
+
+This will automatically force all msgpack serialization and deserialization
+routines (and other packages that use them) to become numpy-aware. 
+Of course, one can also manually pass the encoder and 
+decoder provided by msgpack-numpy to the msgpack routines:
+
+    import msgpack
+    import msgpack_numpy as m
+    import numpy as np
+
+    x = np.random.rand(5)
+    x_enc = msgpack.packb(x, default=m.encode)
+    x_rec = msgpack.unpackb(x_enc, object_hook=m.decode)
+
+msgpack-numpy will try to use the binary (fast) extension in msgpack by default.  
+If msgpack was not compiled with Cython (or if the ``MSGPACK_PUREPYTHON`` 
+variable is set), it will fall back to using the slower pure Python msgpack 
+implementation.
+
+Notes
+-----
+The primary design goal of msgpack-numpy is ensuring preservation of numerical
+data types during msgpack serialization and deserialization. Inclusion of type
+information in the serialized data necessarily incurs some storage overhead; if
+preservation of type information is not needed, one may be able to avoid some
+of this overhead by writing a custom encoder/decoder pair that produces more
+efficient serializations for those specific use cases. 
+
+Numpy arrays with a dtype of 'O' are serialized using pickle as a fallback
+solution. Deserializing them requires unpickling, which executes arbitrary
+code from the payload, so decoding is **disabled by default** (CWE-502):
+``decode``, ``unpackb``/``unpack`` and ``Unpacker`` raise
+``msgpack_numpy.PickleNotAllowed`` when they encounter an object-dtype array.
+If you control both ends, opt in explicitly:
+
+    x_rec = msgpack.unpackb(x_enc, allow_pickle=True)
+
+Never enable ``allow_pickle`` for untrusted input. As the overhead of pickle
+serialization negates one of the reasons to use msgpack, consider writing a
+custom encoder/decoder for your specific use case instead.
+
+Note that numpy arrays deserialized by msgpack-numpy are read-only and must be copied 
+if they are to be modified.
+
+Development
+-----------
+The latest source code can be obtained from
+[GitHub](https://github.com/basetenlabs/b10-msgpack-numpy/). This is the Baseten
+fork of [lebedov/msgpack-numpy](https://github.com/lebedov/msgpack-numpy); the
+distribution is named ``b10-msgpack-numpy`` and still provides the
+``msgpack_numpy`` module.
+
+msgpack-numpy maintains compatibility with python versions 2.7 and 3.5+.
+
+Install [`tox`](https://tox.readthedocs.io/en/latest/) to support testing
+across multiple python versions in your development environment. If you
+use [`conda`](https://docs.conda.io/en/latest/) to install `python` use
+[`tox-conda`](https://github.com/tox-dev/tox-conda) to automatically manage
+testing across all supported python versions.
+    
+    # Using a system python
+    pip install tox
+
+    # Additionally, using a conda-provided python
+    pip install tox tox-conda
+
+Execute tests across supported python versions:
+    
+    tox
+
+Authors
+-------
+See the included [AUTHORS.md](https://github.com/lebedov/msgpack-numpy/blob/master/AUTHORS.md) file for 
+more information.
+
+License
+-------
+This software is licensed under the [BSD License](http://www.opensource.org/licenses/bsd-license).
+See the included [LICENSE.md](https://github.com/lebedov/msgpack-numpy/blob/master/LICENSE.md) file for 
+more information.
+
+See Also
+--------
+[msgpack-numpy-js](https://github.com/emerald-geomodelling/msgpack-numpy-js) by 
+[Egil Möller](https://github.com/redhog) is a JavaScript library that can 
+serialize and deserialize JS typed arrays with msgpack.
+
+[msgpack-numpy-rs](https://github.com/clearviewai/msgpack-numpy-rs) by [
+Terrence Liu](https://github.com/terencezl) is a Rust crate (mostly) compatible with msgpack-numpy 
+that can serialize and deserialize numpy arrays with msgpack.

b10_msgpack_numpy-0.4.8.dist-info/RECORD

@@ -0,0 +1,7 @@
+msgpack_numpy.py,sha256=V81ZsmKczgTtb819qbr1Y-frkO4bLvqk3OMlmYPAjRo,13626
+b10_msgpack_numpy-0.4.8.dist-info/licenses/AUTHORS.md,sha256=X-BEbd1tHHD6KYdHNoWC3B2Rwj1iyPtyKFlbyH_VMjI,745
+b10_msgpack_numpy-0.4.8.dist-info/licenses/LICENSE.md,sha256=UJqnrxmuK1sWgbIrlpYn7p4f0V_36lv59uMB_ETVpsI,1551
+b10_msgpack_numpy-0.4.8.dist-info/METADATA,sha256=Tz-4VejGI2ALyDTxvLTosiG-ytcX9wXOUT5m_B0EeEo,6240
+b10_msgpack_numpy-0.4.8.dist-info/WHEEL,sha256=Uvlg2aNlEKdilsyFSJt1QX5me0eEbCjQzR3nZeFkvy4,109
+b10_msgpack_numpy-0.4.8.dist-info/top_level.txt,sha256=GdppitFJC93hx_GPq6zrJiWq9Is2TU3XrUBMfl7tdbk,14
+b10_msgpack_numpy-0.4.8.dist-info/RECORD,,

b10_msgpack_numpy-0.4.8.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+

b10_msgpack_numpy-0.4.8.dist-info/licenses/AUTHORS.md

@@ -0,0 +1,18 @@
+<!---
+-*- mode:markdown -*-
+vi:ft=markdown
+-->
+
+Authors
+=======
+This software was written and packaged by [Lev E. Givon](https://github.com/lebedov)
+
+Special thanks are due to the following parties for their contributions:
+
+- [Alex Ford](https://github.com/asford) - bug fix.
+- [Colin Jermain](https://github.com/cjermain) - Python 3 support.
+- [Etienne Wodey](https://github.com/airwoodix) - support for ndarrays with dtype='O'
+- [John Tyree](https://github.com/johntyree) - support for numpy scalar booleans.
+- [Mehdi Sadeghi](https://github.com/mehdisadeghi) - bug reports.
+- [Sujoy Roy](https://github.com/tvkpz) - bug reports.
+- [Jose Coutinho](https://github.com/tiagocoutinho) - make msgpack-numpy  params consistent with msgpack-python.

b10_msgpack_numpy-0.4.8.dist-info/licenses/LICENSE.md

@@ -0,0 +1,35 @@
+<!---
+-*- mode:markdown -*-
+vi:ft=markdown
+-->
+License
+=======
+
+Copyright (c) 2013-2022, Lev E. Givon.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following
+  disclaimer in the documentation and/or other materials provided
+  with the distribution.
+* Neither the name of Lev E. Givon nor the names of any
+  contributors may be used to endorse or promote products derived
+  from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

b10_msgpack_numpy-0.4.8.dist-info/top_level.txt

@@ -0,0 +1 @@
+msgpack_numpy

msgpack_numpy.py

@@ -0,0 +1,343 @@
+#!/usr/bin/env python
+
+"""
+Support for serialization of numpy data types with msgpack.
+"""
+
+# Copyright (c) 2013-2022, Lev E. Givon
+# All rights reserved.
+# Distributed under the terms of the BSD license:
+# http://www.opensource.org/licenses/bsd-license
+
+import sys
+import functools
+import pickle
+import warnings
+
+import msgpack
+from msgpack import Packer as _Packer, Unpacker as _Unpacker, \
+    unpack as _unpack, unpackb as _unpackb
+import numpy as np
+
+if sys.version_info >= (3, 0):
+    def ndarray_to_bytes(obj):
+        if obj.dtype == 'O':
+            return obj.dumps()
+        else:
+            if sys.platform == 'darwin':
+                return obj.tobytes()
+            else:
+                return obj.data if obj.flags['C_CONTIGUOUS'] else obj.tobytes()
+
+    num_to_bytes = lambda obj: obj.data
+
+    def tostr(x):
+        if isinstance(x, bytes):
+            return x.decode()
+        else:
+            return str(x)
+else:
+    def ndarray_to_bytes(obj):
+        if obj.dtype == 'O':
+            return obj.dumps()
+        else:
+            if sys.platform == 'darwin':
+                return obj.tobytes()
+            else:
+                return memoryview(obj.data) if obj.flags['C_CONTIGUOUS'] else obj.tobytes()
+
+    num_to_bytes = lambda obj: memoryview(obj.data)
+
+    def tostr(x):
+        return x
+
+def encode(obj, chain=None):
+    """
+    Data encoder for serializing numpy data types.
+    """
+
+    if isinstance(obj, np.ndarray):
+        # If the dtype is structured, store the interface description;
+        # otherwise, store the corresponding array protocol type string:
+        if obj.dtype.kind in ('V', 'O'):
+            kind = bytes(obj.dtype.kind, 'ascii')
+            descr = obj.dtype.descr
+        else:
+            kind = b''
+            descr = obj.dtype.str
+
+        return {b'nd': True,
+                b'type': descr,
+                b'kind': kind,
+                b'shape': obj.shape,
+                b'data': ndarray_to_bytes(obj)}
+    elif isinstance(obj, (np.bool_, np.number)):
+        return {b'nd': False,
+                b'type': obj.dtype.str,
+                b'data': num_to_bytes(obj)}
+    elif isinstance(obj, complex):
+        return {b'complex': True,
+                b'data': obj.__repr__()}
+    else:
+        return obj if chain is None else chain(obj)
+
+class PickleNotAllowed(ValueError):
+    """
+    Raised when an object-dtype ndarray would require unpickling and
+    ``allow_pickle`` is ``False`` (the default).
+
+    Object-dtype arrays are serialized by pickle, so deserializing them
+    executes arbitrary code from the payload. Never enable ``allow_pickle``
+    for data that was not produced by a trusted encoder.
+    """
+
+def decode(obj, chain=None, allow_pickle=False):
+    """
+    Decoder for deserializing numpy data types.
+
+    :param obj: The decoded msgpack object.
+    :param chain: Optional next decoder in the chain, called for objects
+        that this decoder does not recognize.
+    :param allow_pickle: Whether to unpickle object-dtype ndarrays. This is
+        disabled by default because unpickling untrusted data allows
+        arbitrary code execution (CWE-502). Only set this to ``True`` for
+        payloads you produced yourself.
+    :raises PickleNotAllowed: If an object-dtype ndarray is encountered and
+        ``allow_pickle`` is ``False``.
+    """
+
+    try:
+        if b'nd' in obj:
+            if obj[b'nd'] is True:
+
+                # Check if b'kind' is in obj to enable decoding of data
+                # serialized with older versions (#20) or data
+                # that had dtype == 'O' (#46):
+                if b'kind' in obj and obj[b'kind'] == b'V':
+                    descr = [tuple(tostr(t) if type(t) is bytes else t for t in d) \
+                             for d in obj[b'type']]
+                elif b'kind' in obj and obj[b'kind'] == b'O':
+                    if not allow_pickle:
+                        raise PickleNotAllowed(
+                            'refusing to unpickle an object-dtype ndarray; '
+                            'pass allow_pickle=True (decode/unpackb/Unpacker) '
+                            'to opt in for trusted payloads only')
+                    return pickle.loads(obj[b'data'])
+                else:
+                    descr = obj[b'type']
+                return np.ndarray(buffer=obj[b'data'],
+                                  dtype=_unpack_dtype(descr),
+                                  shape=obj[b'shape'])
+            else:
+                descr = obj[b'type']
+                return np.frombuffer(obj[b'data'],
+                            dtype=_unpack_dtype(descr))[0]
+        elif b'complex' in obj:
+            return complex(tostr(obj[b'data']))
+        else:
+            return obj if chain is None else chain(obj)
+    except KeyError:
+        return obj if chain is None else chain(obj)
+
+def _unpack_dtype(dtype):
+    """
+    Unpack dtype descr, recursively unpacking nested structured dtypes.
+    """
+
+    if isinstance(dtype, (list, tuple)):
+        # Unpack structured dtypes of the form: (name, type, *shape)
+        dtype = [
+            (subdtype[0], _unpack_dtype(subdtype[1])) + tuple(subdtype[2:])
+            for subdtype in dtype
+        ]
+    return np.dtype(dtype)
+
+if msgpack.version < (1, 0, 0):
+    warnings.warn('support for msgpack < 1.0.0 will be removed in a future release',
+                   DeprecationWarning)
+
+# Maintain support for msgpack < 0.4.0:
+if msgpack.version < (0, 4, 0):
+    class Packer(_Packer):
+        def __init__(self, default=None,
+                     encoding='utf-8',
+                     unicode_errors='strict',
+                     use_single_float=False,
+                     autoreset=1):
+            default = functools.partial(encode, chain=default)
+            super(Packer, self).__init__(default=default,
+                                         encoding=encoding,
+                                         unicode_errors=unicode_errors,
+                                         use_single_float=use_single_float,
+                                         autoreset=autoreset)
+    class Unpacker(_Unpacker):
+        def __init__(self, file_like=None, read_size=0, use_list=None,
+                     object_hook=None,
+                     object_pairs_hook=None, list_hook=None, encoding='utf-8',
+                     unicode_errors='strict', max_buffer_size=0,
+                     allow_pickle=False):
+            object_hook = functools.partial(decode, chain=object_hook,
+                                            allow_pickle=allow_pickle)
+            super(Unpacker, self).__init__(file_like=file_like,
+                                           read_size=read_size,
+                                           use_list=use_list,
+                                           object_hook=object_hook,
+                                           object_pairs_hook=object_pairs_hook,
+                                           list_hook=list_hook,
+                                           encoding=encoding,
+                                           unicode_errors=unicode_errors,
+                                           max_buffer_size=max_buffer_size)
+
+elif msgpack.version < (1, 0, 0):
+    class Packer(_Packer):
+        def __init__(self, default=None,
+                     unicode_errors='strict',
+                     use_single_float=False,
+                     autoreset=1,
+                     use_bin_type=True,
+                     strict_types=False):
+            default = functools.partial(encode, chain=default)
+            super(Packer, self).__init__(default=default,
+                                         unicode_errors=unicode_errors,
+                                         use_single_float=use_single_float,
+                                         autoreset=autoreset,
+                                         use_bin_type=use_bin_type,
+                                         strict_types=strict_types)
+
+    class Unpacker(_Unpacker):
+        def __init__(self, file_like=None, read_size=0, use_list=None,
+                     raw=False,
+                     object_hook=None,
+                     object_pairs_hook=None, list_hook=None,
+                     unicode_errors='strict', max_buffer_size=0,
+                     ext_hook=msgpack.ExtType,
+                     allow_pickle=False):
+            object_hook = functools.partial(decode, chain=object_hook,
+                                            allow_pickle=allow_pickle)
+            super(Unpacker, self).__init__(file_like=file_like,
+                                           read_size=read_size,
+                                           use_list=use_list,
+                                           raw=raw,
+                                           object_hook=object_hook,
+                                           object_pairs_hook=object_pairs_hook,
+                                           list_hook=list_hook,
+                                           unicode_errors=unicode_errors,
+                                           max_buffer_size=max_buffer_size,
+                                           ext_hook=ext_hook)
+
+else:
+    class Packer(_Packer):
+        def __init__(self,
+                     default=None,
+                     use_single_float=False,
+                     autoreset=True,
+                     use_bin_type=True,
+                     strict_types=False,
+                     datetime=False,
+                     unicode_errors=None):
+            default = functools.partial(encode, chain=default)
+            super(Packer, self).__init__(default=default,
+                                         use_single_float=use_single_float,
+                                         autoreset=autoreset,
+                                         use_bin_type=use_bin_type,
+                                         strict_types=strict_types,
+                                         datetime=datetime,
+                                         unicode_errors=unicode_errors)
+
+    class Unpacker(_Unpacker):
+        def __init__(self,
+                     file_like=None,
+                     read_size=0,
+                     use_list=True,
+                     raw=False,
+                     timestamp=0,
+                     strict_map_key=True,
+                     object_hook=None,
+                     object_pairs_hook=None,
+                     list_hook=None,
+                     unicode_errors=None,
+                     max_buffer_size=100 * 1024 * 1024,
+                     ext_hook=msgpack.ExtType,
+                     max_str_len=-1,
+                     max_bin_len=-1,
+                     max_array_len=-1,
+                     max_map_len=-1,
+                     max_ext_len=-1,
+                     allow_pickle=False):
+            object_hook = functools.partial(decode, chain=object_hook,
+                                            allow_pickle=allow_pickle)
+            super(Unpacker, self).__init__(file_like=file_like,
+                                           read_size=read_size,
+                                           use_list=use_list,
+                                           raw=raw,
+                                           timestamp=timestamp,
+                                           strict_map_key=strict_map_key,
+                                           object_hook=object_hook,
+                                           object_pairs_hook=object_pairs_hook,
+                                           list_hook=list_hook,
+                                           unicode_errors=unicode_errors,
+                                           max_buffer_size=max_buffer_size,
+                                           ext_hook=ext_hook,
+                                           max_str_len=max_str_len,
+                                           max_bin_len=max_bin_len,
+                                           max_array_len=max_array_len,
+                                           max_map_len=max_map_len,
+                                           max_ext_len=max_ext_len)
+
+def pack(o, stream, **kwargs):
+    """
+    Pack an object and write it to a stream.
+    """
+
+    packer = Packer(**kwargs)
+    stream.write(packer.pack(o))
+
+def packb(o, **kwargs):
+    """
+    Pack an object and return the packed bytes.
+    """
+
+    return Packer(**kwargs).pack(o)
+
+def unpack(stream, **kwargs):
+    """
+    Unpack a packed object from a stream.
+    """
+
+    allow_pickle = kwargs.pop('allow_pickle', False)
+    object_hook = kwargs.get('object_hook')
+    kwargs['object_hook'] = functools.partial(decode, chain=object_hook,
+                                              allow_pickle=allow_pickle)
+    return _unpack(stream, **kwargs)
+
+def unpackb(packed, **kwargs):
+    """
+    Unpack a packed object.
+    """
+
+    allow_pickle = kwargs.pop('allow_pickle', False)
+    object_hook = kwargs.get('object_hook')
+    kwargs['object_hook'] = functools.partial(decode, chain=object_hook,
+                                              allow_pickle=allow_pickle)
+    return _unpackb(packed, **kwargs)
+
+load = unpack
+loads = unpackb
+dump = pack
+dumps = packb
+
+def patch():
+    """
+    Monkey patch msgpack module to enable support for serializing numpy types.
+    """
+
+    setattr(msgpack, 'Packer', Packer)
+    setattr(msgpack, 'Unpacker', Unpacker)
+    setattr(msgpack, 'load', unpack)
+    setattr(msgpack, 'loads', unpackb)
+    setattr(msgpack, 'dump', pack)
+    setattr(msgpack, 'dumps', packb)
+    setattr(msgpack, 'pack', pack)
+    setattr(msgpack, 'packb', packb)
+    setattr(msgpack, 'unpack', unpack)
+    setattr(msgpack, 'unpackb', unpackb)