PyStellarDB 0.10.0__py2.py3-none-any.whl → 0.13.1__py2.py3-none-any.whl

PyStellarDB-0.13.1.dist-info/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2014 Transwarp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

{PyStellarDB-0.10.0.dist-info → PyStellarDB-0.13.1.dist-info}/METADATA

@@ -1,38 +1,34 @@
 Metadata-Version: 2.1
 Name: PyStellarDB
-Version: 0.10.0
+Version: 0.13.1
 Summary: Python interface to StellarDB
 Home-page: https://github.com/WarpCloud/PyStellarDB
 Author: Zhiping Wang
 Author-email: zhiping.wang@transwarp.io
 License: Apache License, Version 2.0
-Platform: UNKNOWN
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Database :: Front-Ends
-Requires-Python: >=2.7,<=3.7
-Provides-Extra: kerberos
-Provides-Extra: hive
-Provides-Extra: sqlalchemy
-Provides-Extra: presto
+Requires-Python: >=2.7
+License-File: LICENSE
 Requires-Dist: future
 Requires-Dist: python-dateutil
 Requires-Dist: pyhive
 Requires-Dist: sasl
 Requires-Dist: thrift
-Requires-Dist: thrift-sasl
-Requires-Dist: pyspark (>=2.4.0)
+Requires-Dist: thrift-sasl >=0.3.0
 Provides-Extra: hive
-Requires-Dist: sasl (>=0.2.1); extra == 'hive'
-Requires-Dist: thrift (>=0.10.0); extra == 'hive'
-Requires-Dist: thrift-sasl (>=0.1.0); extra == 'hive'
+Requires-Dist: sasl >=0.2.1 ; extra == 'hive'
+Requires-Dist: thrift >=0.10.0 ; extra == 'hive'
 Provides-Extra: kerberos
-Requires-Dist: requests-kerberos (>=0.12.0); extra == 'kerberos'
+Requires-Dist: requests-kerberos >=0.12.0 ; extra == 'kerberos'
 Provides-Extra: presto
-Requires-Dist: requests (>=1.0.0); extra == 'presto'
+Requires-Dist: requests >=1.0.0 ; extra == 'presto'
+Provides-Extra: pyspark
+Requires-Dist: pyspark >=2.4.0 ; extra == 'pyspark'
 Provides-Extra: sqlalchemy
-Requires-Dist: sqlalchemy (>=1.3.0); extra == 'sqlalchemy'
+Requires-Dist: sqlalchemy >=1.3.0 ; extra == 'sqlalchemy'
 
 PyStellarDB
 ===========
@@ -129,7 +125,7 @@ Execute Graph Query and change to a PySpark RDD object
 
     from pyspark import SparkContext
     from pystellardb import stellar_hive
-
+    
     sc = SparkContext("local", "Demo App")
 
     conn = stellar_hive.StellarConnection(host="localhost", port=10000, graph_name='pokemon')
@@ -154,7 +150,7 @@ Execute Hive Query and change to a PySpark RDD object
 
     from pyspark import SparkContext
     from pystellardb import stellar_hive
-
+    
     sc = SparkContext("local", "Demo App")
 
     conn = stellar_hive.StellarConnection(host="localhost", port=10000)
@@ -175,15 +171,11 @@ Dependencies
 Required:
 ------------
 
-- Python 2.7+ / Less than Python 3.7
+- Python 2.7+ / Python 3
 
 System SASL
 ------------
 
-Different systems require different packages to be installed to enable SASL support
-in Impyla. Some examples of how to install the packages on different distributions
-follow.
-
 Ubuntu:
 
 .. code-block:: bash
@@ -198,14 +190,14 @@ RHEL/CentOS:
     yum install cyrus-sasl-md5 cyrus-sasl-plain cyrus-sasl-gssapi cyrus-sasl-devel
     yum install gcc-c++ python-devel.x86_64     #Update python and gcc if needed
 
-    # If your Python environment is 3.X, then you may need to compile and reinstall Python 
     # if pip3 install fails with a message like 'Can't connect to HTTPS URL because the SSL module is not available'
+    # you may need to update ssl & reinstall python
 
     # 1. Download a higher version of openssl, e.g: https://www.openssl.org/source/openssl-1.1.1k.tar.gz
     # 2. Install openssl: ./config && make && make install
     # 3. Link openssl: echo /usr/local/lib64/ > /etc/ld.so.conf.d/openssl-1.1.1.conf
     # 4. Update dynamic lib: ldconfig -v
-    # 5. Download a Python source package
+    # 5. Uninstall Python & Download a new Python source package
     # 6. vim Modules/Setup, search '_socket socketmodule.c', uncomment
     #    _socket socketmodule.c
     #    SSL=/usr/local/ssl
@@ -215,11 +207,19 @@ RHEL/CentOS:
     #
     # 7. Install Python: ./configure && make && make install
 
+Windows:
+
+.. code-block:: bash
+
+    # There are 3 ways of installing sasl for python on windows
+    # 1. (recommended) Download a .whl version of sasl from https://www.lfd.uci.edu/~gohlke/pythonlibs/#sasl
+    # 2. (recommended) If using anaconda, use conda install sasl.
+    # 3. Install Microsoft Visual C++ 9.0/14.0 buildtools for python2.7/3.x, then pip install sasl.
+
 Notices
 =======
 
-If you install pystellardb >= 0.9, then it will install a beeline command into system.
-Delete /usr/local/bin/beeline if you don't need it. 
+Pystellardb >= 0.9 contains beeline installation to /usr/local/bin/beeline.
 
 Requirements
 ============
@@ -233,8 +233,83 @@ PyHive works with
 - For Hive: `HiveServer2 <https://cwiki.apache.org/confluence/display/Hive/Setting+up+HiveServer2>`_ daemon
 
 
+Windows Kerberos Configuration
+==============================
+
+Windows Kerberos configuration can be a little bit tricky and may need a few instructions.
+First, you'll need to install & configure Kerberos for Windows.
+Get it from http://web.mit.edu/kerberos/dist/
+
+After installation, configure the environment variables.
+Make sure the position of your Kerberos variable is ahead of JDK variable, avoid using kinit command located in JDK path.
+
+Find /etc/krb5.conf on your KDC, copy it into krb5.ini on Windows with some modifications.
+e.g.(krb5.conf on KDC):
+
+.. code-block:: bash
+
+    [logging]
+    default = FILE:/var/log/krb5libs.log
+    kdc = FILE:/var/log/krb5kdc.log
+    admin_server = FILE:/var/log/kadmind.log
+
+    [libdefaults]
+    default_realm = DEFAULT
+    dns_lookup_realm = false
+    dns_lookup_kdc = false
+    ticket_lifetime = 24h
+    renew_lifetime = 7d
+    forwardable = true
+    allow_weak_crypto = true
+    udp_preference_limit = 32700
+    default_ccache_name = FILE:/tmp/krb5cc_%{uid}
+
+    [realms]
+    DEFAULT = {
+    kdc = host1:1088
+    kdc = host2:1088
+    }
+
+Modify it, delete [logging] and default_ccache_name in [libdefaults]:
+
+.. code-block:: bash
+
+    [libdefaults]
+    default_realm = DEFAULT
+    dns_lookup_realm = false
+    dns_lookup_kdc = false
+    ticket_lifetime = 24h
+    renew_lifetime = 7d
+    forwardable = true
+    allow_weak_crypto = true
+    udp_preference_limit = 32700
+
+    [realms]
+    DEFAULT = {
+    kdc = host1:1088
+    kdc = host2:1088
+    }
+
+Above is your krb5.ini for Kerberos on Windows. Put it at 3 places:
+
+    C:\ProgramData\MIT\Kerberos5\krb5.ini
+
+    C:\Program Files\MIT\Kerberos\krb5.ini
+
+    C:\Windows\krb5.ini
+
+
+Finally, configure hosts file at: C:/Windows/System32/drivers/etc/hosts
+Add ip mappings of host1, host2 in the previous example. e.g.
+
+.. code-block:: bash
+
+    10.6.6.96     host1
+    10.6.6.97     host2
+
+Now, you can try running kinit in your command line!
+
 Testing
 =======
 
 On his way
-

PyStellarDB-0.13.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+pystellardb/__init__.py,sha256=JOl41NviMN-qDV0Z8ZPmhNIxvgyauGGJHdB4A-8MhqM,93
+pystellardb/_version.py,sha256=Enm9C1OfbLkTuBMZ_UZNpU-dIdr1fSxOb6XVICKMawU,498
+pystellardb/graph_types.py,sha256=S8XIRwKP2-DLbOpgUHXzbs68SHFZeydlKs7h4jVcdIg,12351
+pystellardb/stellar_hive.py,sha256=Bes99go4oKszP0RiD3OYG3W5g0Sx0cnaXf2yWOosXk0,14010
+pystellardb/stellar_rdd.py,sha256=TYwsWYeCxfOliGq1kV3ArNXdye55cKWZF7s9M9nDdt4,1324
+PyStellarDB-0.13.1.dist-info/LICENSE,sha256=1qDFxrywejs7xNBfOr6T-7lOuqDgSNIES77kTYege3w,560
+PyStellarDB-0.13.1.dist-info/METADATA,sha256=1ZD8HOiC_gp2E3mMcyQVDdJynVWar6EANpDALj52V1w,9390
+PyStellarDB-0.13.1.dist-info/WHEEL,sha256=_4XEmVmaBFWtekSGrbfOGNjC2I5lUr0lZSRblBllIFA,109
+PyStellarDB-0.13.1.dist-info/top_level.txt,sha256=DRk-SeGVCdVAzv2CwFmdu75Yo7DgjUA3Hpu-9l8qPuU,12
+PyStellarDB-0.13.1.dist-info/RECORD,,

{PyStellarDB-0.10.0.dist-info → PyStellarDB-0.13.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.31.1)
+Generator: setuptools (70.1.1)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any

pystellardb/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2021-05-27T19:02:40+0800",
+ "date": "2024-08-20T10:49:46+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "43719bd5c27d93c6809c68f12f87e8a6b4e94dc1",
- "version": "0.10.0"
+ "full-revisionid": "b83cfb5780b16466ce79235a756422bb00844b1b",
+ "version": "0.13.1"
 }
 '''  # END VERSION_JSON
 

pystellardb/graph_types.py

@@ -69,7 +69,7 @@ class Vertex(GraphElement):
         return m
 
     def __str__(self):
-        return json.dumps(self.toJSON())
+        return json.dumps(self.toJSON(), ensure_ascii=False)
 
     @staticmethod
     def parseVertexFromJson(json_str):
@@ -84,23 +84,26 @@ class Vertex(GraphElement):
         if 'labels' not in m:
             raise ValueError("Could not find label in JSON")
 
-        if '__uid' not in m['properties']:
+        prop_dict = m['properties']
+
+        if '__uid' not in prop_dict:
             raise ValueError("Could not find uid in JSON")
 
-        vertex = Vertex(m['properties']['__uid'], m['labels'][0])
+        vertex = Vertex(prop_dict['__uid'], m['labels'][0])
 
-        for key in m['properties'].keys():
+        for key in prop_dict.keys():
             if key != '__uid' and key != '__tags':
-                vertex.setFeild(key, m['properties'][key])
+                vertex.setFeild(key, prop_dict[key])
 
-        vertex.setTags(m['properties']['__tags'])
+        if '__tags' in prop_dict:
+            vertex.setTags(prop_dict['__tags'])
 
         return vertex
 
     @staticmethod
     def parseUidFromRK(rk):
         """Parse user-defined id from vertex row key in byte array"""
-        return ''.join([chr(x) for x in rk[:-2]])
+        return bytearray([x & 0xFF for x in rk[:-2]]).decode('utf-8')
 
     @staticmethod
     def parseLabelIdxFromRK(rk):
@@ -110,6 +113,31 @@ class Vertex(GraphElement):
         label_in_little_endian.reverse()
         return int(binascii.hexlify(bytearray(label_in_little_endian)), 16)
 
+    @staticmethod
+    def parseShardIdFromRKV18(rk):
+        """Parse shard id from vertex row key in byte array for graphSchema V18"""
+        shard_id = (rk[0] & 0xFF) << 8
+        shard_id |= rk[1] & 0xF0
+        return int(shard_id >> 4)
+
+    @staticmethod
+    def parseLabelIdxFromRKV18(rk):
+        """Parse label index from vertex row key in byte array for graphSchema V18"""
+        label_index = (rk[1] & 0x0F) << 8
+        label_index |= rk[2] & 0xFF
+        return int(label_index)
+
+    @staticmethod
+    def parseInnerIdFromRKV18(rk, offset):
+        """Parse long type inner id from vertex row key in byte array for graphSchema V18"""
+        ID_LEN = 8
+        inner_id = rk[offset + ID_LEN - 1] & 0x00FF
+        inner_id |= (rk[offset + ID_LEN - 2] & 0x00FF) << 8
+        inner_id |= (rk[offset + ID_LEN - 3] & 0x00FF) << 16
+        inner_id |= (rk[offset + ID_LEN - 4] & 0x00FF) << 24
+        inner_id |= (rk[offset + ID_LEN - 5] & 0x00FF) << 32
+        return int(inner_id)
+
 
 class Edge(GraphElement):
     """
@@ -159,7 +187,7 @@ class Edge(GraphElement):
         return m
 
     def __str__(self):
-        return json.dumps(self.toJSON())
+        return json.dumps(self.toJSON(), ensure_ascii=False)
 
     @staticmethod
     def parseEdgeFromJson(schema, json_str):
@@ -176,12 +204,18 @@ class Edge(GraphElement):
 
         edge = Edge(m['labels'][0])
 
+        prop_dict = m['properties']
+
         # parse start node
         if 'startKey' not in m:
             raise ValueError("Could not find start node entity key in JSON")
 
-        startUid = Vertex.parseUidFromRK(m['startKey'])
-        startLabelIdx = Vertex.parseLabelIdxFromRK(m['startKey'])
+        if schema.getVersion() == 18:
+            startUid = prop_dict['__srcuid']
+            startLabelIdx = Vertex.parseLabelIdxFromRKV18(m['startKey'])
+        else:
+            startUid = Vertex.parseUidFromRK(m['startKey'])
+            startLabelIdx = Vertex.parseLabelIdxFromRK(m['startKey'])
         startLabel = schema.getVertexLabel(startLabelIdx)
 
         if startLabel is None:
@@ -195,8 +229,12 @@ class Edge(GraphElement):
         if 'endKey' not in m:
             raise ValueError("Could not find end node entity key in JSON")
 
-        endUid = Vertex.parseUidFromRK(m['endKey'])
-        endLabelIdx = Vertex.parseLabelIdxFromRK(m['endKey'])
+        if schema.getVersion() == 18:
+            endUid = prop_dict['__dstuid']
+            endLabelIdx = Vertex.parseLabelIdxFromRKV18(m['endKey'])
+        else:
+            endUid = Vertex.parseUidFromRK(m['endKey'])
+            endLabelIdx = Vertex.parseLabelIdxFromRK(m['endKey'])
         endLabel = schema.getVertexLabel(endLabelIdx)
 
         if endLabel is None:
@@ -207,16 +245,17 @@ class Edge(GraphElement):
         edge.setEndNode(Vertex(endUid, endLabel))
 
         # parse extra edge id
-        if '__uid' in m['properties']:
-            edge.setUid(m['properties']['__uid'])
+        if '__uid' in prop_dict:
+            edge.setUid(prop_dict['__uid'])
 
         # parse properties
-        for key in m['properties'].keys():
+        for key in prop_dict.keys():
             if key != '__uid' and key != '__tags':
                 edge.setFeild(key, m['properties'][key])
 
         # parse tags
-        edge.setTags(m['properties']['__tags'])
+        if '__tags' in prop_dict:
+            edge.setTags(prop_dict['__tags'])
 
         return edge
 
@@ -298,6 +337,9 @@ class GraphSchema(object):
 
         return None
 
+    def getVersion(self):
+        return self._schema_version
+
     def toJSON(self):
         m = {
             '__VERSION': self._schema_version,
@@ -320,7 +362,7 @@ class GraphSchema(object):
         return m
 
     def __str__(self):
-        return json.dumps(self.toJSON())
+        return json.dumps(self.toJSON(), ensure_ascii=False)
 
     @staticmethod
     def parseSchemaFromJson(json_str):

pystellardb/stellar_hive.py

@@ -175,6 +175,7 @@ class StellarConnection(object):
                 assert response.serverProtocolVersion == protocol_version, \
                     "Unable to handle protocol version {}".format(response.serverProtocolVersion)
                 with contextlib.closing(self.cursor()) as cursor:
+                    cursor.execute('set query.lang=sql')
                     cursor.execute('USE `{}`'.format(database))
         except:
             self._transport.close()
@@ -304,7 +305,7 @@ class StellarCursor(hive.Cursor):
         elif type == 'int':
             return int(data)
         elif type == 'long':
-            return long(data)
+            return int(data)
         elif type == 'float' or type == 'double':
             return float(data)
         elif type == 'CruxType:Node' or type == 'GraphNode':
@@ -323,9 +324,9 @@ class StellarCursor(hive.Cursor):
     def _parseList(self, type, data):
         """Parse 'CruxType:List' type"""
         parsed_data = json.loads(data)
-        newType = type[len('CruxType:List') + 1:-2]
+        newType = type[len('CruxType:List') + 1:type.find('>')]
 
-        return [self._convertData(newType, entry) for entry in parsed_data]
+        return [self._convertData(newType, json.dumps(entry)) for entry in parsed_data]
 
     def _parseMap(self, type, data):
         """Parse 'CruxType:Map' type"""

pystellardb/stellar_rdd.py

@@ -6,8 +6,13 @@ from __future__ import absolute_import
 import abc
 from future.utils import with_metaclass
 import logging
-from pyspark import RDD, SparkContext
-from pyspark.serializers import BatchedSerializer
+
+try:
+    import pyspark
+    from pyspark import RDD, SparkContext
+    from pyspark.serializers import BatchedSerializer
+except ImportError:
+    pyspark = None
 
 _logger = logging.getLogger(__name__)
 
@@ -20,6 +25,9 @@ def transformToRDD(cursor, sc, parallelism=1):
     param sc: SparkContext
     param parallelism: Parallelism of RDD
     """
+    if not pyspark:
+        raise ImportError("Could not import pyspark! Please run `pip install pyspark` first in your environment!")
+
     # Get all data from cursor
     data = cursor.fetchall()
 

PyStellarDB-0.10.0.dist-info/RECORD

@@ -1,9 +0,0 @@
-PyStellarDB-0.10.0.dist-info/METADATA,sha256=a56Ql_hi8Qs7um-EJ09SdQNN-g7m9tF8Ts-UE-vET_c,7449
-PyStellarDB-0.10.0.dist-info/RECORD,,
-PyStellarDB-0.10.0.dist-info/WHEEL,sha256=gduuPyBvFJQSQ0zdyxF7k0zynDXbIbvg5ZBHoXum5uk,110
-PyStellarDB-0.10.0.dist-info/top_level.txt,sha256=DRk-SeGVCdVAzv2CwFmdu75Yo7DgjUA3Hpu-9l8qPuU,12
-pystellardb/__init__.py,sha256=JOl41NviMN-qDV0Z8ZPmhNIxvgyauGGJHdB4A-8MhqM,93
-pystellardb/_version.py,sha256=D2S0_MWfM10J4kvlKlKQ5Bxc8TtsWanAaDppYEAl_G4,498
-pystellardb/graph_types.py,sha256=-zMzW2V2egpuDUSiIuD-JZpehkzvqljOLOM8aqaspFc,10770
-pystellardb/stellar_hive.py,sha256=jJxwKXTdmdPMoi6b61LghXZX3B6cX9-a5UinVsUbpEA,13930
-pystellardb/stellar_rdd.py,sha256=IQjK0WDO2FaIERqT-cwkRFcoqCgwCpBZXGlcYEcdALI,1116