PyStellarDB 0.11.0__py2.py3-none-any.whl → 0.13.2__py2.py3-none-any.whl

PyStellarDB-0.13.2.dist-info/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2014 Transwarp, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

{PyStellarDB-0.11.0.dist-info → PyStellarDB-0.13.2.dist-info}/METADATA

@@ -1,37 +1,34 @@
 Metadata-Version: 2.1
 Name: PyStellarDB
-Version: 0.11.0
+Version: 0.13.2
 Summary: Python interface to StellarDB
 Home-page: https://github.com/WarpCloud/PyStellarDB
 Author: Zhiping Wang
 Author-email: zhiping.wang@transwarp.io
 License: Apache License, Version 2.0
-Platform: UNKNOWN
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Database :: Front-Ends
-Requires-Python: >=2.7,<=3.7
-Provides-Extra: hive
-Provides-Extra: sqlalchemy
-Provides-Extra: kerberos
-Provides-Extra: presto
+Requires-Python: >=2.7
+License-File: LICENSE
 Requires-Dist: future
 Requires-Dist: python-dateutil
 Requires-Dist: pyhive
 Requires-Dist: sasl
 Requires-Dist: thrift
-Requires-Dist: thrift-sasl (>=0.3.0)
-Requires-Dist: pyspark (>=2.4.0)
+Requires-Dist: thrift-sasl >=0.3.0
 Provides-Extra: hive
-Requires-Dist: sasl (>=0.2.1); extra == 'hive'
-Requires-Dist: thrift (>=0.10.0); extra == 'hive'
+Requires-Dist: sasl >=0.2.1 ; extra == 'hive'
+Requires-Dist: thrift >=0.10.0 ; extra == 'hive'
 Provides-Extra: kerberos
-Requires-Dist: requests-kerberos (>=0.12.0); extra == 'kerberos'
+Requires-Dist: requests-kerberos >=0.12.0 ; extra == 'kerberos'
 Provides-Extra: presto
-Requires-Dist: requests (>=1.0.0); extra == 'presto'
+Requires-Dist: requests >=1.0.0 ; extra == 'presto'
+Provides-Extra: pyspark
+Requires-Dist: pyspark >=2.4.0 ; extra == 'pyspark'
 Provides-Extra: sqlalchemy
-Requires-Dist: sqlalchemy (>=1.3.0); extra == 'sqlalchemy'
+Requires-Dist: sqlalchemy >=1.3.0 ; extra == 'sqlalchemy'
 
 PyStellarDB
 ===========
@@ -128,7 +125,7 @@ Execute Graph Query and change to a PySpark RDD object
 
     from pyspark import SparkContext
     from pystellardb import stellar_hive
-
+    
     sc = SparkContext("local", "Demo App")
 
     conn = stellar_hive.StellarConnection(host="localhost", port=10000, graph_name='pokemon')
@@ -153,7 +150,7 @@ Execute Hive Query and change to a PySpark RDD object
 
     from pyspark import SparkContext
     from pystellardb import stellar_hive
-
+    
     sc = SparkContext("local", "Demo App")
 
     conn = stellar_hive.StellarConnection(host="localhost", port=10000)
@@ -174,15 +171,11 @@ Dependencies
 Required:
 ------------
 
-- Python 2.7+ / Less than Python 3.7
+- Python 2.7+ / Python 3
 
 System SASL
 ------------
 
-Different systems require different packages to be installed to enable SASL support.
-Some examples of how to install the packages on different distributions
-follow.
-
 Ubuntu:
 
 .. code-block:: bash
@@ -197,14 +190,14 @@ RHEL/CentOS:
     yum install cyrus-sasl-md5 cyrus-sasl-plain cyrus-sasl-gssapi cyrus-sasl-devel
     yum install gcc-c++ python-devel.x86_64     #Update python and gcc if needed
 
-    # If your Python environment is 3.X, then you may need to compile and reinstall Python 
     # if pip3 install fails with a message like 'Can't connect to HTTPS URL because the SSL module is not available'
+    # you may need to update ssl & reinstall python
 
     # 1. Download a higher version of openssl, e.g: https://www.openssl.org/source/openssl-1.1.1k.tar.gz
     # 2. Install openssl: ./config && make && make install
     # 3. Link openssl: echo /usr/local/lib64/ > /etc/ld.so.conf.d/openssl-1.1.1.conf
     # 4. Update dynamic lib: ldconfig -v
-    # 5. Download a Python source package
+    # 5. Uninstall Python & Download a new Python source package
     # 6. vim Modules/Setup, search '_socket socketmodule.c', uncomment
     #    _socket socketmodule.c
     #    SSL=/usr/local/ssl
@@ -221,13 +214,12 @@ Windows:
     # There are 3 ways of installing sasl for python on windows
     # 1. (recommended) Download a .whl version of sasl from https://www.lfd.uci.edu/~gohlke/pythonlibs/#sasl
     # 2. (recommended) If using anaconda, use conda install sasl.
-    # 3. Install Microsoft Visual C++ 9.0/14.0 buildtools for python2.7/3.x, then pip install sasl(under test).
+    # 3. Install Microsoft Visual C++ 9.0/14.0 buildtools for python2.7/3.x, then pip install sasl.
 
 Notices
 =======
 
-If you install pystellardb >= 0.9, then it will install a beeline command into system.
-Delete /usr/local/bin/beeline if you don't need it. 
+Pystellardb >= 0.9 contains beeline installation to /usr/local/bin/beeline.
 
 Requirements
 ============
@@ -244,12 +236,12 @@ PyHive works with
 Windows Kerberos Configuration
 ==============================
 
-If you're connecting to databases using Kerberos authentication from Windows platform,
-you'll need to install & configure Kerberos for Windows first.
+Windows Kerberos configuration can be a little bit tricky and may need a few instructions.
+First, you'll need to install & configure Kerberos for Windows.
 Get it from http://web.mit.edu/kerberos/dist/
 
 After installation, configure the environment variables.
-Make sure your Kerberos variable is set ahead of JDK variable(If you have JDK), because JDK also has kinit etc.
+Make sure the position of your Kerberos variable is ahead of JDK variable, avoid using kinit command located in JDK path.
 
 Find /etc/krb5.conf on your KDC, copy it into krb5.ini on Windows with some modifications.
 e.g.(krb5.conf on KDC):
@@ -298,7 +290,7 @@ Modify it, delete [logging] and default_ccache_name in [libdefaults]:
     kdc = host2:1088
     }
 
-This is your krb5.ini for Windows Kerberos. Put it at those 3 places:
+Above is your krb5.ini for Kerberos on Windows. Put it at 3 places:
 
     C:\ProgramData\MIT\Kerberos5\krb5.ini
 
@@ -307,7 +299,7 @@ This is your krb5.ini for Windows Kerberos. Put it at those 3 places:
     C:\Windows\krb5.ini
 
 
-Finally, configure hosts at: C:/Windows/System32/drivers/etc/hosts
+Finally, configure hosts file at: C:/Windows/System32/drivers/etc/hosts
 Add ip mappings of host1, host2 in the previous example. e.g.
 
 .. code-block:: bash
@@ -315,11 +307,9 @@ Add ip mappings of host1, host2 in the previous example. e.g.
     10.6.6.96     host1
     10.6.6.97     host2
 
-Now, you can run kinit in the command line!
+Now, you can try running kinit in your command line!
 
 Testing
 =======
 
 On his way
-
-

PyStellarDB-0.13.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+pystellardb/__init__.py,sha256=JOl41NviMN-qDV0Z8ZPmhNIxvgyauGGJHdB4A-8MhqM,93
+pystellardb/_version.py,sha256=Vt7qCjCMBamE10PReIKwIvI02pMh8mdLJE8ZY2c6T54,498
+pystellardb/graph_types.py,sha256=j9ZEvnTVRFOttg28rcYvOFzfoOBJcRxXxySKIzEcR-I,13098
+pystellardb/stellar_hive.py,sha256=Bes99go4oKszP0RiD3OYG3W5g0Sx0cnaXf2yWOosXk0,14010
+pystellardb/stellar_rdd.py,sha256=TYwsWYeCxfOliGq1kV3ArNXdye55cKWZF7s9M9nDdt4,1324
+PyStellarDB-0.13.2.dist-info/LICENSE,sha256=1qDFxrywejs7xNBfOr6T-7lOuqDgSNIES77kTYege3w,560
+PyStellarDB-0.13.2.dist-info/METADATA,sha256=sY89aLWXtPh-MetCwvHzXvwj37_-fqujLLoOqKjMaf8,9390
+PyStellarDB-0.13.2.dist-info/WHEEL,sha256=_4XEmVmaBFWtekSGrbfOGNjC2I5lUr0lZSRblBllIFA,109
+PyStellarDB-0.13.2.dist-info/top_level.txt,sha256=DRk-SeGVCdVAzv2CwFmdu75Yo7DgjUA3Hpu-9l8qPuU,12
+PyStellarDB-0.13.2.dist-info/RECORD,,

{PyStellarDB-0.11.0.dist-info → PyStellarDB-0.13.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.31.1)
+Generator: setuptools (70.1.1)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any

pystellardb/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2022-03-22T19:24:40+0800",
+ "date": "2024-09-05T19:42:39+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "f97fb8f8f488a4f3201f61b29a1bc421a3c88ac2",
- "version": "0.11.0"
+ "full-revisionid": "9e31319f3dbef3dc053f379b94f099e358d589a5",
+ "version": "0.13.2"
 }
 '''  # END VERSION_JSON
 

pystellardb/graph_types.py

@@ -18,6 +18,7 @@ class GraphElement(with_metaclass(abc.ABCMeta, object)):
         self._label = label
         self._fields = {}
         self._tags = []
+        self._rowKeyHexString = None
 
     def getLabel(self):
         return self._label
@@ -40,6 +41,12 @@ class GraphElement(with_metaclass(abc.ABCMeta, object)):
     def setTags(self, newTags):
         self._tags = newTags
 
+    def setRowKeyHexString(self, rowkey):
+        self._rowKeyHexString = rowkey
+
+    def getRowKeyHexString(self):
+        return self._rowKeyHexString
+
 
 class Vertex(GraphElement):
     """
@@ -58,6 +65,7 @@ class Vertex(GraphElement):
             'type': 'vertex',
             'label': self._label,
             'uid': self._uid,
+            'RowKeyHexString': self._rowKeyHexString,
         }
 
         if self._tags is not None and len(self._tags) > 0:
@@ -69,7 +77,7 @@ class Vertex(GraphElement):
         return m
 
     def __str__(self):
-        return json.dumps(self.toJSON())
+        return json.dumps(self.toJSON(), ensure_ascii=False)
 
     @staticmethod
     def parseVertexFromJson(json_str):
@@ -84,16 +92,22 @@ class Vertex(GraphElement):
         if 'labels' not in m:
             raise ValueError("Could not find label in JSON")
 
-        if '__uid' not in m['properties']:
+        prop_dict = m['properties']
+
+        if '__uid' not in prop_dict:
             raise ValueError("Could not find uid in JSON")
 
-        vertex = Vertex(m['properties']['__uid'], m['labels'][0])
+        vertex = Vertex(prop_dict['__uid'], m['labels'][0])
 
-        for key in m['properties'].keys():
+        for key in prop_dict.keys():
             if key != '__uid' and key != '__tags':
-                vertex.setFeild(key, m['properties'][key])
+                vertex.setFeild(key, prop_dict[key])
+
+        if '__tags' in prop_dict:
+            vertex.setTags(prop_dict['__tags'])
 
-        vertex.setTags(m['properties']['__tags'])
+        rk = " ".join(map(lambda x: str(x), m['entityKey']))
+        vertex.setRowKeyHexString(rk)
 
         return vertex
 
@@ -110,6 +124,31 @@ class Vertex(GraphElement):
         label_in_little_endian.reverse()
         return int(binascii.hexlify(bytearray(label_in_little_endian)), 16)
 
+    @staticmethod
+    def parseShardIdFromRKV18(rk):
+        """Parse shard id from vertex row key in byte array for graphSchema V18"""
+        shard_id = (rk[0] & 0xFF) << 8
+        shard_id |= rk[1] & 0xF0
+        return int(shard_id >> 4)
+
+    @staticmethod
+    def parseLabelIdxFromRKV18(rk):
+        """Parse label index from vertex row key in byte array for graphSchema V18"""
+        label_index = (rk[1] & 0x0F) << 8
+        label_index |= rk[2] & 0xFF
+        return int(label_index)
+
+    @staticmethod
+    def parseInnerIdFromRKV18(rk, offset):
+        """Parse long type inner id from vertex row key in byte array for graphSchema V18"""
+        ID_LEN = 8
+        inner_id = rk[offset + ID_LEN - 1] & 0x00FF
+        inner_id |= (rk[offset + ID_LEN - 2] & 0x00FF) << 8
+        inner_id |= (rk[offset + ID_LEN - 3] & 0x00FF) << 16
+        inner_id |= (rk[offset + ID_LEN - 4] & 0x00FF) << 24
+        inner_id |= (rk[offset + ID_LEN - 5] & 0x00FF) << 32
+        return int(inner_id)
+
 
 class Edge(GraphElement):
     """
@@ -148,6 +187,7 @@ class Edge(GraphElement):
             'euid': self._uid,
             'startNode': self._startNode.toJSON(),
             'endNode': self._endNode.toJSON(),
+            'RowKeyHexString': self._rowKeyHexString,
         }
 
         if self._tags is not None and len(self._tags) > 0:
@@ -159,7 +199,7 @@ class Edge(GraphElement):
         return m
 
     def __str__(self):
-        return json.dumps(self.toJSON())
+        return json.dumps(self.toJSON(), ensure_ascii=False)
 
     @staticmethod
     def parseEdgeFromJson(schema, json_str):
@@ -176,27 +216,42 @@ class Edge(GraphElement):
 
         edge = Edge(m['labels'][0])
 
+        rk = " ".join(map(lambda x: str(x), m['entityKey']))
+        edge.setRowKeyHexString(rk)
+
+        prop_dict = m['properties']
+
         # parse start node
         if 'startKey' not in m:
             raise ValueError("Could not find start node entity key in JSON")
 
-        startUid = Vertex.parseUidFromRK(m['startKey'])
-        startLabelIdx = Vertex.parseLabelIdxFromRK(m['startKey'])
+        if schema.getVersion() == 18:
+            startUid = prop_dict['__srcuid']
+            startLabelIdx = Vertex.parseLabelIdxFromRKV18(m['startKey'])
+        else:
+            startUid = Vertex.parseUidFromRK(m['startKey'])
+            startLabelIdx = Vertex.parseLabelIdxFromRK(m['startKey'])
         startLabel = schema.getVertexLabel(startLabelIdx)
 
         if startLabel is None:
             raise ValueError(
                 'Could not find start node label with label index `{}`'.format(
                     startLabelIdx))
-
-        edge.setStartNode(Vertex(startUid, startLabel))
+        
+        start_node = Vertex(startUid, startLabel)
+        start_node.setRowKeyHexString(" ".join(map(lambda x: str(x), m['entityKey'][:8])))
+        edge.setStartNode(start_node)
 
         # parse end node
         if 'endKey' not in m:
             raise ValueError("Could not find end node entity key in JSON")
 
-        endUid = Vertex.parseUidFromRK(m['endKey'])
-        endLabelIdx = Vertex.parseLabelIdxFromRK(m['endKey'])
+        if schema.getVersion() == 18:
+            endUid = prop_dict['__dstuid']
+            endLabelIdx = Vertex.parseLabelIdxFromRKV18(m['endKey'])
+        else:
+            endUid = Vertex.parseUidFromRK(m['endKey'])
+            endLabelIdx = Vertex.parseLabelIdxFromRK(m['endKey'])
         endLabel = schema.getVertexLabel(endLabelIdx)
 
         if endLabel is None:
@@ -204,19 +259,22 @@ class Edge(GraphElement):
                 'Could not find end node label with label index `{}`'.format(
                     endLabelIdx))
 
-        edge.setEndNode(Vertex(endUid, endLabel))
+        end_node = Vertex(endUid, endLabel)
+        end_node.setRowKeyHexString(" ".join(map(lambda x: str(x), m['entityKey'][8:16])))
+        edge.setEndNode(end_node)
 
         # parse extra edge id
-        if '__uid' in m['properties']:
-            edge.setUid(m['properties']['__uid'])
+        if '__uid' in prop_dict:
+            edge.setUid(prop_dict['__uid'])
 
         # parse properties
-        for key in m['properties'].keys():
+        for key in prop_dict.keys():
             if key != '__uid' and key != '__tags':
                 edge.setFeild(key, m['properties'][key])
 
         # parse tags
-        edge.setTags(m['properties']['__tags'])
+        if '__tags' in prop_dict:
+            edge.setTags(prop_dict['__tags'])
 
         return edge
 
@@ -298,6 +356,9 @@ class GraphSchema(object):
 
         return None
 
+    def getVersion(self):
+        return self._schema_version
+
     def toJSON(self):
         m = {
             '__VERSION': self._schema_version,
@@ -320,7 +381,7 @@ class GraphSchema(object):
         return m
 
     def __str__(self):
-        return json.dumps(self.toJSON())
+        return json.dumps(self.toJSON(), ensure_ascii=False)
 
     @staticmethod
     def parseSchemaFromJson(json_str):

pystellardb/stellar_hive.py

@@ -305,7 +305,7 @@ class StellarCursor(hive.Cursor):
         elif type == 'int':
             return int(data)
         elif type == 'long':
-            return long(data)
+            return int(data)
         elif type == 'float' or type == 'double':
             return float(data)
         elif type == 'CruxType:Node' or type == 'GraphNode':
@@ -324,9 +324,9 @@ class StellarCursor(hive.Cursor):
     def _parseList(self, type, data):
         """Parse 'CruxType:List' type"""
         parsed_data = json.loads(data)
-        newType = type[len('CruxType:List') + 1:-2]
+        newType = type[len('CruxType:List') + 1:type.find('>')]
 
-        return [self._convertData(newType, entry) for entry in parsed_data]
+        return [self._convertData(newType, json.dumps(entry)) for entry in parsed_data]
 
     def _parseMap(self, type, data):
         """Parse 'CruxType:Map' type"""

pystellardb/stellar_rdd.py

@@ -6,8 +6,13 @@ from __future__ import absolute_import
 import abc
 from future.utils import with_metaclass
 import logging
-from pyspark import RDD, SparkContext
-from pyspark.serializers import BatchedSerializer
+
+try:
+    import pyspark
+    from pyspark import RDD, SparkContext
+    from pyspark.serializers import BatchedSerializer
+except ImportError:
+    pyspark = None
 
 _logger = logging.getLogger(__name__)
 
@@ -20,6 +25,9 @@ def transformToRDD(cursor, sc, parallelism=1):
     param sc: SparkContext
     param parallelism: Parallelism of RDD
     """
+    if not pyspark:
+        raise ImportError("Could not import pyspark! Please run `pip install pyspark` first in your environment!")
+
     # Get all data from cursor
     data = cursor.fetchall()
 

PyStellarDB-0.11.0.dist-info/RECORD

@@ -1,9 +0,0 @@
-PyStellarDB-0.11.0.dist-info/METADATA,sha256=u6LLBzhsgZ1CfCagDPa2KyLveV_DdEpxJz0dhicxL0s,9690
-PyStellarDB-0.11.0.dist-info/RECORD,,
-PyStellarDB-0.11.0.dist-info/WHEEL,sha256=gduuPyBvFJQSQ0zdyxF7k0zynDXbIbvg5ZBHoXum5uk,110
-PyStellarDB-0.11.0.dist-info/top_level.txt,sha256=DRk-SeGVCdVAzv2CwFmdu75Yo7DgjUA3Hpu-9l8qPuU,12
-pystellardb/__init__.py,sha256=JOl41NviMN-qDV0Z8ZPmhNIxvgyauGGJHdB4A-8MhqM,93
-pystellardb/_version.py,sha256=tZcdkmH0v4bTLZeo-KGiXZxkD4WbeXBKdH2pUoCaDmA,498
-pystellardb/graph_types.py,sha256=uWBLqPJBKLJ3OoeyFa59thpka0fcYaAwDTcKBH9zeaE,10790
-pystellardb/stellar_hive.py,sha256=SMTM-C65kNA7fn0pziW_UNEk-GnxLAtt9Nt0Js1gzX8,13987
-pystellardb/stellar_rdd.py,sha256=IQjK0WDO2FaIERqT-cwkRFcoqCgwCpBZXGlcYEcdALI,1116