sop4py 2.0.0__py3-none-any.whl

sop/transaction.py

@@ -0,0 +1,167 @@
+import json
+import call_go
+import uuid
+import context
+
+from enum import Enum
+from dataclasses import dataclass, asdict
+
+
+class TransactionMode(Enum):
+    """
+    Transaction mode enumeration.
+
+    Args:
+        Enum (_type_):
+        NoCheck - No check on commit, most performant but does NOT guarantee ACIDity
+        ForWriting - Mode for writing/updates/deletes.
+        ForReading - Mode for reading only. Checks all read items' version # on commit to guarantee ACIDity.
+    """
+
+    NoCheck = 0
+    ForWriting = 1
+    ForReading = 2
+
+
+# 250, should generate 1MB file segment. Formula: 250 X 4096 = 1MB
+# Given a 50 slot size per node, should be able to manage 825,000 B-Tree items (key/value pairs).
+#
+# Formula: 250 * 66 * 50 = 825,000
+# Or if you use 100 slot size per node, 'will give you 1,650,000 items, or assuming you have about 65%
+# b-tree utilization, 1,072,500 usable space.
+MIN_HASH_MOD_VALUE = 250
+# 750k, should generate 3GB file segment.  Formula: 750k X 4096 = 3GB
+MAX_HASH_MOD_VALUE = 750000
+
+
+@dataclass
+class ErasureCodingConfig:
+    """
+    Erasure Coding config is used to package the parameter configuration of Reed Solomon based EC replication.
+    This is a special algorithm for replication allowing full operational capability even if you lose a half of your
+    storage drives. :)
+
+    For example, if you have 100% redundancy on four drives, losing two drives SOP will still be able to give you Read & Write.
+    BUT of course, your IT needs to replace the drives and allow SOP to auto-reconstruct the redundant "shards" so your setup
+    can offer tolerance once again.
+
+    Returns:
+        _type_: _description_
+    """
+
+    data_shards_count: int
+    parity_shards_count: int
+    base_folder_paths_across_drives: str
+    repair_corrupted_shards: bool
+
+    def __eq__(self, other):
+        return (
+            self.data_shards_count == other.data_shards_count
+            and self.parity_shards_count == other.parity_shards_count
+            and self.repair_corrupted_shards == other.repair_corrupted_shards
+            and self.base_folder_paths_across_drives
+            == other.base_folder_paths_across_drives
+        )
+
+    def __hash__(self):
+        return hash(
+            (
+                self.data_shards_count,
+                self.parity_shards_count,
+                self.base_folder_paths_across_drives,
+                self.repair_corrupted_shards,
+            )
+        )
+
+
+@dataclass
+class TransationOptions:
+    mode: int
+    # max_time in Python is in minutes, SOP in Golang will convert that to respective time.duration value.
+    max_time: int
+    # Registry hash mod, minimum value is 250, max is 750000. Hash mod is used on Registry map on disk.
+    # At 250, 1MB segment file is generated. See comment about the equivalent in Golang side (for now).
+    registry_hash_mod: int
+    # Stores' base folder path (home folder).
+    stores_folders: str
+    # EC config.
+    erasure_config: dict[str, ErasureCodingConfig]
+
+
+class TransactionError(Exception):
+    """Base exception for transaction-related errors."""
+
+    pass
+
+
+class InvalidTransactionStateError(TransactionError):
+    """Raised when a transaction is attempted in an invalid state."""
+
+    pass
+
+
+class Transaction:
+    """
+    Transaction object is used to manage transaction (begin, commit, rollback).
+    Delegates API calls to the SOP library that does Direct IO to disk drives w/ built-in L1/L2 caching.
+    """
+
+    def __init__(self, ctx: context.Context, options: TransationOptions):
+        self.options = options
+        self.transaction_id = uuid.UUID(int=0)
+
+        res = call_go.manage_transaction(ctx.id, 1, json.dumps(asdict(options)))
+
+        if res == None:
+            raise TransactionError("unable to create a Tranasaction object in SOP")
+        try:
+            self.transaction_id = uuid.UUID(res)
+        except:
+            # if res can't be converted to UUID, it is expected to be an error msg from SOP.
+            raise TransactionError(res)
+
+    def begin(self):
+        if self.transaction_id == uuid.UUID(int=0):
+            raise InvalidTransactionStateError("transaction_id is missing")
+        res = call_go.manage_transaction(0, 2, str(self.transaction_id))
+        if res != None:
+            raise TransactionError(f"Transaction begin failed, details {res}")
+
+    def commit(
+        self,
+        ctx: context.Context,
+    ):
+        """Commit will finalize the transaction, all b-tree management operations to the backend storage. The committed changes
+        will start to reflect on succeeding transactions b-tree store fetches/operations.
+
+        Args:
+            ctx (context.Context): _description_
+
+        Raises:
+            InvalidTransactionStateError: _description_
+            TransactionError: _description_
+        """
+        if self.transaction_id == uuid.UUID(int=0):
+            raise InvalidTransactionStateError("transaction_id is missing")
+        res = call_go.manage_transaction(ctx.id, 3, str(self.transaction_id))
+        if res != None:
+            raise TransactionError(f"Transaction commit failed, details {res}")
+
+    def rollback(
+        self,
+        ctx: context.Context,
+    ):
+        """Undo or rollback the changes done within the transaction.
+
+        Args:
+            ctx (context.Context): _description_
+
+        Raises:
+            InvalidTransactionStateError: _description_
+            TransactionError: _description_
+        """
+        if self.transaction_id == uuid.UUID(int=0):
+            raise InvalidTransactionStateError("transaction_id is missing")
+        res = call_go.manage_transaction(ctx.id, 4, str(self.transaction_id))
+        if res != None:
+            raise TransactionError(f"Transaction rollback failed, details {res}")

sop4py-2.0.0.dist-info/METADATA

@@ -0,0 +1,124 @@
+Metadata-Version: 2.4
+Name: sop4py
+Version: 2.0.0
+Summary: Scalable Objects Persistence for Python.
+Author-email: Gerardo Recinto <gerardorecinto@yahoo.com>
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: libjsondb_amd64darwin.dylib
+Requires-Dist: libjsondb_amd64darwin.h
+Requires-Dist: libjsondb_arm64darwin.dylib
+Requires-Dist: libjsondb_arm64darwin.h
+Requires-Dist: libjsondb_amd64linux.so
+Requires-Dist: libjsondb_amd64linux.h
+Requires-Dist: libjsondb_arm64linux.so
+Requires-Dist: libjsondb_arm64linux.h
+Requires-Dist: libjsondb_amd64windows.dll
+Requires-Dist: libjsondb_amd64windows.h
+
+# What is SOP?
+
+Scalable Objects Persistence (SOP) is a raw storage engine that bakes together a set of storage related features & algorithms in order to provide the most efficient & reliable (ACID attributes of transactions) technique (known) of storage management and rich search, as it brings to the application, the raw muscle of "raw storage", direct IO communications w/ disk drives. In a code library form factor.
+
+# SOP supported Hardware/OS
+SOP supports popular architectures & Operating Systems such as Linux, Darwin & Microsoft Windows, in both ARM64 & AMD64 architectures. For Windows, only AMD64 is supported since it is the only architecture Windows is available in.
+
+# SOP Dependencies
+* Redis, you will need to have one of the recent or latest version of Redis for use in SOP caching.
+* More than one Disk Drives(recommended is around four or more, for replication) with plenty of drive space available, for storage management. Example:
+```
+    /disk1
+    /disk2
+    /disk3
+    /disk4
+```
+
+# SOP for Python package
+Following steps outlines how to use the Scalable Objects Persistence code library for Python:
+* Install the package using: pip install sop-python-beta-3
+* Follow standard Python package import and start coding to use the SOP for Python code library for data management. Import the sop package in your python code file.
+* Specify Home base folders where Store info & Registry data files will be stored.
+* Specify Erasure Coding (EC) configuration details which will be used by SOP's EC based replication.
+* Create a transaction
+* Begin a transaction
+* Create a new B-tree, or Open an existing B-tree
+* Manage data, do some CRUD operations
+* Commit the transaction
+
+Below is an example code black for illustrating the above steps. For other SOP B-tree examples, you can checkout the code in the unit tests test_btree.py & test_btree_idx.py files that comes w/ the SOP package you downloaded from pypi.
+
+```
+import sop.transaction
+import sop.btree
+import sop.context
+
+stores_folders = ("/disk1", "/disk2")
+ec = {
+    # Erasure Config default entry(key="") will allow different B-tree(tables) to share same EC structure.
+    "": transaction.ErasureCodingConfig(
+        2,  # two data shards
+        2,  # two parity shards
+        (
+            # 4 disk drives paths
+            "/disk1",
+            "/disk2",
+            "/disk3",
+            "/disk4",
+        ),
+        # False means Auto repair of failed reads from (shards') disk drive will not get repaired.
+        False,
+    )
+}
+
+# Transaction Options (to).
+to = transaction.TransationOptions(
+    transaction.TransactionMode.ForWriting.value,
+    # commit timeout of 5mins
+    5,
+    # Min Registry hash mod value is 250, you can specify higher value like 1000. A 250 hashmod
+    # will use 1MB sized file segments. Good for demo, but for Prod, perhaps a bigger value is better.
+    transaction.MIN_HASH_MOD_VALUE,
+    # Store info & Registry home base folders. Array of strings of two elements, one for Active & another, for passive folder.
+    stores_folders,
+    # Erasure Coding config as shown above.
+    ec,
+)
+
+# Context object.
+ctx = context.Context()
+
+# initialize/open SOP global Redis connection
+ro = RedisOptions()
+Redis.open_connection(ro)
+
+t = transaction.Transaction(ctx, to)
+t.begin()
+
+cache = btree.CacheConfig()
+
+# "barstoreec" is new b-tree name, 2nd parameter set to True specifies B-tree Key field to be native data type
+bo = btree.BtreeOptions("barstoreec", True, cache_config=cache)
+bo.set_value_data_size(btree.ValueDataSize.Small)
+
+# create the new "barstoreec" b-tree store.
+b3 = btree.Btree.new(ctx, bo, t)
+
+# Since we've specified Native data type = True in BtreeOptions, we can use "integer" values as Key.
+l = [
+    btree.Item(1, "foo"),
+]
+
+# Add Item to the B-tree,
+b3.add(ctx, l)
+
+# Commit the transaction to finalize the new B-tree (store) change.
+t.commit(ctx)
+```
+
+# SOP in Github
+SOP open source project (MIT license) is in github. You can checkout the "...sop/jsondb/" package which contains the Go code enabling general purpose JSON data management & the Python wrapper, coding guideline of which, was described above.
+
+Please feel free to join the SOP project if you have the bandwidth and participate/co-own/lead! the project engineering.

sop4py-2.0.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+sop/__init__.py,sha256=kCWbTRTReQH3_Or-COEOdCr5H5EEdh0UqYIsEnrpxdQ,22
+sop/btree.py,sha256=mWGi7jJyi-oOY6tZQMRl7xW8kE5UQbcFpqiW3JXeCrQ,20612
+sop/call_go.py,sha256=xO2DOAd1xdsVTnpWiWdq4oKyGz4pyV8WO16LjIvRjR4,8201
+sop/context.py,sha256=jiKZayAu-NlvH47mPjMTaNoczjx-3J_kQGgKGRtEkS8,1117
+sop/libjsondb_amd64darwin.dylib,sha256=J1yd9wxA0C_OC8wufoQnVLJ4WEgXL1iWFDdpZEVo8f4,9132992
+sop/libjsondb_amd64darwin.h,sha256=V6vvqBwxSgbTmexvtwZikAlsQNdAXZq8gr9DvfVG1bo,2860
+sop/libjsondb_amd64linux.h,sha256=V6vvqBwxSgbTmexvtwZikAlsQNdAXZq8gr9DvfVG1bo,2860
+sop/libjsondb_amd64linux.so,sha256=KK8rvyi3sbffH7UK_FYJLgviGAL98g_bZjc3dzsf3lQ,20317952
+sop/libjsondb_amd64windows.dll,sha256=ON3hj591vClFz3Zl8NSc_PmBtTnvHWgeACnSH3KrRd0,19708240
+sop/libjsondb_amd64windows.h,sha256=U5CQxqyjYTG33HYjiUw1Kpf9JvhaJugRdyXUGecq9Vw,3156
+sop/libjsondb_arm64darwin.dylib,sha256=3NiXoK83NFh8hkrun1591MaIpAGV_YRDLuDf3xcdyUc,7700914
+sop/libjsondb_arm64darwin.h,sha256=V6vvqBwxSgbTmexvtwZikAlsQNdAXZq8gr9DvfVG1bo,2860
+sop/libjsondb_arm64linux.h,sha256=V6vvqBwxSgbTmexvtwZikAlsQNdAXZq8gr9DvfVG1bo,2860
+sop/libjsondb_arm64linux.so,sha256=w_T66HroMqGOsNGAX8rr3xiSQiDJmmBGmNy_Lt3BU8E,18502536
+sop/redis.py,sha256=WHc4OUpvKEM2mwVCK_RBDaf6zWUpduKT4hQ9ZDcEWao,1172
+sop/test_btree.py,sha256=4hUvZM1eXU2SrdgjZSXRtDFV9nWnTF-aDfFeJ8XZpbs,13071
+sop/test_btree_idx.py,sha256=FcKosiDEaBkBw-zhHcP3EjdB3_FOcTo7YkGKaQtBnUo,2231
+sop/transaction.py,sha256=63fzhC8LanQJ6OIeZCdg-9dVHsOOyN_qtFpXli5DJu0,5753
+sop4py-2.0.0.dist-info/METADATA,sha256=q5DvN9vMIMjn43u7Kfy0QTAdK-MXh3s3pEdGLT2qnOc,5033
+sop4py-2.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sop4py-2.0.0.dist-info/top_level.txt,sha256=Xg42qRXwoOKelr7Lc8cpmy9uuBU3LtD3tPLHVnt0aHs,4
+sop4py-2.0.0.dist-info/RECORD,,

sop4py-2.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sop4py-2.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sop