altastata 0.1.0__tar.gz

altastata-0.1.0/PKG-INFO

@@ -0,0 +1,114 @@
+Metadata-Version: 2.4
+Name: altastata
+Version: 0.1.0
+Summary: A Python package for Altastata data processing and machine learning integration
+Home-page: https://github.com/sergevil/altastata-python-package
+Author: Serge Vilvovsky
+Author-email: serge.vilvovsky@altastata.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: py4j==0.10.9.5
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Make sure you have py4j0.10.9.8.jar or similar at altastata/lib directory
+
+# for example for Windows
+cp /c/Users/serge/AppData/Local/Packages/PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0/LocalCache/local-packages/share/py4j/py4j0.10.9.8.jar altastata/lib/
+
+# Make sure you have altastata-hadoop jar (created without bouncy castle) and separate bouncy castle jars
+
+# for example
+# go to altastata-hadoop
+
+gradle clean build shadowJar -PexcludeBouncyCastle=true copyDeps
+
+# to build this one
+
+cp ../mycloud/altastata-hadoop/build/libs/altastata-hadoop-all.jar altastata/lib/
+cp ../mycloud/altastata-hadoop/build/libs_dependency/bc*-jdk18on-*.jar altastata/lib/
+
+# verify that the jar is ok (it was corrupted in Linux)
+jar tf altastata/lib/py4j0.10.9.5.jar | grep GatewayServer
+
+# if py4j file is corrupted, run
+wget https://repo1.maven.org/maven2/net/sf/py4j/py4j/0.10.9.5/py4j-0.10.9.5.jar -O altastata/lib/py4j0.10.9.5.jar
+
+# if you want to change the logs level copy and modify this file
+cp ../mycloud/altastata-hadoop/src/main/resources/logback.xml altastata/lib/
+
+# install
+pip install -e .
+
+# test
+python test_script.py
+
+# build docker
+docker buildx build --platform linux/amd64,linux/arm64 --push -t ghcr.io/sergevil/altastata/jupyter-datascience:2024a_latest -f openshift/Dockerfile .
+
+# push to the registry if needed
+docker push ghcr.io/sergevil/altastata/jupyter-datascience:2024a_latest
+
+# run docker
+docker run --name altastata-jupyter -d -p 8888:8888 -v /Users/sergevilvovsky/.altastata:/opt/app-root/src/.altastata:rw -v /Users/sergevilvovsky/Desktop:/opt/app-root/src/Desktop:rw ghcr.io/sergevil/altastata/jupyter-datascience:2024a_latest
+
+## Usage in Python Code
+
+```python
+from altastata import AltaStataFunctions
+from altastata.altastata_pytorch_dataset import register_altastata_functions
+
+# Configuration parameters
+user_properties = """#My Properties
+#Sun Jan 05 12:10:23 EST 2025
+AWSSecretKey=vcJXbtg/YGApAUpY9sjsj1xvmpz9MUPTYMxY+hDn5zZ3Fmc1BuVS34zoTRDQJ7XAvu2Z0+piCEN3TA5OArj77FlL4doYDZx7YWXUopwUhMVyBvP+gT4buHc3hkf1FvHYElbUe3yX/57fnaYP1Nwg1zN9fupzEOGtCMjy39e9Xj4vvVgXo/+YW6ogG8uXi5JA9Fm2aG7hEWQstjwu5shcMT+Q6BR2SOtkAB8B9gYlCIt7ciJ4ikkAKqtfQ8TWkOsN
+media-player=vlcj
+myuser=bob123
+accounttype=amazon-s3-secure
+AWSAccessKeyId=ZWnrkxX43me3l1YBCGX42RhdzXmhP4q4rEOcquLZJIFWCEA9+sVA+hnRYTFcJoJ5nn0luDmQJJkYaayvtAP1IG6/0h4d4sWb+1NQ/hVozOdQMezUSp+z2Wruv4WX6TQpmz12N7zqQALMDD6qi5hTiv2QLJY084ufcoMZzmK1E0uw3jTG6Pci03Zy8TFbhhbuag88Stc9thyoN44ou/d5/8Id0AruvE0EK2Q7Jg0AZZI\\=
+region=us-east-1
+kms-region=us-east-2
+metadata-encryption=RSA
+password-timeout-interval=9000000000
+acccontainer-prefix=altastata-myorgrsa444-
+logging.level.root=WARN
+logging.level.com.altastata=WARN
+logging.level.org.apache.http=ERROR
+logging.level.software.amazon=ERROR"""
+
+private_key = """-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3,F26EBECE6DDAEC52
+
+poe21ejZGZQ0GOe+EJjDdJpNvJcq/Yig9aYXY2rCGyxXLGVFeYJFg7z6gMCjIpSd
+aprW/0R8L1a2TKbs7f4K5LkSAZ98cd7N45DtIR6B4JFrDGK3LI48/XH3GT3c4OfS
+3LYldvy4XeIOAtOTTCoyhN0145ZLSoeEQ7MO3rGK0va3RGLtPWKgeZXH9j5O1Ch4
+BvPGMaKapUcgc1slj1GI4Lr+MDSrJKnUNovnVTIClS2rXTEkTri3cPLwcgWjyQIi
+BKVnobUD8Gm9irtUD6GeHrkz6Z7ELF3ctSBRSYCg+1FCvRBuljmS2C2aIiE1cu0/
+6KcqBnjEPAs250832uhAkZWj5WedIwJv+sJoGJaAUWyOfgG7DHa2HuKeR9KPD2kS
+6EygoQtQlXgSvdgZNALtIEfStmnrblTyP9Bh4JU9UzKnE6Tu5h7CjyuzkE0wgIXB
+RxgfbURfdDWs22ujLBbWPGfdY+KdNrnmSqxYahKtq6B+99+xuI0GMzX3/rLpOdF0
+AGwfa1xNe8/B/Nt+e2FXIhT2xOuH8K3sDn3/FKwy1qIsK+4g5iL6Q0xj07ujkiSI
+wZ0X2gtg3L2DW8Y6B8gBdSmDGH+vNX5/CLNn9Ly1VUoMGgs4fUmd3FFZTxiIbpim
+rQgQBHP4l1NsSqDrEyplKG83ejloLaVG+hUY1MGv5tF7B1Ta7j8bwoMTmyVCtCrC
+P+a7ShdrBUsD2TDhilZhwZcWl0a+FfzR47+faJs/9pSTkyFFp3D4xgKAdME1lvcI
+wV5BUmp5CEmbeB4r/+BlFttRZBLBXT1sq80YyQIVLumq0Livao9mOg==
+-----END RSA PRIVATE KEY-----"""
+
+# Create an instance of AltaStataFunctions
+altastata_functions = AltaStataFunctions.from_credentials(user_properties, private_key)
+altastata_functions.set_password("123")
+
+# register the altastata functions
+register_altastata_functions(altastata_functions, "bob123_rsa")
+```

altastata-0.1.0/README.md

@@ -0,0 +1,91 @@
+# Make sure you have py4j0.10.9.8.jar or similar at altastata/lib directory
+
+# for example for Windows
+cp /c/Users/serge/AppData/Local/Packages/PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0/LocalCache/local-packages/share/py4j/py4j0.10.9.8.jar altastata/lib/
+
+# Make sure you have altastata-hadoop jar (created without bouncy castle) and separate bouncy castle jars
+
+# for example
+# go to altastata-hadoop
+
+gradle clean build shadowJar -PexcludeBouncyCastle=true copyDeps
+
+# to build this one
+
+cp ../mycloud/altastata-hadoop/build/libs/altastata-hadoop-all.jar altastata/lib/
+cp ../mycloud/altastata-hadoop/build/libs_dependency/bc*-jdk18on-*.jar altastata/lib/
+
+# verify that the jar is ok (it was corrupted in Linux)
+jar tf altastata/lib/py4j0.10.9.5.jar | grep GatewayServer
+
+# if py4j file is corrupted, run
+wget https://repo1.maven.org/maven2/net/sf/py4j/py4j/0.10.9.5/py4j-0.10.9.5.jar -O altastata/lib/py4j0.10.9.5.jar
+
+# if you want to change the logs level copy and modify this file
+cp ../mycloud/altastata-hadoop/src/main/resources/logback.xml altastata/lib/
+
+# install
+pip install -e .
+
+# test
+python test_script.py
+
+# build docker
+docker buildx build --platform linux/amd64,linux/arm64 --push -t ghcr.io/sergevil/altastata/jupyter-datascience:2024a_latest -f openshift/Dockerfile .
+
+# push to the registry if needed
+docker push ghcr.io/sergevil/altastata/jupyter-datascience:2024a_latest
+
+# run docker
+docker run --name altastata-jupyter -d -p 8888:8888 -v /Users/sergevilvovsky/.altastata:/opt/app-root/src/.altastata:rw -v /Users/sergevilvovsky/Desktop:/opt/app-root/src/Desktop:rw ghcr.io/sergevil/altastata/jupyter-datascience:2024a_latest
+
+## Usage in Python Code
+
+```python
+from altastata import AltaStataFunctions
+from altastata.altastata_pytorch_dataset import register_altastata_functions
+
+# Configuration parameters
+user_properties = """#My Properties
+#Sun Jan 05 12:10:23 EST 2025
+AWSSecretKey=vcJXbtg/YGApAUpY9sjsj1xvmpz9MUPTYMxY+hDn5zZ3Fmc1BuVS34zoTRDQJ7XAvu2Z0+piCEN3TA5OArj77FlL4doYDZx7YWXUopwUhMVyBvP+gT4buHc3hkf1FvHYElbUe3yX/57fnaYP1Nwg1zN9fupzEOGtCMjy39e9Xj4vvVgXo/+YW6ogG8uXi5JA9Fm2aG7hEWQstjwu5shcMT+Q6BR2SOtkAB8B9gYlCIt7ciJ4ikkAKqtfQ8TWkOsN
+media-player=vlcj
+myuser=bob123
+accounttype=amazon-s3-secure
+AWSAccessKeyId=ZWnrkxX43me3l1YBCGX42RhdzXmhP4q4rEOcquLZJIFWCEA9+sVA+hnRYTFcJoJ5nn0luDmQJJkYaayvtAP1IG6/0h4d4sWb+1NQ/hVozOdQMezUSp+z2Wruv4WX6TQpmz12N7zqQALMDD6qi5hTiv2QLJY084ufcoMZzmK1E0uw3jTG6Pci03Zy8TFbhhbuag88Stc9thyoN44ou/d5/8Id0AruvE0EK2Q7Jg0AZZI\\=
+region=us-east-1
+kms-region=us-east-2
+metadata-encryption=RSA
+password-timeout-interval=9000000000
+acccontainer-prefix=altastata-myorgrsa444-
+logging.level.root=WARN
+logging.level.com.altastata=WARN
+logging.level.org.apache.http=ERROR
+logging.level.software.amazon=ERROR"""
+
+private_key = """-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3,F26EBECE6DDAEC52
+
+poe21ejZGZQ0GOe+EJjDdJpNvJcq/Yig9aYXY2rCGyxXLGVFeYJFg7z6gMCjIpSd
+aprW/0R8L1a2TKbs7f4K5LkSAZ98cd7N45DtIR6B4JFrDGK3LI48/XH3GT3c4OfS
+3LYldvy4XeIOAtOTTCoyhN0145ZLSoeEQ7MO3rGK0va3RGLtPWKgeZXH9j5O1Ch4
+BvPGMaKapUcgc1slj1GI4Lr+MDSrJKnUNovnVTIClS2rXTEkTri3cPLwcgWjyQIi
+BKVnobUD8Gm9irtUD6GeHrkz6Z7ELF3ctSBRSYCg+1FCvRBuljmS2C2aIiE1cu0/
+6KcqBnjEPAs250832uhAkZWj5WedIwJv+sJoGJaAUWyOfgG7DHa2HuKeR9KPD2kS
+6EygoQtQlXgSvdgZNALtIEfStmnrblTyP9Bh4JU9UzKnE6Tu5h7CjyuzkE0wgIXB
+RxgfbURfdDWs22ujLBbWPGfdY+KdNrnmSqxYahKtq6B+99+xuI0GMzX3/rLpOdF0
+AGwfa1xNe8/B/Nt+e2FXIhT2xOuH8K3sDn3/FKwy1qIsK+4g5iL6Q0xj07ujkiSI
+wZ0X2gtg3L2DW8Y6B8gBdSmDGH+vNX5/CLNn9Ly1VUoMGgs4fUmd3FFZTxiIbpim
+rQgQBHP4l1NsSqDrEyplKG83ejloLaVG+hUY1MGv5tF7B1Ta7j8bwoMTmyVCtCrC
+P+a7ShdrBUsD2TDhilZhwZcWl0a+FfzR47+faJs/9pSTkyFFp3D4xgKAdME1lvcI
+wV5BUmp5CEmbeB4r/+BlFttRZBLBXT1sq80YyQIVLumq0Livao9mOg==
+-----END RSA PRIVATE KEY-----"""
+
+# Create an instance of AltaStataFunctions
+altastata_functions = AltaStataFunctions.from_credentials(user_properties, private_key)
+altastata_functions.set_password("123")
+
+# register the altastata functions
+register_altastata_functions(altastata_functions, "bob123_rsa")
+```

altastata-0.1.0/altastata/__init__.py

@@ -0,0 +1,25 @@
+from .altastata_functions import AltaStataFunctions
+from .altastata_pytorch_dataset import AltaStataPyTorchDataset
+
+# Lazy import for TensorFlow
+def _import_tensorflow_dataset():
+    from .altastata_tensorflow_dataset import AltaStataTensorFlowDataset
+    return AltaStataTensorFlowDataset
+
+# Create a lazy loader for TensorFlow dataset
+class _LazyTensorFlowDataset:
+    def __init__(self):
+        self._dataset_class = None
+    
+    def __call__(self, *args, **kwargs):
+        if self._dataset_class is None:
+            self._dataset_class = _import_tensorflow_dataset()
+        return self._dataset_class(*args, **kwargs)
+
+AltaStataTensorFlowDataset = _LazyTensorFlowDataset()
+
+__all__ = [
+    'AltaStataFunctions',
+    'AltaStataPyTorchDataset',
+    'AltaStataTensorFlowDataset'
+]

altastata-0.1.0/altastata/altastata_functions.py

@@ -0,0 +1,199 @@
+from .base_gateway import BaseGateway
+
+from typing import List
+from py4j.java_gateway import JavaGateway, JavaObject, GatewayParameters, CallbackServerParameters, java_import
+from py4j.java_collections import JavaList
+
+import io
+import os
+import mmap
+
+class AltaStataFunctions(BaseGateway):
+    def __init__(self, port=25333):
+        """
+        Base initialization. This should not be called directly.
+        Use from_account_dir or from_credentials instead.
+        """
+        super().__init__(port)
+
+    @classmethod
+    def from_account_dir(cls, account_dir_path, port=25333):
+        """
+        Create an instance using account directory path.
+        
+        Args:
+            account_dir_path (str): Path to the account directory
+            port (int, optional): Port number for the gateway. Defaults to 25333.
+            
+        Returns:
+            AltaStataFunctions: New instance initialized with account directory
+        """
+        instance = cls(port)
+        instance.altastata_file_system = instance.gateway.jvm.com.altastata.api.AltaStataFileSystem(account_dir_path)
+        return instance
+
+    @classmethod
+    def from_credentials(cls, user_properties, private_key_encrypted, port=25333):
+        """
+        Create an instance using user properties and private key.
+        
+        Args:
+            user_properties (str): User properties string
+            private_key_encrypted (str): Encrypted private key
+            port (int, optional): Port number for the gateway. Defaults to 25333.
+            
+        Returns:
+            AltaStataFunctions: New instance initialized with credentials
+        """
+        instance = cls(port)
+        instance.altastata_file_system = instance.gateway.jvm.com.altastata.api.AltaStataFileSystem(user_properties, private_key_encrypted)
+        return instance
+
+    def convert_java_list_to_python(self, java_list):
+        # Ensure the input is a JavaList
+        if not isinstance(java_list, JavaList):
+            raise TypeError("Expected a JavaList but got something else.")
+
+        # Convert JavaList to Python list
+        python_list = [item for item in java_list]
+
+        return python_list
+
+    def python_list_to_java_arraylist(self, python_list: list) -> 'JavaObject':
+        # Create a Java ArrayList instance
+        java_arraylist = self.gateway.jvm.java.util.ArrayList()
+
+        # Add each element from the Python list to the Java ArrayList
+        for item in python_list:
+            java_arraylist.add(item)
+
+        return java_arraylist
+
+    def python_list_to_java_array(self, python_list: list) -> JavaObject:
+        string_class = self.gateway.jvm.java.lang.String
+
+        java_array = self.gateway.new_array(string_class, len(python_list))
+        for i in range(len(python_list)):
+            java_array[i] = python_list[i]
+
+        return java_array
+
+    def set_password(self, account_password: str):
+        result = self.altastata_file_system.setPassword(account_password)
+
+        # Process the result
+        return result
+
+    def create_file(self, cloud_file_path, buffer=None):
+        """
+        Create a new file version on cloud and add the buffer (may be empty).
+        This operation is fast but does not guarantee streaming order.
+        
+        Args:
+            cloud_file_path (str): The file path on the cloud
+            buffer (bytes, optional): Initial buffer to store in the file. Defaults to None (empty buffer).
+            
+        Returns:
+            CloudFileOperationStatus: Status of the file creation operation
+        """
+        if buffer is None:
+            buffer = bytes()
+            
+        return self.altastata_file_system.createFile(cloud_file_path, buffer)
+
+    def append_buffer_to_file(self, cloud_file_path, buffer, snapshot_time=None):
+        """
+        Append the buffer as an output stream to the File version.
+        
+        Args:
+            cloud_file_path (str): The file path on the cloud
+            buffer (bytes): The buffer to append
+            snapshot_time (Long, optional): File version creation time. Defaults to None (current time).
+            
+        Raises:
+            IOException: If there is an error during the append operation
+        """
+        self.altastata_file_system.appendBufferToFile(cloud_file_path, snapshot_time, buffer)
+
+    def store(self, localFilesOrDirectories: List[str], localFSPrefix: str, cloudPathPrefix: str, waitUntilDone: bool):
+        # Call the Java method
+        java_list = self.altastata_file_system.store(self.python_list_to_java_arraylist(localFilesOrDirectories), localFSPrefix, cloudPathPrefix, waitUntilDone)
+
+        # Convert the result to a Python list
+        return self.convert_java_list_to_python(java_list)
+
+    def retrieve_files(self, output_dir, cloud_path_prefix, including_subdirectories, snapshot_time, is_streaming, wait_until_done):
+        # Call the Java method
+        java_list = self.altastata_file_system.retrieve(output_dir, cloud_path_prefix, including_subdirectories, snapshot_time, is_streaming, wait_until_done)
+
+        # Convert the Java List to Python List
+        return self.convert_java_list_to_python(java_list)
+
+    def delete_files(self, cloud_path_prefix, including_subdirectories, time_interval_start, time_interval_end):
+        # Call the Java method
+        java_list = self.altastata_file_system.delete(cloud_path_prefix, including_subdirectories, time_interval_start, time_interval_end)
+
+        # Convert the Java List to Python List
+        return self.convert_java_list_to_python(java_list)
+
+    def share_files(self, cloud_path_prefix: str, including_subdirectories: bool, time_interval_start: str, time_interval_end: str, users: list) -> list:
+        # Call the Java method
+        java_list = self.altastata_file_system.share(cloud_path_prefix, including_subdirectories, time_interval_start, time_interval_end, self.python_list_to_java_array(users))
+
+        # Convert the Java List to Python List
+        return self.convert_java_list_to_python(java_list)
+
+    def list_cloud_files_versions(self, cloudPathPrefix, includingSubdirectories, timeIntervalStart, timeIntervalEnd):
+        # Call the Java method and return the iterator
+        return self.altastata_file_system.listCloudFilesVersions(cloudPathPrefix, includingSubdirectories, timeIntervalStart, timeIntervalEnd)
+
+    def get_buffer(self, cloudFilePath, snapshotTime, startPosition, howManyChunksInParallel, size):
+        """
+        Calls the Java method to get a byte array buffer and returns it as a Python bytes object.
+        """
+        java_byte_array = self.altastata_file_system.getBuffer(cloudFilePath, snapshotTime, startPosition, howManyChunksInParallel, size)
+
+        # Convert Java byte array to Python bytes
+        python_bytes = bytes(java_byte_array)
+        return python_bytes
+
+    def get_buffer_via_mapped_file(self, mappedFilePath, cloudFilePath, snapshotTime, startPosition, howManyChunksInParallel, size):
+        """
+        Calls the Java method to fill a mapped file.
+        """
+        self.altastata_file_system.fillMappedFile(mappedFilePath, cloudFilePath, snapshotTime, startPosition, howManyChunksInParallel,
+                                                  size)
+
+        # Open the file and create a memory map
+        with open(mappedFilePath, "r+b") as f:
+            mmapped_file = mmap.mmap(f.fileno(), 0)
+
+            # Access data in Python
+            contents = mmapped_file[:]
+
+            # Close the memory map
+            mmapped_file.close()
+
+            os.remove(mappedFilePath)
+
+            return contents
+
+        print("Temporary mapped file does not exist.")
+
+    def get_java_input_stream(self, cloud_file_path, snapshot_time, start_position, how_many_chunks_in_parallel):
+
+        # Call the Java method to get the InputStream
+        java_input_stream = self.altastata_file_system.getFileInputStream(cloud_file_path, snapshot_time, start_position, how_many_chunks_in_parallel)
+
+        # Return the reference to Java InputStream, that should be processed by
+        return java_input_stream
+
+    def get_buffer_from_input_stream(self, java_input_stream, buffer_size):
+
+        return self.altastata_file_system.readBufferFromInputStream(java_input_stream, buffer_size)
+
+    def get_file_attribute(self, cloud_file_path, snapshot_time, name):
+        return self.altastata_file_system.getFileAttribute(cloud_file_path, snapshot_time, name)
+
+
+