cidp 0.0.1__tar.gz

cidp-0.0.1/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.1
+Name: cidp
+Version: 0.0.1
+Summary: CIDP Python SDK
+Home-page: https://gitlab.cidp.io/common/pypi/cidp
+Author: Example Author
+Author-email: skt.mccho@sk.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6

cidp-0.0.1/cidp/KubernetesController.py

@@ -0,0 +1,68 @@
+class KubernetesController():
+    def __init__(self, config_path):
+        def load_kubeconfig(config_path):
+            from kubernetes import config, client
+            config.load_kube_config(config_file=config_path)
+            return client.CoreV1Api()
+
+        self.api = load_kubeconfig(config_path)
+
+    def get_my_pod_spec(self):
+        import os
+        pods = self.api.list_pod_for_all_namespaces(watch=False)
+        for pod in pods.items:
+            if pod.metadata.labels is not None and pod.metadata.labels.get("statefulset.kubernetes.io/pod-name",
+                                                                           "none") == os.environ["HOSTNAME"]:
+                return pod
+
+    def get_pod_namespace(self, pod_spec):
+        return pod_spec.metadata.namespace
+
+    def get_my_pod_namespace(self):
+        return self.get_pod_namespace(self.get_my_pod_spec())
+
+    def get_pod_selector(self, pod_spec, key):
+        return pod_spec.metadata.labels[key]
+
+    def get_my_pod_app_selector(self):
+        return self.get_pod_selector(self.get_my_pod_spec(), "app")
+
+    def get_list_used_nodeport(self):
+        used_ports = []
+        services = self.api.list_service_for_all_namespaces(watch=False)
+        for service in services.items:
+            if service.spec.type == "NodePort":
+                for port in service.spec.ports:
+                    used_ports.append(port.node_port)
+        return used_ports
+
+    def get_available_ports(self, start=32000, end=32600, count=1):
+        import random
+        used_ports = self.get_list_used_nodeport()
+        port_range = range(start, end + 1)
+        ports = [port for port in port_range if port not in used_ports]
+        return random.sample(ports, count)
+
+    def create_spark_nodeport(self, namespace, service_name, app_name, driver_port, blockmanager_port):
+        from kubernetes import client
+        service = client.V1Service(api_version="v1",
+                                   kind="Service",
+                                   metadata=client.V1ObjectMeta(name=service_name),
+                                   spec=client.V1ServiceSpec(selector={"app": app_name},
+                                                             type="NodePort",
+                                                             ports=[client.V1ServicePort(name="driver-port",
+                                                                                         protocol="TCP",
+                                                                                         port=driver_port,
+                                                                                         target_port=driver_port,
+                                                                                         node_port=driver_port),
+                                                                    client.V1ServicePort(name="blockmanager-port",
+                                                                                         protocol="TCP",
+                                                                                         port=blockmanager_port,
+                                                                                         target_port=blockmanager_port,
+                                                                                         node_port=blockmanager_port)]
+                                                             )
+                                   )
+        self.api.create_namespaced_service(namespace=namespace, body=service)
+
+    def remove_spark_nodeport(self, namespace, service_name):
+        self.api.delete_namespaced_service(service_name, namespace)

cidp-0.0.1/cidp/SparkSessionBuilder.py

@@ -0,0 +1,54 @@
+from typing import (Any, Dict)
+from pyspark.sql import SparkSession
+
+
+class SparkSessionBuilder:
+    def __init__(self, k8s_config="~/.kube/ciap_prd.conf") -> None:
+        from datetime import datetime
+        from pyspark import SparkConf
+        import uuid
+        self.spark_session = None
+        self._k8s_api = KubernetesController(k8s_config)
+        self._timestamp = datetime.now().strftime("%Y%m%d%H%M")
+        self._suffix = uuid.uuid4().hex[:8]
+        self._app_id = f"{self._timestamp}-{self._suffix}"
+        self._service_name = f"spark-{self._app_id}"
+        self._options = SparkConf()
+        self._my_namespace = self._k8s_api.get_my_pod_namespace()
+        self._my_appname = self._k8s_api.get_my_pod_app_selector()
+        self.appName(f"{self._my_appname}-{self._app_id}")
+
+    def config(self, k: str, v: Any) -> "SparkSessionBuilder":
+        self._options.set(k, v)
+        return self
+
+    def master(self, master: str) -> "SparkSessionBuilder":
+        return self.config("spark.master", master)
+
+    def appName(self, name: str) -> "SparkSessionBuilder":
+        return self.config("spark.app.name", name)
+
+    def getOrCreate(self) -> SparkSession:
+        def get_spark_nodeports():
+            return self._k8s_api.get_available_ports(count=2)
+
+        driver_port, blockmanager_port = get_spark_nodeports()
+        self._k8s_api.create_spark_nodeport(self._my_namespace,
+                                            self._service_name,
+                                            self._my_appname,
+                                            driver_port,
+                                            blockmanager_port)
+        (self.config("spark.driver.host", f"{self._my_appname}-{self._app_id}.spark-endpoint.cidp.io")
+         .config("spark.driver.port", driver_port)
+         .config("spark.blockManager.port", blockmanager_port))
+
+        from pyspark.sql import SparkSession
+        if self.spark_session is None:
+            self.spark_session = SparkSession.builder.config(conf=self._options).getOrCreate()
+        return self.spark_session
+
+    # def __del__(self):
+    #     if self.spark_session is not None:
+    #         self.spark_session.stop()
+    #     self._k8s_api.remove_spark_nodeport(self._my_namespace, self._service_name)
+    #     print("할당 해제")

cidp-0.0.1/cidp.egg-info/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.1
+Name: cidp
+Version: 0.0.1
+Summary: CIDP Python SDK
+Home-page: https://gitlab.cidp.io/common/pypi/cidp
+Author: Example Author
+Author-email: skt.mccho@sk.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6

cidp-0.0.1/cidp.egg-info/SOURCES.txt

@@ -0,0 +1,8 @@
+setup.py
+cidp/KubernetesController.py
+cidp/SparkSessionBuilder.py
+cidp/__init__.py
+cidp.egg-info/PKG-INFO
+cidp.egg-info/SOURCES.txt
+cidp.egg-info/dependency_links.txt
+cidp.egg-info/top_level.txt

cidp-0.0.1/cidp.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

cidp-0.0.1/cidp.egg-info/top_level.txt

@@ -0,0 +1 @@
+cidp

cidp-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

cidp-0.0.1/setup.py

@@ -0,0 +1,22 @@
+import setuptools
+
+# with open("README.md", "r") as fh:
+#     long_description = fh.read()
+
+setuptools.setup(
+    name="cidp", # Replace with your own username
+    version="0.0.1",
+    author="Example Author",
+    author_email="skt.mccho@sk.com",
+    description="CIDP Python SDK",
+    # long_description=long_description,
+    # long_description_content_type="text/markdown",
+    url="https://gitlab.cidp.io/common/pypi/cidp",
+    packages=setuptools.find_packages(),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires='>=3.6',
+)