decentriq_util 0.1.0.dev9__tar.gz → 0.1.0.dev11__tar.gz

{decentriq_util-0.1.0.dev9 → decentriq_util-0.1.0.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: decentriq_util
-Version: 0.1.0.dev9
+Version: 0.1.0.dev11
 Summary: Utility functions for Python code run in Decentriq workers
 Author: hafenr
 Author-email: robin.hafen@gmail.com

decentriq_util-0.1.0.dev11/decentriq_util/logger.py

@@ -0,0 +1,9 @@
+import logging
+
+
+logger = logging.getLogger("decentriq_util")
+ch = logging.StreamHandler()
+ch.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+logger.addHandler(ch)

{decentriq_util-0.1.0.dev9 → decentriq_util-0.1.0.dev11}/decentriq_util/spark/__init__.py

@@ -4,8 +4,8 @@ from contextlib import contextmanager
 import multiprocessing
 import math
 import os
-import shutil
 import psutil
+from ..logger import logger
 
 import pyarrow.parquet as pq
 from pyspark.sql import SparkSession
@@ -36,7 +36,7 @@ def _get_cgroup_memory() -> int:
     if max_memory is not None:
         return max_memory
     else:
-        print("Unable to determine available memory from cgroup, assuming 4G")
+        logger.info("Unable to determine available memory from cgroup, assuming 4G")
         return 4 * 1024 * 1024 * 1024
 
 
@@ -118,9 +118,9 @@ def _get_num_partitions(
             break
         target_partition_size = target_partition_size / 2
 
-    print(f"File size: {file_size}")
-    print(f"Max target partition size: {max_target_partition_size}")
-    print(f"Target partition size: {target_partition_size}")
+    logger.info(f"File size: {file_size}")
+    logger.info(f"Max target partition size: {max_target_partition_size}")
+    logger.info(f"Target partition size: {target_partition_size}")
 
     num_partitions = int(math.ceil(file_size / target_partition_size))
 
@@ -148,7 +148,6 @@ def _determine_optimal_spark_settings(
         ("spark.driver.cores", str(cpu_count)),
         ("spark.driver.memory", str(spark_memory_4096)),
     ]
-    print("Spark settings:\n" + f"\n".join([str(x) for x in settings]))
 
     return settings
 
@@ -192,6 +191,9 @@ def _create_spark_session(
     )
     for (key, value) in config:
         ss = ss.config(key, value)
+
+    logger.info("Spark settings:\n" + f"\n".join([str(x) for x in config]))
+
     return ss.getOrCreate()
 
 

{decentriq_util-0.1.0.dev9 → decentriq_util-0.1.0.dev11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "decentriq_util"
-version = "0.1.0-dev9"
+version = "0.1.0-dev11"
 description = "Utility functions for Python code run in Decentriq workers"
 authors = ["hafenr <robin.hafen@gmail.com>"]