decentriq_util 0.1.0.dev10__tar.gz → 0.1.0.dev12__tar.gz

{decentriq_util-0.1.0.dev10 → decentriq_util-0.1.0.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: decentriq_util
-Version: 0.1.0.dev10
+Version: 0.1.0.dev12
 Summary: Utility functions for Python code run in Decentriq workers
 Author: hafenr
 Author-email: robin.hafen@gmail.com

decentriq_util-0.1.0.dev12/decentriq_util/logger.py

@@ -0,0 +1,9 @@
+import logging
+
+
+logger = logging.getLogger("decentriq_util")
+ch = logging.StreamHandler()
+ch.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+logger.addHandler(ch)

{decentriq_util-0.1.0.dev10 → decentriq_util-0.1.0.dev12}/decentriq_util/spark/__init__.py

@@ -4,8 +4,8 @@ from contextlib import contextmanager
 import multiprocessing
 import math
 import os
-import shutil
 import psutil
+from ..logger import logger
 
 import pyarrow.parquet as pq
 from pyspark.sql import SparkSession
@@ -36,7 +36,7 @@ def _get_cgroup_memory() -> int:
     if max_memory is not None:
         return max_memory
     else:
-        print("Unable to determine available memory from cgroup, assuming 4G")
+        logger.info("Unable to determine available memory from cgroup, assuming 4G")
         return 4 * 1024 * 1024 * 1024
 
 
@@ -101,7 +101,7 @@ def _get_num_partitions(
 
     # Try to make partitions as big as possible s.t. each CPU is saturated,
     # while accounting for 50% overhead in spark.
-    task_per_cpu = 3  # try to get at least this many tasks/partitions per CPU
+    task_per_cpu = 2  # try to get at least this many tasks/partitions per CPU
     max_target_partition_size = max(
         int(0.5 * available_memory / (task_per_cpu * cpu_count)) // megabyte_in_bytes * megabyte_in_bytes,
         min_partition_size
@@ -118,9 +118,9 @@ def _get_num_partitions(
             break
         target_partition_size = target_partition_size / 2
 
-    print(f"File size: {file_size}")
-    print(f"Max target partition size: {max_target_partition_size}")
-    print(f"Target partition size: {target_partition_size}")
+    logger.info(f"File size: {file_size}")
+    logger.info(f"Max target partition size: {max_target_partition_size}")
+    logger.info(f"Target partition size: {target_partition_size}")
 
     num_partitions = int(math.ceil(file_size / target_partition_size))
 
@@ -192,7 +192,7 @@ def _create_spark_session(
     for (key, value) in config:
         ss = ss.config(key, value)
 
-    print("Spark settings:\n" + f"\n".join([str(x) for x in config]))
+    logger.info("Spark settings:\n" + f"\n".join([str(x) for x in config]))
 
     return ss.getOrCreate()
 

{decentriq_util-0.1.0.dev10 → decentriq_util-0.1.0.dev12}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "decentriq_util"
-version = "0.1.0-dev10"
+version = "0.1.0-dev12"
 description = "Utility functions for Python code run in Decentriq workers"
 authors = ["hafenr <robin.hafen@gmail.com>"]