cars 1.0.0a3__cp311-cp311-win_amd64.whl → 1.0.0a4__cp311-cp311-win_amd64.whl

cars/core/cars_logging.py

@@ -74,7 +74,7 @@ class ProfilingFilter(logging.Filter):  # pylint: disable=R0903
     """
 
     def filter(self, record):
-        """ "
+        """
         Filter message
         """
         return "PROFILING_LOG" not in record.msg
@@ -85,12 +85,12 @@ class ProfilinglHandler(logging.FileHandler):  # pylint: disable=R0903
     Profiling
     """
 
-    def __init__(self, log_file):
+    def __init__(self, filename, mode="a", encoding=None, delay=False):
         """
         Init
         """
-        self.sender = LogSender(log_file)
-        logging.FileHandler.__init__(self, log_file, "a")
+        super().__init__(filename, mode, encoding, delay)
+        self.sender = LogSender(filename)
 
     def emit(self, record):
         """
@@ -100,6 +100,26 @@ class ProfilinglHandler(logging.FileHandler):  # pylint: disable=R0903
             self.sender.write_log(self.format(record) + "\n")
 
 
+class WorkerHandler(logging.FileHandler):  # pylint: disable=R0903
+    """
+    Profiling
+    """
+
+    def __init__(self, filename, mode="a", encoding=None, delay=False):
+        """
+        Init
+        """
+        super().__init__(filename, mode, encoding, delay)
+        self.sender = LogSender(filename)
+
+    def emit(self, record):
+        """
+        Emit
+        """
+        if "PROFILING" not in record.levelname:
+            self.sender.write_log(self.format(record) + "\n")
+
+
 class LogSender:  # pylint: disable=R0903
     """
     LogSender
@@ -121,12 +141,13 @@ class LogSender:  # pylint: disable=R0903
             unlock(file)
 
 
-def setup_logging(
+def setup_logging(  # pylint: disable=too-many-positional-arguments
     loglevel="PROGRESS",
     out_dir=None,
     log_dir=None,
     pipeline="",
     in_worker=False,
+    global_log_file=None,
 ):
     """
     Setup the CARS logging configuration
@@ -145,25 +166,16 @@ def setup_logging(
     if not isinstance(numeric_level, int):
         raise ValueError("Invalid log level: %s" % loglevel)
 
-    def add_handler_name(config, handler_name):
+    def add_handler_name(config, handler_name, filtered_logger=None):
         """
         add handler name in known handlers of loggers
         """
         for key in config["loggers"].keys():
-            config["loggers"][key]["handlers"].append(handler_name)
-
-    def add_handler_to_logging(
-        log_file_name, formatter, log_level, handler_name
-    ):
-        """
-        Add handler to logging
-        """
-        formatter_log = logging.Formatter(formatter)
-        h_log_file = ProfilinglHandler(log_file_name)
-        h_log_file.setFormatter(formatter_log)
-        h_log_file.setLevel(log_level)
-        h_log_file.set_name(handler_name)
-        logging.getLogger().addHandler(h_log_file)
+            if filtered_logger is not None:
+                if key in filtered_logger:
+                    config["loggers"][key]["handlers"].append(handler_name)
+            else:
+                config["loggers"][key]["handlers"].append(handler_name)
 
     logging_config = {
         "version": 1,
@@ -208,23 +220,38 @@ def setup_logging(
         },
     }
 
+    # add global log file
+    if global_log_file is not None:
+        os.makedirs(os.path.dirname(global_log_file), exist_ok=True)
+
+        handler_global_main = "file_global_main"
+        logging_config["handlers"][handler_global_main] = {
+            "class": "logging.FileHandler",
+            "filename": global_log_file,
+            "level": min(numeric_level, logging.INFO),
+            "mode": "a",
+            "formatter": "standard",
+            "filters": ["no_profiling"],
+        }
+        add_handler_name(logging_config, handler_global_main)
+
     # add file formaters:
     if out_dir is not None:
-        if not os.path.exists(out_dir):
-            os.makedirs(out_dir)
         log_file = os.path.join(
             out_dir,
             "{}_{}.log".format(
                 datetime.now().strftime("%y-%m-%d_%Hh%Mm"), pipeline
             ),
         )
+        os.makedirs(os.path.dirname(log_file), exist_ok=True)
         handler_main = "file_main"
         logging_config["handlers"][handler_main] = {
             "class": "logging.FileHandler",
             "filename": log_file,
             "level": min(numeric_level, logging.INFO),
-            "mode": "w",
+            "mode": "a",
             "formatter": "standard",
+            "filters": ["no_profiling"],
         }
         add_handler_name(logging_config, handler_main)
 
@@ -238,56 +265,60 @@ def setup_logging(
         logging_config["handlers"][handler_main_profiling] = {
             "class": "logging.FileHandler",
             "filename": profiling_file,
-            "level": min(numeric_level, PROFILING_LOG),
-            "mode": "w",
+            "level": PROFILING_LOG,
+            "mode": "a",
             "formatter": "standard",
         }
         add_handler_name(logging_config, handler_main_profiling)
 
     if not in_worker:
         add_handler_name(logging_config, "stdout")
-        logging.config.dictConfig(logging_config)
     else:
         # remove stdout as handler
         del logging_config["handlers"]["stdout"]
-        logging.config.dictConfig(logging_config)
 
         # add file handlers
-        if log_dir is not None:
-            if not os.path.exists(log_dir):
-                os.makedirs(log_dir)
 
         # change level of root logger in workerss
-        logging.getLogger().setLevel(min(numeric_level, PROFILING_LOG))
-        profiling_logger.setLevel(min(numeric_level, PROFILING_LOG))
-        # Add filter to existing logger
-        for handler in logging.root.handlers:
-            handler.addFilter(ProfilingFilter())
+        handler_workers = "file_workers"
+        handler_workers_profiling = "file_workers_profiling"
+        logging_config["loggers"]["profiling_logger"] = {
+            "handlers": [],
+            "level": PROFILING_LOG,
+            "propagate": False,
+        }
+
         # sett handlers
         log_file_workers = os.path.join(
             log_dir,
             "workers.log",
         )
-        handler_workers = "file_workers"
-        add_handler_to_logging(
-            log_file_workers,
-            logging_config["formatters"]["workers"]["format"],
-            min(numeric_level, logging.INFO),
-            handler_workers,
-        )
+        os.makedirs(os.path.dirname(log_file_workers), exist_ok=True)
+
+        logging_config["handlers"][handler_workers] = {
+            "class": "cars.core.cars_logging.WorkerHandler",
+            "filename": log_file_workers,
+            "level": min(numeric_level, logging.INFO),
+            "formatter": "workers",
+        }
+        add_handler_name(logging_config, handler_workers)
 
         # profiling
         log_file_workers_profiling = os.path.join(
             log_dir,
             "profiling.log",
         )
-        handler_workers_profiling = "file_workers_profiling"
-        add_handler_to_logging(
-            log_file_workers_profiling,
-            logging_config["formatters"]["workers"]["format"],
-            min(numeric_level, PROFILING_LOG),
-            handler_workers_profiling,
-        )
+
+        logging_config["handlers"][handler_workers_profiling] = {
+            "class": "cars.core.cars_logging.ProfilinglHandler",
+            "filename": log_file_workers_profiling,
+            "level": PROFILING_LOG,
+            "formatter": "workers",
+        }
+        add_handler_name(logging_config, handler_workers_profiling)
+
+    # Create config
+    logging.config.dictConfig(logging_config)
 
 
 def add_progress_message(message):

cars/core/constants.py

@@ -111,7 +111,6 @@ POINT_CLOUD_SOURCE_KEY_ROOT = "source_pc"
 POINT_CLOUD_COORD_EPI_GEOM_I = "coord_epi_geom_i"
 POINT_CLOUD_COORD_EPI_GEOM_J = "coord_epi_geom_j"
 POINT_CLOUD_ID_IM_EPI = "id_im_epi"
-POINT_CLOUD_GLOBAL_ID = "global_id"
 POINT_CLOUD_MATCHES = "point_cloud_matches"
 
 # raster fields (xarray Dataset)

cars/core/datasets.py

@@ -32,11 +32,12 @@ import xarray as xr
 
 # CARS imports
 from cars.core import constants as cst
+from cars.core import inputs
 
 # TODO: refacto constants: define constants here as only concerning datasets
 
 
-def create_im_dataset(
+def create_im_dataset(  # pylint: disable=too-many-positional-arguments
     img: np.ndarray,
     region: List[int],
     largest_size: List[int],
@@ -64,7 +65,9 @@ def create_im_dataset(
     if img_path is not None:
         with rio.open(img_path) as img_srs:
             img_crs = img_srs.profile["crs"]
-            img_transform = img_srs.profile["transform"]
+            img_transform = inputs.rasterio_get_transform(
+                img_path, convention="north"
+            )
 
     if img_crs is None:
         img_crs = "None"