cars 1.0.0rc2__cp312-cp312-win_amd64.whl

cars/core/cars_logging.py

@@ -0,0 +1,402 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+cCars logging module:
+contains cars logging setup logger for main thread
+and workers
+"""
+
+import logging
+import logging.config
+import os
+import platform
+
+# Standard imports
+from datetime import datetime
+from functools import wraps
+
+SYS_PLATFORM = platform.system().lower()
+IS_WIN = "windows" == SYS_PLATFORM
+
+if IS_WIN:
+    import msvcrt  # pylint: disable=E0401
+
+    def lock(file):
+        """Lock file for safe writing (Windows version)"""
+        msvcrt.locking(file.fileno(), msvcrt.LK_LOCK, 0)
+
+    def unlock(file):
+        """Unlock file for safe writing (Windows version)"""
+        msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, 0)
+
+else:
+    import fcntl
+
+    def lock(file):
+        """Lock file for safe writing (Unix version)"""
+        fcntl.flock(file, fcntl.LOCK_EX)
+
+    def unlock(file):
+        """Unlock file for safe writing (Unix version)"""
+        fcntl.flock(file, fcntl.LOCK_UN)
+
+
+PROGRESS = 21
+logging.addLevelName(PROGRESS, "PROGRESS")
+PROFILING_LOG = 15
+logging.addLevelName(PROFILING_LOG, "PROFILING_LOG")
+
+profiling_logger = logging.getLogger("profiling_logger")
+
+
+class ProfilingFilter(logging.Filter):  # pylint: disable=R0903
+    """
+    ProfilingFilter
+    """
+
+    def filter(self, record):
+        """
+        Filter message
+        """
+        return "PROFILING_LOG" not in record.msg
+
+
+class ProfilinglHandler(logging.FileHandler):  # pylint: disable=R0903
+    """
+    Profiling
+    """
+
+    def __init__(self, filename, mode="a", encoding=None, delay=False):
+        """
+        Init
+        """
+        super().__init__(filename, mode, encoding, delay)
+        self.sender = LogSender(filename)
+
+    def emit(self, record):
+        """
+        Emit
+        """
+        if "PROFILING" in record.levelname:
+            self.sender.write_log(self.format(record) + "\n")
+
+
+class WorkerHandler(logging.FileHandler):  # pylint: disable=R0903
+    """
+    Profiling
+    """
+
+    def __init__(self, filename, mode="a", encoding=None, delay=False):
+        """
+        Init
+        """
+        super().__init__(filename, mode, encoding, delay)
+        self.sender = LogSender(filename)
+
+    def emit(self, record):
+        """
+        Emit
+        """
+        if "PROFILING" not in record.levelname:
+            self.sender.write_log(self.format(record) + "\n")
+
+
+class LogSender:  # pylint: disable=R0903
+    """
+    LogSender
+    """
+
+    def __init__(self, log_file):
+        """
+        Init
+        """
+        self.log_file = log_file
+
+    def write_log(self, msg) -> None:
+        """
+        Write log
+        """
+        with open(self.log_file, "a", encoding="utf-8") as file:
+            lock(file)
+            file.write(msg)
+            unlock(file)
+
+
+def setup_logging(  # pylint: disable=too-many-positional-arguments
+    loglevel="PROGRESS",
+    out_dir=None,
+    log_dir=None,
+    pipeline="",
+    in_worker=False,
+    global_log_file=None,
+):
+    """
+    Setup the CARS logging configuration
+
+    :param loglevel: log level default WARNING
+    """
+    # logging
+    if loglevel == "PROGRESS":
+        numeric_level = PROGRESS
+    else:
+        if isinstance(loglevel, int):
+            numeric_level = loglevel
+        else:
+            numeric_level = getattr(logging, loglevel, None)
+
+    if not isinstance(numeric_level, int):
+        raise ValueError("Invalid log level: %s" % loglevel)
+
+    def add_handler_name(config, handler_name, filtered_logger=None):
+        """
+        add handler name in known handlers of loggers
+        """
+        for key in config["loggers"].keys():
+            if filtered_logger is not None:
+                if key in filtered_logger:
+                    config["loggers"][key]["handlers"].append(handler_name)
+            else:
+                config["loggers"][key]["handlers"].append(handler_name)
+
+    logging_config = {
+        "version": 1,
+        "disable_existing_loggers": True,
+        "formatters": {
+            "standard": {
+                "format": "%(asctime)s :: %(levelname)s ::  %(message)s"
+            },
+            "workers": {
+                "format": (
+                    "%(asctime)s :: %(levelname)s "
+                    + ":: %(thread)d :: %(process)d :: %(message)s"
+                )
+            },
+        },
+        "handlers": {
+            "stdout": {
+                "level": numeric_level,
+                "formatter": "standard",
+                "class": "logging.StreamHandler",
+                "stream": "ext://sys.stdout",
+                "filters": ["no_profiling"],
+            }
+        },
+        "filters": {"no_profiling": {"()": ProfilingFilter}},
+        "loggers": {
+            "": {  # root logger
+                "handlers": [],
+                "level": min(numeric_level, PROFILING_LOG),
+                "propagate": False,
+            },
+            "cars": {
+                "handlers": [],
+                "level": min(numeric_level, PROFILING_LOG),
+                "propagate": False,
+            },
+            "__main__": {  # if __name__ == '__main__'
+                "handlers": [],
+                "level": min(numeric_level, PROFILING_LOG),
+                "propagate": False,
+            },
+        },
+    }
+
+    # add global log file
+    if global_log_file is not None:
+        os.makedirs(os.path.dirname(global_log_file), exist_ok=True)
+
+        handler_global_main = "file_global_main"
+        logging_config["handlers"][handler_global_main] = {
+            "class": "logging.FileHandler",
+            "filename": global_log_file,
+            "level": min(numeric_level, logging.INFO),
+            "mode": "a",
+            "formatter": "standard",
+            "filters": ["no_profiling"],
+        }
+        add_handler_name(logging_config, handler_global_main)
+
+    # add file formaters:
+    if out_dir is not None:
+        log_file = os.path.join(
+            out_dir,
+            "{}_{}.log".format(
+                datetime.now().strftime("%y-%m-%d_%Hh%Mm"), pipeline
+            ),
+        )
+        os.makedirs(os.path.dirname(log_file), exist_ok=True)
+        handler_main = "file_main"
+        logging_config["handlers"][handler_main] = {
+            "class": "logging.FileHandler",
+            "filename": log_file,
+            "level": min(numeric_level, logging.INFO),
+            "mode": "a",
+            "formatter": "standard",
+            "filters": ["no_profiling"],
+        }
+        add_handler_name(logging_config, handler_main)
+
+        # profiling for main
+        profiling_dir = os.path.join(out_dir, "profiling")
+        if not os.path.exists(profiling_dir):
+            os.makedirs(profiling_dir)
+        profiling_file = os.path.join(profiling_dir, "profiling.log")
+
+        handler_main_profiling = "file_main_profiling"
+        logging_config["handlers"][handler_main_profiling] = {
+            "class": "logging.FileHandler",
+            "filename": profiling_file,
+            "level": PROFILING_LOG,
+            "mode": "a",
+            "formatter": "standard",
+        }
+        add_handler_name(logging_config, handler_main_profiling)
+
+    if not in_worker:
+        add_handler_name(logging_config, "stdout")
+    else:
+        # remove stdout as handler
+        del logging_config["handlers"]["stdout"]
+
+        # add file handlers
+
+        # change level of root logger in workerss
+        handler_workers = "file_workers"
+        handler_workers_profiling = "file_workers_profiling"
+        logging_config["loggers"]["profiling_logger"] = {
+            "handlers": [],
+            "level": PROFILING_LOG,
+            "propagate": False,
+        }
+
+        # sett handlers
+        log_file_workers = os.path.join(
+            log_dir,
+            "workers.log",
+        )
+        os.makedirs(os.path.dirname(log_file_workers), exist_ok=True)
+
+        logging_config["handlers"][handler_workers] = {
+            "class": "cars.core.cars_logging.WorkerHandler",
+            "filename": log_file_workers,
+            "level": min(numeric_level, logging.INFO),
+            "formatter": "workers",
+        }
+        add_handler_name(logging_config, handler_workers)
+
+        # profiling
+        log_file_workers_profiling = os.path.join(
+            log_dir,
+            "profiling.log",
+        )
+
+        logging_config["handlers"][handler_workers_profiling] = {
+            "class": "cars.core.cars_logging.ProfilinglHandler",
+            "filename": log_file_workers_profiling,
+            "level": PROFILING_LOG,
+            "formatter": "workers",
+        }
+        add_handler_name(logging_config, handler_workers_profiling)
+
+    # Create config
+    logging.config.dictConfig(logging_config)
+
+
+def add_progress_message(message):
+    """
+    Add enforced message with INFO level
+    to stdout and logging file
+
+    :param message: logging message
+    """
+    logging.log(PROGRESS, message)
+
+
+def add_profiling_message(message):
+    """
+    Add enforced message with PROFILING_LOG level
+    to stdout and logging file
+
+    :param message: logging message
+    """
+    logging.log(PROFILING_LOG, message)
+
+
+def wrap_logger(func, log_dir, log_level):
+    """
+    Wrapper logger function to wrap worker func
+    and setup the worker logger
+    :param func: wrapped function
+    :param log_dir: output directory of worker logs
+    :param log_level: logging level of the worker logs
+    """
+
+    @wraps(func)
+    def wrapper_builder(*args, **kwargs):
+        """
+        Wrapper function
+
+        :param argv: args of func
+        :param kwargs: kwargs of func
+        """
+        # init logger
+        try:
+            setup_logging(loglevel=log_level, log_dir=log_dir, in_worker=True)
+            res = func(*args, **kwargs)
+        except Exception as worker_error:
+            logging.exception(worker_error, exc_info=True)
+            raise worker_error
+        return res
+
+    return wrapper_builder
+
+
+def logger_func(*args, **kwargs):
+    """
+    Logger function to wrap worker func (with non local method)
+    and setup the worker logger
+
+    :param argv: args of func
+    :param kwargs: kwargs of func
+    """
+    # Get function to wrap and id_list
+    try:
+        log_dir = kwargs["log_dir"]
+        log_level = kwargs["log_level"]
+        func = kwargs["log_fun"]
+        kwargs.pop("log_dir")
+        kwargs.pop("log_level")
+        kwargs.pop("log_fun")
+    except Exception as exc:  # pylint: disable=W0702 # noqa: B001, E722
+        raise RuntimeError(
+            "Failed in unwrapping. \n Args: {}, \n Kwargs: {}\n".format(
+                args, kwargs
+            )
+        ) from exc
+    # init logger
+    try:
+        setup_logging(loglevel=log_level, log_dir=log_dir, in_worker=True)
+        res = func(*args, **kwargs)
+    except Exception as worker_error:
+        logging.exception(worker_error, exc_info=True)
+        raise worker_error
+    return res

cars/core/constants.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+CARS Constants module
+"""
+
+# general
+ROW = "row"  # cannot be changed because of PANDORA input format
+COL = "col"  # cannot be changed because of PANDORA input format
+BAND = "band"
+BAND_IM = "band_im"
+BAND_CLASSIF = "band_classif"
+BAND_PERFORMANCE_MAP = "band_performance_map"
+BAND_LAYER_INF = "band_inf_layer"
+BAND_LAYER_SUP = "band_sup_layer"
+BAND_FILLING = "filling_type"
+BAND_SOURCE_PC = "source_point_cloud"
+RIO_TAG_PERFORMANCE_MAP_CLASSES = "rio_tag_performance_map_classes"
+X = "x"
+Y = "y"
+Z = "z"
+TEXTURE = "texture"
+PERFORMANCE_MAP_FROM_RISK = "risk_performance_map"
+PERFORMANCE_MAP_FROM_INTERVALS = "intervals_performance_map"
+RESOLUTION = "resolution"
+EPI_FULL_SIZE = "full_epipolar_size"
+ROI = "roi"
+ROI_WITH_MARGINS = "roi_with_margins"
+EPSG = "epsg"
+DISPARITY = "disparity"
+PC_EPSG = "point_cloud_epsg"
+
+BAND_NAMES = "band_names"
+IMAGE_TYPE = "image_type"
+NBITS = "nbits"
+EPSG_WSG84 = 4326
+# stereo keys
+STEREO_REF = "ref"
+STEREO_SEC = "sec"
+
+CROPPED_DISPARITY_RANGE = "cropped_disparity_range"
+
+# epipolar image dataset
+EPI_IMAGE = "im"  # has to be synchronized with the PANDORA input format
+EPI_MSK = "msk"  # has to be synchronized with the PANDORA input format
+EPI_TEXTURE = "texture"
+EPI_CLASSIFICATION = "classif"
+EPI_FILLING = "filling"
+EPI_CONFIDENCE_KEY_ROOT = "confidence"
+EPI_PERFORMANCE_MAP = "performance_map"
+# used in re entrance
+EPI_Z_SUP = "z_sup"
+EPI_Z_INF = "z_inf"
+EPI_AMBIGUITY = "ambiguity"
+EPI_DENOISING_INFO_KEY_ROOT = "denoising"
+EPI_MARGINS = "margins"
+EPI_DISP_MIN = "disp_min"
+EPI_DISP_MAX = "disp_max"
+EPI_VALID_PIXELS = "valid_pixels"
+EPI_NO_DATA_MASK = "no_data_mask"
+EPI_NO_DATA_IMG = "no_data_img"
+EPI_TRANSFORM = "transform"
+EPI_CRS = "crs"
+EPI_GROUND_TRUTH = "epi_ground_truth"
+SENSOR_GROUND_TRUTH = "sensor_ground_truth"
+
+# points cloud fields (xarray Dataset and pandas Dataframe)
+POINT_CLOUD_CORR_MSK = "corr_msk"
+POINT_CLOUD_MSK = "mask"
+POINT_CLOUD_CLR_KEY_ROOT = "texture"
+# base name used to parse
+POINT_CLOUD_PERFORMANCE_MAP_ROOT = "performance_map"
+POINT_CLOUD_PERFORMANCE_MAP_FROM_RISK = (
+    POINT_CLOUD_PERFORMANCE_MAP_ROOT + "_from_risk"
+)
+POINT_CLOUD_PERFORMANCE_MAP_FROM_INTERVALS = (
+    POINT_CLOUD_PERFORMANCE_MAP_ROOT + "_from_intervals"
+)
+POINT_CLOUD_CONFIDENCE_KEY_ROOT = "confidence"
+POINT_CLOUD_AMBIGUITY_KEY_ROOT = "ambiguity"
+POINT_CLOUD_INTERVALS_KEY_ROOT = "intervals"
+# base name used to parse
+POINT_CLOUD_LAYER_SUP_OR_INF_ROOT = "layer_border_z"
+POINT_CLOUD_LAYER_SUP = POINT_CLOUD_LAYER_SUP_OR_INF_ROOT + "_sup"
+POINT_CLOUD_LAYER_INF = POINT_CLOUD_LAYER_SUP_OR_INF_ROOT + "_inf"
+POINT_CLOUD_LAYER_SUP_FROM_RISK = POINT_CLOUD_LAYER_SUP + "_from_risk"
+POINT_CLOUD_LAYER_INF_FROM_RISK = POINT_CLOUD_LAYER_INF + "_from_risk"
+POINT_CLOUD_LAYER_SUP_FROM_INTERVALS = POINT_CLOUD_LAYER_SUP + "_from_intervals"
+POINT_CLOUD_LAYER_INF_FROM_INTERVALS = POINT_CLOUD_LAYER_INF + "_from_intervals"
+POINT_CLOUD_CLASSIF_KEY_ROOT = "classif"
+POINT_CLOUD_FILLING_KEY_ROOT = "filling"
+POINT_CLOUD_SOURCE_KEY_ROOT = "source_pc"
+POINT_CLOUD_COORD_EPI_GEOM_I = "coord_epi_geom_i"
+POINT_CLOUD_COORD_EPI_GEOM_J = "coord_epi_geom_j"
+POINT_CLOUD_ID_IM_EPI = "id_im_epi"
+POINT_CLOUD_MATCHES = "point_cloud_matches"
+
+# raster fields (xarray Dataset)
+RASTER_HGT = "hgt"
+RASTER_HGT_INF = "hgt_inf"
+RASTER_HGT_SUP = "hgt_sup"
+RASTER_WEIGHTS_SUM = "weights_sum"
+RASTER_COLOR_IMG = "img"
+RASTER_MSK = "raster_msk"
+RASTER_CLASSIF = "raster_classif"
+RASTER_NB_PTS = "n_pts"
+RASTER_NB_PTS_IN_CELL = "pts_in_cell"
+RASTER_HGT_MEAN = "hgt_mean"
+RASTER_HGT_STD_DEV = "hgt_stdev"
+RASTER_HGT_INF_MEAN = "hgt_inf_mean"
+RASTER_HGT_INF_STD_DEV = "hgt_inf_stdev"
+RASTER_HGT_SUP_MEAN = "hgt_sup_mean"
+RASTER_HGT_SUP_STD_DEV = "hgt_sup_stdev"
+RASTER_BAND_MEAN = "band_mean"
+RASTER_BAND_STD_DEV = "band_stdev"
+RASTER_CONFIDENCE = "confidence"
+RASTER_PERFORMANCE_MAP = "performance_map"
+RASTER_AMBIGUITY = "ambiguity"
+RASTER_PERFORMANCE_MAP_RAW = "performance_map_raw"
+RASTER_SOURCE_PC = "source_pc"
+RASTER_FILLING = "filling"
+# Geometry constants
+DISP_MODE = "disp"
+MATCHES_MODE = "matches"
+
+# DSM index
+INDEX_DSM_ALT = "dsm"
+INDEX_DSM_WEIGHTS = "weights"
+INDEX_DSM_COLOR = "image"
+INDEX_DSM_MASK = "mask"
+INDEX_DSM_CLASSIFICATION = "classification"
+INDEX_DSM_PERFORMANCE_MAP = "performance_map"
+INDEX_DSM_AMBIGUITY = "ambiguity"
+INDEX_DSM_CONTRIBUTING_PAIR = "contributing_pair"
+INDEX_DSM_FILLING = "filling"
+INDEX_DEM_MIN = "dem_min"
+INDEX_DEM_MAX = "dem_max"
+INDEX_DEM_MEDIAN = "dem_median"
+
+
+# depth map index
+INDEX_DEPTH_MAP_X = "x"
+INDEX_DEPTH_MAP_Y = "y"
+INDEX_DEPTH_MAP_Z = "z"
+INDEX_DEPTH_MAP_COLOR = "image"
+INDEX_DEPTH_MAP_MASK = "mask"
+INDEX_DEPTH_MAP_CLASSIFICATION = "classification"
+INDEX_DEPTH_MAP_PERFORMANCE_MAP = "performance_map"
+INDEX_DEPTH_MAP_AMBIGUITY = "ambiguity"
+INDEX_DEPTH_MAP_FILLING = "filling"
+INDEX_DEPTH_MAP_EPSG = "epsg"
+
+# dsms inputs index
+DSM_CLASSIF = "merging_classification"
+DSM_ALT = "dsm"
+DSM_ALT_INF = "dsm_inf"
+DSM_ALT_SUP = "dsm_sup"
+DSM_WEIGHTS_SUM = "weights"
+DSM_MSK = "mask"
+DSM_NB_PTS = "dsm_n_pts"
+DSM_NB_PTS_IN_CELL = "dsm_pts_in_cell"
+DSM_MEAN = "dsm_mean"
+DSM_STD_DEV = "dsm_std"
+DSM_INF_MEAN = "dsm_inf_mean"
+DSM_INF_STD = "dsm_inf_std"
+DSM_SUP_MEAN = "dsm_sup_mean"
+DSM_SUP_STD = "dsm_sup_std"
+DSM_AMBIGUITY = "ambiguity"
+DSM_CONFIDENCE = "confidence"
+DSM_PERFORMANCE_MAP = "performance_map"
+DSM_SOURCE_PC = "contributing_pair"
+DSM_FILLING = "merging_filling"
+DSM_COLOR = "image"

cars/core/constants_disparity.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+CARS Disparity Constants module
+"""
+
+# disparity map
+MAP = "disp"
+CONFIDENCE = "confidence"
+CONFIDENCE_KEY = "cost_volume_confidence"
+INTERVAL = "interval_bounds"
+INTERVAL_INF = "confidence_from_interval_bounds_inf"
+INTERVAL_SUP = "confidence_from_interval_bounds_sup"
+EPI_DISP_MIN_GRID = "disp_min_grid"
+EPI_DISP_MAX_GRID = "disp_max_grid"
+
+# disparity mask
+VALID = "disp_msk"
+FILLING = "filling"
+INVALID_REF = "msk_invalid_ref"
+INVALID_SEC = "msk_invalid_sec"
+MASKED_REF = "msk_masked_ref"
+MASKED_SEC = "msk_masked_sec"
+OCCLUSION = "msk_occlusion"
+FALSE_MATCH = "msk_false_match"
+INCOMPLETE_DISP = "msk_incomplete_disp"
+STOPPED_INTERP = "msk_stopped_interp"
+FILLED_OCCLUSION = "msk_filled_occlusion"
+FILLED_FALSE_MATCH = "msk_filled_false_match"
+INSIDE_SEC_ROI = "msk_inside_sec_roi"
+DISP_TO_0 = "msk_disp_to_0"
+INCOMPLETE_VARIABLE_DISP = "msk_incomplete_variable_disp"