py4dgeo 0.4.0__cp39-cp39-win_amd64.whl → 0.6.0__cp39-cp39-win_amd64.whl

py4dgeo/__init__.py

@@ -1,7 +1,15 @@
 from py4dgeo.logger import set_py4dgeo_logfile
 from py4dgeo.cloudcompare import CloudCompareM3C2
-from py4dgeo.epoch import Epoch, read_from_las, read_from_xyz, save_epoch, load_epoch
-from py4dgeo.m3c2 import M3C2
+from py4dgeo.epoch import (
+    Epoch,
+    read_from_las,
+    read_from_xyz,
+    save_epoch,
+    load_epoch,
+)
+from py4dgeo.m3c2 import M3C2, write_m3c2_results_to_las
+from py4dgeo.m3c2ep import M3C2EP
+from py4dgeo.registration import iterative_closest_point
 from py4dgeo.segmentation import (
     RegionGrowingAlgorithm,
     SpatiotemporalAnalysis,
@@ -9,9 +17,12 @@ from py4dgeo.segmentation import (
     temporal_averaging,
 )
 from py4dgeo.util import (
-    ensure_test_data_availability,
+    __version__,
+    find_file,
     MemoryPolicy,
     set_memory_policy,
     get_num_threads,
     set_num_threads,
 )
+
+from py4dgeo.pbm3c2 import *

py4dgeo/epoch.py

@@ -1,3 +1,5 @@
+from py4dgeo.logger import logger_context
+from py4dgeo.registration import Transformation
 from py4dgeo.util import (
     Py4DGeoError,
     append_file_extension,
@@ -6,6 +8,7 @@ from py4dgeo.util import (
     make_contiguous,
     is_iterable,
 )
+from numpy.lib.recfunctions import append_fields
 
 import dateparser
 import datetime
@@ -15,28 +18,56 @@ import logging
 import numpy as np
 import os
 import tempfile
+import typing
 import zipfile
 
-import py4dgeo._py4dgeo as _py4dgeo
-
+import _py4dgeo
 
 logger = logging.getLogger("py4dgeo")
 
-
 # This integer controls the versioning of the epoch file format. Whenever the
 # format is changed, this version should be increased, so that py4dgeo can warn
 # about incompatibilities of py4dgeo with loaded data. This version is intentionally
 # different from py4dgeo's version, because not all releases of py4dgeo necessarily
 # change the epoch file format and we want to be as compatible as possible.
-PY4DGEO_EPOCH_FILE_FORMAT_VERSION = 2
+PY4DGEO_EPOCH_FILE_FORMAT_VERSION = 4
+
+
+class NumpyArrayEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        return json.JSONEncoder.default(self, obj)
 
 
 class Epoch(_py4dgeo.Epoch):
-    def __init__(self, cloud: np.ndarray, timestamp=None):
+    def __init__(
+        self,
+        cloud: np.ndarray,
+        normals: np.ndarray = None,
+        additional_dimensions: np.ndarray = None,
+        timestamp=None,
+        scanpos_info: dict = None,
+    ):
         """
 
         :param cloud:
             The point cloud array of shape (n, 3).
+
+        :param normals:
+            The point cloud normals of shape (n, 3) where n is the
+            same as the number of points in the point cloud.
+
+        :param additional_dimensions:
+            A numpy array of additional, per-point data in the point cloud. The
+            numpy data type is expected to be a structured dtype, so that the data
+            columns are accessible by their name.
+
+        :param timestamp:
+            The point cloud timestamp, default is None.
+
+        :param scanpos_info:
+            The point scan positions information, default is None..
         """
         # Check the given array shapes
         if len(cloud.shape) != 2 or cloud.shape[1] != 3:
@@ -46,12 +77,66 @@ class Epoch(_py4dgeo.Epoch):
         cloud = as_double_precision(cloud)
         cloud = make_contiguous(cloud)
 
+        # Set identity transformation
+        self._transformations = []
+
+        # Make sure that given normals are DP and contiguous as well
+        if normals is not None:
+            normals = make_contiguous(as_double_precision(normals))
+        self._normals = normals
+
         # Set metadata properties
         self.timestamp = timestamp
+        self.scanpos_info = scanpos_info
+
+        # Set the additional information (e.g. segment ids, normals, etc)
+        self.additional_dimensions = additional_dimensions
 
         # Call base class constructor
         super().__init__(cloud)
 
+    @property
+    def normals(self):
+        # Maybe calculate normals
+        if self._normals is None:
+            raise Py4DGeoError(
+                "Normals for this Epoch have not been calculated! Please use Epoch.calculate_normals or load externally calculated normals."
+            )
+
+        return self._normals
+
+    def calculate_normals(
+        self, radius=1.0, orientation_vector: np.ndarray = np.array([0, 0, 1])
+    ):
+        """Calculate point cloud normals
+
+        :param radius:
+            The radius used to determine the neighborhood of a point.
+
+        :param orientation_vector:
+            A vector to determine orientation of the normals. It should point "up".
+        """
+
+        # Ensure that the KDTree is built
+        if self.kdtree.leaf_parameter() == 0:
+            self.build_kdtree()
+
+        # Allocate memory for the normals
+        self._normals = np.empty(self.cloud.shape, dtype=np.float64)
+
+        # Reuse the multiscale code with a single radius in order to
+        # avoid code duplication.
+        with logger_context("Calculating point cloud normals:"):
+            _py4dgeo.compute_multiscale_directions(
+                self,
+                self.cloud,
+                [radius],
+                orientation_vector,
+                self._normals,
+            )
+
+        return self.normals
+
     @property
     def timestamp(self):
         return self._timestamp
@@ -60,6 +145,46 @@ class Epoch(_py4dgeo.Epoch):
     def timestamp(self, timestamp):
         self._timestamp = normalize_timestamp(timestamp)
 
+    @property
+    def scanpos_info(self):
+        return self._scanpos_info
+
+    @scanpos_info.setter
+    def scanpos_info(self, scanpos_info):
+        if isinstance(scanpos_info, list):
+            self._scanpos_info = scanpos_info
+        elif isinstance(scanpos_info, dict):
+            self._scanpos_info = scan_positions_info_from_dict(scanpos_info)
+        else:
+            self._scanpos_info = None
+
+    @property
+    def scanpos_id(self):
+        return (
+            self.additional_dimensions["scanpos_id"]
+            .reshape(self.cloud.shape[0])
+            .astype(np.int32)
+        )
+
+    @scanpos_id.setter
+    def scanpos_id(self, scanpos_id):
+        if self.additional_dimensions is None:
+            additional_columns = np.empty(
+                shape=(self.cloud.shape[0], 1),
+                dtype=np.dtype([("scanpos_id", "<i4")]),
+            )
+            additional_columns["scanpos_id"] = np.array(
+                scanpos_id, dtype=np.int32
+            ).reshape(-1, 1)
+            self.additional_dimensions = additional_columns
+        else:
+            scanpos_id = np.array(scanpos_id, dtype=np.int32)
+            new_additional_dimensions = append_fields(
+                self.additional_dimensions, "scanpos_id", scanpos_id, usemask=False
+            )
+
+            self.additional_dimensions = new_additional_dimensions
+
     @property
     def metadata(self):
         """Provide the metadata of this epoch as a Python dictionary
@@ -72,6 +197,7 @@ class Epoch(_py4dgeo.Epoch):
 
         return {
             "timestamp": None if self.timestamp is None else str(self.timestamp),
+            "scanpos_info": None if self.scanpos_info is None else self.scanpos_info,
         }
 
     def build_kdtree(self, leaf_size=10, force_rebuild=False):
@@ -91,6 +217,89 @@ class Epoch(_py4dgeo.Epoch):
             logger.info(f"Building KDTree structure with leaf parameter {leaf_size}")
             self.kdtree.build_tree(leaf_size)
 
+    def transform(
+        self,
+        transformation: typing.Optional[Transformation] = None,
+        affine_transformation: typing.Optional[np.ndarray] = None,
+        rotation: typing.Optional[np.ndarray] = None,
+        translation: typing.Optional[np.ndarray] = None,
+        reduction_point: typing.Optional[np.ndarray] = None,
+    ):
+        """Transform the epoch with an affine transformation
+
+        :param transformation:
+            A Transformation object that describes the transformation to apply.
+            If this argument is given, the other arguments are ignored.
+            This parameter is typically used if the transformation was calculated
+            by py4dgeo itself.
+        :type transformation: Transformation
+        :param affine_transformation:
+            A 4x4 or 3x4 matrix representing the affine transformation. Given
+            as a numpy array. If this argument is given, the rotation and
+            translation arguments are ignored.
+        :type transformation: np.ndarray
+        :param rotation:
+            A 3x3 matrix specifying the rotation to apply
+        :type rotation: np.ndarray
+        :param translation:
+            A vector specifying the translation to apply
+        :type translation: np.ndarray
+        :param reduction_point:
+            A translation vector to apply before applying rotation and scaling.
+            This is used to increase the numerical accuracy of transformation.
+            If a transformation is given, this argument is ignored.
+        :type reduction_point: np.ndarray
+        """
+
+        # Extract the affine transformation and reduction point from the given transformation
+        if transformation is not None:
+            assert isinstance(transformation, Transformation)
+            affine_transformation = transformation.affine_transformation
+            reduction_point = transformation.reduction_point
+
+        # Build the transformation if it is not explicitly given
+        if affine_transformation is None:
+            trafo = np.identity(4, dtype=np.float64)
+            trafo[:3, :3] = rotation
+            trafo[:3, 3] = translation
+        else:
+            # If it was given, make a copy and potentially resize it
+            trafo = affine_transformation.copy()
+            if trafo.shape[0] == 3:
+                trafo.resize((4, 4), refcheck=False)
+                trafo[3, 3] = 1
+
+        if reduction_point is None:
+            reduction_point = np.array([0, 0, 0], dtype=np.float64)
+
+        # Ensure contiguous DP memory
+        trafo = as_double_precision(make_contiguous(trafo))
+
+        # Invalidate the KDTree
+        self.kdtree.invalidate()
+
+        # Apply the actual transformation as efficient C++
+        _py4dgeo.transform_pointcloud_inplace(self.cloud, trafo, reduction_point)
+
+        # Store the transformation
+        self._transformations.append(
+            Transformation(affine_transformation=trafo, reduction_point=reduction_point)
+        )
+
+    @property
+    def transformation(self):
+        """Access the affine transformations that were applied to this epoch
+
+        In order to set this property please use the transform method instead,
+        which will make sure to also apply the transformation.
+
+        :returns:
+            Returns a list of applied transformations. These are given
+            as a tuple of a 4x4 matrix defining the affine transformation
+            and the reduction point used when applying it.
+        """
+        return self._transformations
+
     def save(self, filename):
         """Save this epoch to a file
 
@@ -109,7 +318,6 @@ class Epoch(_py4dgeo.Epoch):
             with zipfile.ZipFile(
                 filename, mode="w", compression=zipfile.ZIP_BZIP2
             ) as zf:
-
                 # Write the epoch file format version number
                 zf.writestr("EPOCH_FILE_FORMAT", str(PY4DGEO_EPOCH_FILE_FORMAT_VERSION))
 
@@ -119,14 +327,55 @@ class Epoch(_py4dgeo.Epoch):
                     json.dump(self.metadata, f)
                 zf.write(metadatafile, arcname="metadata.json")
 
+                # Write the transformation into a file
+                trafofile = os.path.join(tmp_dir, "trafo.json")
+                with open(trafofile, "w") as f:
+                    json.dump(
+                        [t.__dict__ for t in self._transformations],
+                        f,
+                        cls=NumpyArrayEncoder,
+                    )
+                zf.write(trafofile, arcname="trafo.json")
+
                 # Write the actual point cloud array using laspy - LAZ compression
                 # is far better than any compression numpy + zipfile can do.
                 cloudfile = os.path.join(tmp_dir, "cloud.laz")
-                header = laspy.LasHeader(version="1.4", point_format=6)
-                lasfile = laspy.LasData(header)
+                hdr = laspy.LasHeader(version="1.4", point_format=6)
+                hdr.x_scale = 0.00025
+                hdr.y_scale = 0.00025
+                hdr.z_scale = 0.00025
+                mean_extent = np.mean(self.cloud, axis=0)
+                hdr.x_offset = int(mean_extent[0])
+                hdr.y_offset = int(mean_extent[1])
+                hdr.z_offset = int(mean_extent[2])
+                lasfile = laspy.LasData(hdr)
                 lasfile.x = self.cloud[:, 0]
                 lasfile.y = self.cloud[:, 1]
                 lasfile.z = self.cloud[:, 2]
+
+                # define dimensions for normals below:
+                if self._normals is not None:
+                    lasfile.add_extra_dim(
+                        laspy.ExtraBytesParams(
+                            name="NormalX", type="f8", description="X axis of normals"
+                        )
+                    )
+                    lasfile.add_extra_dim(
+                        laspy.ExtraBytesParams(
+                            name="NormalY", type="f8", description="Y axis of normals"
+                        )
+                    )
+                    lasfile.add_extra_dim(
+                        laspy.ExtraBytesParams(
+                            name="NormalZ", type="f8", description="Z axis of normals"
+                        )
+                    )
+                    lasfile.NormalX = self.normals[:, 0]
+                    lasfile.NormalY = self.normals[:, 1]
+                    lasfile.NormalZ = self.normals[:, 2]
+                else:
+                    logger.info("Saving a file without normals.")
+
                 lasfile.write(cloudfile)
                 zf.write(cloudfile, arcname="cloud.laz")
 
@@ -152,11 +401,12 @@ class Epoch(_py4dgeo.Epoch):
         with tempfile.TemporaryDirectory() as tmp_dir:
             # Open the ZIP archive
             with zipfile.ZipFile(filename, mode="r") as zf:
-
                 # Read the epoch file version number and compare to current
                 version = int(zf.read("EPOCH_FILE_FORMAT").decode())
-                if version != PY4DGEO_EPOCH_FILE_FORMAT_VERSION:
-                    raise Py4DGeoError("Epoch file format is out of date!")
+                if version > PY4DGEO_EPOCH_FILE_FORMAT_VERSION:
+                    raise Py4DGeoError(
+                        "Epoch file format not known - please update py4dgeo!"
+                    )
 
                 # Read the metadata JSON file
                 metadatafile = zf.extract("metadata.json", path=tmp_dir)
@@ -167,14 +417,26 @@ class Epoch(_py4dgeo.Epoch):
                 cloudfile = zf.extract("cloud.laz", path=tmp_dir)
                 lasfile = laspy.read(cloudfile)
                 cloud = np.vstack((lasfile.x, lasfile.y, lasfile.z)).transpose()
-
+                try:
+                    normals = np.vstack(
+                        (lasfile.NormalX, lasfile.NormalY, lasfile.NormalZ)
+                    ).transpose()
+                except AttributeError:
+                    normals = None
                 # Construct the epoch object
-                epoch = Epoch(cloud, **metadata)
+                epoch = Epoch(cloud, normals=normals, **metadata)
 
                 # Restore the KDTree object
                 kdtreefile = zf.extract("kdtree", path=tmp_dir)
                 epoch.kdtree.load_index(kdtreefile)
 
+                # Read the transformation if it exists
+                if version >= 3:
+                    trafofile = zf.extract("trafo.json", path=tmp_dir)
+                    with open(trafofile, "r") as f:
+                        trafo = json.load(f)
+                    epoch._transformations = [Transformation(**t) for t in trafo]
+
         return epoch
 
     def __getstate__(self):
@@ -242,37 +504,95 @@ def _as_tuple(x):
     return (x,)
 
 
-def read_from_xyz(*filenames, other_epoch=None, **parse_opts):
+def read_from_xyz(
+    *filenames,
+    xyz_columns=[0, 1, 2],
+    normal_columns=[],
+    additional_dimensions={},
+    **parse_opts,
+):
     """Create an epoch from an xyz file
 
     :param filename:
         The filename to read from. Each line in the input file is expected
         to contain three space separated numbers.
     :type filename: str
-    :param other_epoch:
-        An existing epoch that we want to be compatible with.
-    :type other_epoch: py4dgeo.Epoch
+    :param xyz_columns:
+        The column indices of X, Y and Z coordinates. Defaults to [0, 1, 2].
+    :type xyz_columns: list
+    :param normal_columns:
+        The column indices of the normal vector components. Leave empty, if
+        your data file does not contain normals, otherwise exactly three indices
+        for the x, y and z components need to be given.
+    :type normal_columns: list
     :param parse_opts:
         Additional options forwarded to numpy.genfromtxt. This can be used
         to e.g. change the delimiter character, remove header_lines or manually
         specify which columns of the input contain the XYZ coordinates.
+    :param additional_dimensions:
+        A dictionary, mapping column indices to names of additional data dimensions.
+        They will be read from the file and are accessible under their names from the
+        created Epoch objects.
+        Additional column indexes start with 3.
     :type parse_opts: dict
     """
 
     # Resolve the given path
     filename = find_file(filenames[0])
 
-    # Read the first cloud
+    # Ensure that usecols is not passed by the user, we need to use this
+    if "usecols" in parse_opts:
+        raise Py4DGeoError(
+            "read_from_xyz cannot be customized by using usecols, please use xyz_columns, normal_columns or additional_dimensions instead!"
+        )
+
+    # Read the point cloud
+    logger.info(f"Reading point cloud from file '{filename}'")
+
     try:
-        logger.info(f"Reading point cloud from file '{filename}'")
-        cloud = np.genfromtxt(filename, dtype=np.float64, **parse_opts)
+        cloud = np.genfromtxt(
+            filename, dtype=np.float64, usecols=xyz_columns, **parse_opts
+        )
     except ValueError:
-        raise Py4DGeoError(
-            "Malformed XYZ file - all rows are expected to have exactly three columns"
+        raise Py4DGeoError("Malformed XYZ file")
+
+    # Potentially read normals
+    normals = None
+    if normal_columns:
+        if len(normal_columns) != 3:
+            raise Py4DGeoError("normal_columns need to be a list of three integers!")
+
+        try:
+            normals = np.genfromtxt(
+                filename, dtype=np.float64, usecols=normal_columns, **parse_opts
+            )
+        except ValueError:
+            raise Py4DGeoError("Malformed XYZ file")
+
+    # Potentially read additional_dimensions passed by the user
+    additional_columns = np.empty(
+        shape=(cloud.shape[0], 1),
+        dtype=np.dtype([(name, "<f8") for name in additional_dimensions.values()]),
+    )
+
+    add_cols = list(sorted(additional_dimensions.keys()))
+    try:
+        parsed_additionals = np.genfromtxt(
+            filename, dtype=np.float64, usecols=add_cols, **parse_opts
         )
+        # Ensure that the parsed array is two-dimensional, even if only
+        # one additional dimension was given (avoids an edge case)
+        parsed_additionals = parsed_additionals.reshape(-1, 1)
+    except ValueError:
+        raise Py4DGeoError("Malformed XYZ file")
 
-    # Construct the new Epoch object
-    new_epoch = Epoch(cloud=cloud)
+    for i, col in enumerate(add_cols):
+        additional_columns[additional_dimensions[col]] = parsed_additionals[
+            :, i
+        ].reshape(-1, 1)
+
+    # Finalize the construction of the new epoch
+    new_epoch = Epoch(cloud, normals=normals, additional_dimensions=additional_columns)
 
     if len(filenames) == 1:
         # End recursion and return non-tuple to make the case that the user
@@ -281,20 +601,31 @@ def read_from_xyz(*filenames, other_epoch=None, **parse_opts):
     else:
         # Go into recursion
         return (new_epoch,) + _as_tuple(
-            read_from_xyz(*filenames[1:], other_epoch=new_epoch, **parse_opts)
+            read_from_xyz(
+                *filenames[1:],
+                xyz_columns=xyz_columns,
+                normal_columns=normal_columns,
+                additional_dimensions=additional_dimensions,
+                **parse_opts,
+            )
         )
 
 
-def read_from_las(*filenames, other_epoch=None):
+def read_from_las(*filenames, normal_columns=[], additional_dimensions={}):
     """Create an epoch from a LAS/LAZ file
 
     :param filename:
         The filename to read from. It is expected to be in LAS/LAZ format
         and will be processed using laspy.
     :type filename: str
-    :param other_epoch:
-        An existing epoch that we want to be compatible with.
-    :type other_epoch: py4dgeo.Epoch
+    :param normal_columns:
+        The column names of the normal vector components, e.g. "NormalX", "nx", "normal_x" etc., keep in mind that there
+        must be exactly 3 columns. Leave empty, if your data file does not contain normals.
+    :type normal_columns: list
+    :param additional_dimensions:
+        A dictionary, mapping names of additional data dimensions in the input
+        dataset to additional data dimensions in our epoch data structure.
+    :type additional_dimensions: dict
     """
 
     # Resolve the given path
@@ -304,16 +635,44 @@ def read_from_las(*filenames, other_epoch=None):
     logger.info(f"Reading point cloud from file '{filename}'")
     lasfile = laspy.read(filename)
 
+    cloud = np.vstack(
+        (
+            lasfile.x,
+            lasfile.y,
+            lasfile.z,
+        )
+    ).transpose()
+
+    normals = None
+    if normal_columns:
+        if len(normal_columns) != 3:
+            raise Py4DGeoError("normal_columns need to be a list of three strings!")
+
+        normals = np.vstack(
+            [
+                lasfile.points[normal_columns[0]],
+                lasfile.points[normal_columns[1]],
+                lasfile.points[normal_columns[2]],
+            ]
+        ).transpose()
+
+    # set scan positions
+    # build additional_dimensions dtype structure
+    additional_columns = np.empty(
+        shape=(cloud.shape[0], 1),
+        dtype=np.dtype([(name, "<f8") for name in additional_dimensions.values()]),
+    )
+    for column_id, column_name in additional_dimensions.items():
+        additional_columns[column_name] = np.array(
+            lasfile.points[column_id], dtype=np.int32
+        ).reshape(-1, 1)
+
     # Construct Epoch and go into recursion
     new_epoch = Epoch(
-        np.vstack(
-            (
-                lasfile.x,
-                lasfile.y,
-                lasfile.z,
-            )
-        ).transpose(),
+        cloud,
+        normals=normals,
         timestamp=lasfile.header.creation_date,
+        additional_dimensions=additional_columns,
     )
 
     if len(filenames) == 1:
@@ -323,7 +682,11 @@ def read_from_las(*filenames, other_epoch=None):
     else:
         # Go into recursion
         return (new_epoch,) + _as_tuple(
-            read_from_las(*filenames[1:], other_epoch=new_epoch)
+            read_from_las(
+                *filenames[1:],
+                normal_columns=normal_columns,
+                additional_dimensions=additional_dimensions,
+            )
         )
 
 
@@ -356,3 +719,27 @@ def normalize_timestamp(timestamp):
             return parsed
 
     raise Py4DGeoError(f"The timestamp '{timestamp}' was not understood by py4dgeo.")
+
+
+def scan_positions_info_from_dict(info_dict: dict):
+    if info_dict is None:
+        return None
+    if not isinstance(info_dict, dict):
+        raise Py4DGeoError(f"The input scan position information should be dictionary.")
+        return None
+    # Compatible with both integer key and string key as index of the scan positions in json file
+    # load scan positions from dictionary, standardize loading via json format dumps to string key
+    scanpos_dict_load = json.loads(json.dumps(info_dict))
+    sps_list = []
+    for i in range(1, 1 + len(scanpos_dict_load)):
+        sps_list.append(scanpos_dict_load[str(i)])
+
+    for sp in sps_list:
+        sp_check = True
+        sp_check = False if len(sp["origin"]) != 3 else sp_check
+        sp_check = False if not isinstance(sp["sigma_range"], float) else sp_check
+        sp_check = False if not isinstance(sp["sigma_scan"], float) else sp_check
+        sp_check = False if not isinstance(sp["sigma_yaw"], float) else sp_check
+        if not sp_check:
+            raise Py4DGeoError("Scan positions load failed, please check format. ")
+    return sps_list

py4dgeo/fallback.py

@@ -3,7 +3,7 @@
 from py4dgeo.m3c2 import M3C2
 
 import numpy as np
-import py4dgeo._py4dgeo as _py4dgeo
+import _py4dgeo
 
 
 def radius_workingset_finder(params: _py4dgeo.WorkingSetFinderParameters) -> np.ndarray: