ssb-sgis 1.0.4__py3-none-any.whl → 1.0.6__py3-none-any.whl

sgis/io/dapla_functions.py

@@ -11,6 +11,7 @@ import geopandas as gpd
 import joblib
 import pandas as pd
 import pyarrow
+import pyarrow.parquet as pq
 import shapely
 from geopandas import GeoDataFrame
 from geopandas import GeoSeries
@@ -18,6 +19,7 @@ from geopandas.io.arrow import _geopandas_to_arrow
 from pandas import DataFrame
 from pyarrow import ArrowInvalid
 
+from ..geopandas_tools.general import get_common_crs
 from ..geopandas_tools.sfilter import sfilter
 
 PANDAS_FALLBACK_INFO = " Set pandas_fallback=True to ignore this error."
@@ -63,6 +65,7 @@ def read_geopandas(
     if not isinstance(gcs_path, (str | Path | os.PathLike)):
         kwargs |= {"file_system": file_system, "pandas_fallback": pandas_fallback}
 
+        cols = {}
         if mask is not None:
             if not isinstance(gcs_path, GeoSeries):
                 bounds_series: GeoSeries = get_bounds_series(
@@ -95,14 +98,25 @@ def read_geopandas(
                 paths = list(gcs_path)
 
         if threads is None:
-            threads = min(len(gcs_path), int(multiprocessing.cpu_count())) or 1
+            threads = min(len(paths), int(multiprocessing.cpu_count())) or 1
 
         # recursive read with threads
         with joblib.Parallel(n_jobs=threads, backend="threading") as parallel:
             dfs: list[GeoDataFrame] = parallel(
                 joblib.delayed(read_geopandas)(x, **kwargs) for x in paths
             )
-        df = pd.concat(dfs)
+
+        if dfs:
+            df = pd.concat(dfs, ignore_index=True)
+            try:
+                df = GeoDataFrame(df)
+            except Exception as e:
+                if not pandas_fallback:
+                    print(e)
+                    raise e
+        else:
+            df = GeoDataFrame(cols | {"geometry": []})
+
         if mask is not None:
             return sfilter(df, mask)
         return df
@@ -131,6 +145,8 @@ def read_geopandas(
                     raise e.__class__(
                         f"{e.__class__.__name__}: {e} for {df}." + more_txt
                     ) from e
+            except Exception as e:
+                raise e.__class__(f"{e.__class__.__name__}: {e} for {gcs_path}.") from e
 
     else:
         with file_system.open(gcs_path, mode="rb") as file:
@@ -148,6 +164,10 @@ def read_geopandas(
                     raise e.__class__(
                         f"{e.__class__.__name__}: {e} for {df}. " + more_txt
                     ) from e
+            except Exception as e:
+                raise e.__class__(
+                    f"{e.__class__.__name__}: {e} for {df}." + more_txt
+                ) from e
 
     if mask is not None:
         return sfilter(df, mask)
@@ -159,14 +179,14 @@ def _get_bounds_parquet(
 ) -> tuple[list[float], dict] | tuple[None, None]:
     with file_system.open(path) as f:
         try:
-            num_rows = pyarrow.parquet.read_metadata(f).num_rows
+            num_rows = pq.read_metadata(f).num_rows
         except ArrowInvalid as e:
             if not file_system.isfile(f):
                 return None, None
             raise ArrowInvalid(e, path) from e
         if not num_rows:
             return None, None
-        meta = pyarrow.parquet.read_schema(f).metadata
+        meta = pq.read_schema(f).metadata
     try:
         meta = json.loads(meta[b"geo"])["columns"]["geometry"]
     except KeyError as e:
@@ -182,7 +202,7 @@ def _get_bounds_parquet(
 
 def _get_columns(path: str | Path, file_system: dp.gcs.GCSFileSystem) -> pd.Index:
     with file_system.open(path) as f:
-        schema = pyarrow.parquet.read_schema(f)
+        schema = pq.read_schema(f)
         index_cols = _get_index_cols(schema)
         return pd.Index(schema.names).difference(index_cols)
 
@@ -266,17 +286,13 @@ def get_bounds_series(
             for path in paths
         )
     crss = {json.dumps(x[1]) for x in bounds}
-    crss = {
-        crs
-        for crs in crss
-        if not any(str(crs).lower() == txt for txt in ["none", "null"])
-    }
-    if not crss:
-        crs = None
-    elif len(crss) == 1:
-        crs = next(iter(crss))
-    else:
-        raise ValueError(f"crs mismatch: {crss}")
+    crs = get_common_crs(
+        [
+            crs
+            for crs in crss
+            if not any(str(crs).lower() == txt for txt in ["none", "null"])
+        ]
+    )
     return GeoSeries(
         [shapely.box(*bbox[0]) if bbox[0] is not None else None for bbox in bounds],
         index=paths,
@@ -355,7 +371,7 @@ def write_geopandas(
                 schema_version=None,
                 write_covering_bbox=write_covering_bbox,
             )
-            pyarrow.parquet.write_table(table, buffer, compression="snappy", **kwargs)
+            pq.write_table(table, buffer, compression="snappy", **kwargs)
         return
 
     layer = kwargs.pop("layer", None)

sgis/maps/explore.py

@@ -22,6 +22,7 @@ import matplotlib
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
+import shapely
 import xyzservices
 from folium import plugins
 from geopandas import GeoDataFrame
@@ -219,7 +220,7 @@ class Explore(Map):
         "OpenStreetMap",
         "dark",
         "norge_i_bilder",
-        "grunnkart",
+        # "grunnkart",
     )
 
     def __init__(
@@ -464,9 +465,9 @@ class Explore(Map):
             pass
 
         try:
-            sample = to_gdf(to_shapely(sample)).explode(ignore_index=True)
+            sample = to_gdf(to_shapely(sample)).pipe(make_all_singlepart)
         except Exception:
-            sample = to_gdf(to_shapely(to_bbox(sample))).explode(ignore_index=True)
+            sample = to_gdf(to_shapely(to_bbox(sample))).pipe(make_all_singlepart)
 
         random_point = sample.sample_points(size=1)
 
@@ -666,7 +667,12 @@ class Explore(Map):
             if not len(gdf):
                 continue
 
-            gdf = self._to_single_geom_type(gdf)
+            gdf = self._to_single_geom_type(make_all_singlepart(gdf))
+
+            if not len(gdf):
+                continue
+
+            gdf.geometry = shapely.force_2d(gdf.geometry.values)
             gdf = self._prepare_gdf_for_map(gdf)
 
             gjs = self._make_geojson(
@@ -737,7 +743,12 @@ class Explore(Map):
             if not len(gdf):
                 continue
 
-            gdf = self._to_single_geom_type(gdf)
+            gdf = self._to_single_geom_type(make_all_singlepart(gdf))
+
+            if not len(gdf):
+                continue
+
+            gdf.geometry = shapely.force_2d(gdf.geometry.values)
             gdf = self._prepare_gdf_for_map(gdf)
 
             classified = self._classify_from_bins(gdf, bins=self.bins)

sgis/maps/map.py

@@ -16,6 +16,7 @@ from geopandas import GeoDataFrame
 from geopandas import GeoSeries
 from jenkspy import jenks_breaks
 from mapclassify import classify
+from pandas.errors import PerformanceWarning
 from shapely import Geometry
 
 from ..geopandas_tools.conversion import to_gdf
@@ -41,6 +42,8 @@ except ImportError:
 warnings.filterwarnings(
     action="ignore", category=matplotlib.MatplotlibDeprecationWarning
 )
+warnings.filterwarnings(action="ignore", category=PerformanceWarning)
+
 pd.options.mode.chained_assignment = None
 
 

sgis/maps/maps.py

@@ -389,7 +389,6 @@ def samplemap(
 
         sample = sample.clip(mask).explode(ignore_index=True).sample(1)
 
-    print(locals())
     random_point = sample.sample_points(size=1)
 
     try:

sgis/networkanalysis/closing_network_holes.py

@@ -3,17 +3,78 @@
 import geopandas as gpd
 import numpy as np
 import pandas as pd
+import shapely
 from geopandas import GeoDataFrame
 from geopandas import GeoSeries
 from pandas import DataFrame
-from shapely import shortest_line
 
 from ..geopandas_tools.conversion import coordinate_array
+from ..geopandas_tools.general import get_line_segments
 from ..geopandas_tools.neighbors import k_nearest_neighbors
+from ..geopandas_tools.sfilter import sfilter
 from .nodes import make_edge_wkt_cols
 from .nodes import make_node_ids
 
 
+def get_k_nearest_points_for_deadends(
+    lines: GeoDataFrame, k: int, max_distance: int
+) -> GeoDataFrame:
+
+    assert lines.index.is_unique
+    lines = lines.assign(_range_idx_left=range(len(lines)))
+    points = (
+        lines.assign(
+            geometry=lambda x: x.extract_unique_points().values,
+            _range_idx_right=range(len(lines)),
+        )
+        .explode(index_parts=False)
+        .sort_index()
+    )
+
+    points_grouper = points.groupby("_range_idx_right")["geometry"]
+    nodes = pd.concat(
+        [
+            points_grouper.nth(0),
+            points_grouper.nth(-1),
+        ]
+    )
+    nodes.index.name = "_range_idx_right"
+    nodes = nodes.reset_index()
+
+    def has_no_duplicates(nodes):
+        counts = nodes.geometry.value_counts()
+        return nodes.geometry.isin(counts[counts == 1].index)
+
+    deadends = nodes[has_no_duplicates].reset_index(drop=True)
+
+    deadends_buffered = deadends.assign(geometry=lambda x: x.buffer(max_distance))
+
+    segs_by_deadends = (
+        sfilter(lines, deadends_buffered)
+        .pipe(get_line_segments)
+        .sjoin(deadends_buffered)
+        .loc[lambda x: x["_range_idx_left"] != x["_range_idx_right"]]
+    )
+
+    lines_between = shapely.shortest_line(
+        segs_by_deadends.geometry.values,
+        deadends.loc[segs_by_deadends["index_right"].values].geometry.values,
+    )
+    segs_by_deadends.geometry.loc[:] = shapely.get_point(lines_between, 0)
+
+    length_mapper = dict(enumerate(shapely.length(lines_between)))
+    sorted_lengths = dict(
+        reversed(sorted(length_mapper.items(), key=lambda item: item[1]))
+    )
+    nearest_first = segs_by_deadends.iloc[list(sorted_lengths)]
+
+    k_nearest_per_deadend = nearest_first.geometry.groupby(level=0).apply(
+        lambda x: x.head(k)
+    )
+
+    return GeoDataFrame({"geometry": k_nearest_per_deadend.values}, crs=lines.crs)
+
+
 def close_network_holes(
     gdf: GeoDataFrame,
     max_distance: int | float,
@@ -94,13 +155,22 @@ def close_network_holes(
     lines, nodes = make_node_ids(gdf)
 
     # remove duplicates of lines going both directions
-    lines["sorted"] = [
-        "_".join(sorted([s, t]))
-        for s, t in zip(lines["source"], lines["target"], strict=True)
+    lines["_sorted"] = [
+        "_".join(sorted([source, target])) + str(round(length, 4))
+        for source, target, length in zip(
+            lines["source"], lines["target"], lines.length, strict=True
+        )
     ]
 
+    lines = lines.drop_duplicates("_sorted").drop(columns="_sorted")
+
+    # new_lines, angles = _close_holes_all_lines(
     new_lines: GeoSeries = _close_holes_all_lines(
-        lines.drop_duplicates("sorted"), nodes, max_distance, max_angle, idx_start=1
+        lines,
+        nodes,
+        max_distance,
+        max_angle,
+        idx_start=1,
     )
 
     new_lines = gpd.GeoDataFrame(
@@ -128,15 +198,6 @@ def close_network_holes(
     return pd.concat([lines, new_lines], ignore_index=True)
 
 
-def get_angle(array_a: np.ndarray, array_b: np.ndarray) -> np.ndarray:
-    dx = array_b[:, 0] - array_a[:, 0]
-    dy = array_b[:, 1] - array_a[:, 1]
-
-    angles_rad = np.arctan2(dx, dy)
-    angles_degrees = np.degrees(angles_rad)
-    return angles_degrees
-
-
 def close_network_holes_to_deadends(
     gdf: GeoDataFrame,
     max_distance: int | float,
@@ -221,7 +282,7 @@ def _close_holes_all_lines(
 ) -> GeoSeries:
     k = min(len(nodes), 50)
 
-    # make point gdf for the deadends and the other endpoint of the deadend lines
+    # make points for the deadends and the other endpoint of the deadend lines
     deadends_target = lines.loc[lines["n_target"] == 1].rename(
         columns={"target_wkt": "wkt", "source_wkt": "wkt_other_end"}
     )
@@ -251,6 +312,7 @@ def _close_holes_all_lines(
     # and endpoints of the new lines in lists, looping through the k neighbour points
     new_sources: list[str] = []
     new_targets: list[str] = []
+    # all_angles = []
     for i in np.arange(idx_start, k):
         # to break out of the loop if no new_targets that meet the condition are found
         len_now = len(new_sources)
@@ -270,8 +332,11 @@ def _close_holes_all_lines(
             deadends_other_end_array, deadends_array
         )
 
-        angles_difference = np.abs(
-            np.abs(angles_deadend_to_deadend_other_end) - np.abs(angles_deadend_to_node)
+        def get_angle_difference(angle1, angle2):
+            return np.abs((angle1 - angle2 + 180) % 360 - 180)
+
+        angles_difference = get_angle_difference(
+            angles_deadend_to_deadend_other_end, angles_deadend_to_node
         )
 
         angles_difference[
@@ -284,14 +349,18 @@ def _close_holes_all_lines(
         to_idx = indices[condition]
         to_wkt = nodes.iloc[to_idx]["wkt"]
 
+        # all_angles = all_angles + [
+        #     diff
+        #     for f, diff in zip(from_wkt, angles_difference[condition], strict=True)
+        #     if f not in new_sources
+        # ]
+
         # now add the wkts to the lists of new sources and targets. If the source
         # is already added, the new wks will not be added again
         new_targets = new_targets + [
             t for f, t in zip(from_wkt, to_wkt, strict=True) if f not in new_sources
         ]
-        new_sources = new_sources + [
-            f for f, _ in zip(from_wkt, to_wkt, strict=True) if f not in new_sources
-        ]
+        new_sources = new_sources + [f for f in from_wkt if f not in new_sources]
 
         # break out of the loop when no new new_targets meet the condition
         if len_now == len(new_sources):
@@ -300,7 +369,16 @@ def _close_holes_all_lines(
     # make GeoSeries with straight lines
     new_sources = gpd.GeoSeries.from_wkt(new_sources, crs=lines.crs)
     new_targets = gpd.GeoSeries.from_wkt(new_targets, crs=lines.crs)
-    return shortest_line(new_sources, new_targets)
+    return shapely.shortest_line(new_sources, new_targets)  # , all_angles
+
+
+def get_angle(array_a: np.ndarray, array_b: np.ndarray) -> np.ndarray:
+    dx = array_b[:, 0] - array_a[:, 0]
+    dy = array_b[:, 1] - array_a[:, 1]
+
+    angles_rad = np.arctan2(dx, dy)
+    angles_degrees = np.degrees(angles_rad)
+    return angles_degrees
 
 
 def _find_holes_deadends(
@@ -347,7 +425,7 @@ def _find_holes_deadends(
     to_geom = deadends.loc[to_idx, "geometry"].reset_index(drop=True)
 
     # GeoDataFrame with straight lines
-    new_lines = shortest_line(from_geom, to_geom)
+    new_lines = shapely.shortest_line(from_geom, to_geom)
     new_lines = gpd.GeoDataFrame({"geometry": new_lines}, geometry="geometry", crs=crs)
 
     return new_lines

sgis/networkanalysis/cutting_lines.py

@@ -7,19 +7,16 @@ import pandas as pd
 from geopandas import GeoDataFrame
 from pandas import DataFrame
 from pandas import Series
-from shapely import extract_unique_points
 from shapely import force_2d
 from shapely.geometry import LineString
 from shapely.geometry import Point
 
-from ..geopandas_tools.buffer_dissolve_explode import buff
-from ..geopandas_tools.conversion import to_gdf
+from ..geopandas_tools.general import _split_lines_by_points_along_line
 from ..geopandas_tools.geometry_types import get_geom_type
-from ..geopandas_tools.neighbors import get_k_nearest_neighbors
 from ..geopandas_tools.point_operations import snap_all
 from ..geopandas_tools.point_operations import snap_within_distance
-from ..geopandas_tools.sfilter import sfilter_split
-from .nodes import make_edge_coords_cols
+
+PRECISION = 1e-6
 
 
 def split_lines_by_nearest_point(
@@ -75,8 +72,6 @@ def split_lines_by_nearest_point(
     Not all lines were split. That is because some points were closest to an endpoint
     of a line.
     """
-    PRECISION = 1e-6
-
     if not len(gdf) or not len(points):
         return gdf
 
@@ -103,145 +98,7 @@ def split_lines_by_nearest_point(
     else:
         snapped = snap_all(points, gdf)
 
-    # find the lines that were snapped to (or are very close because of float rounding)
-    snapped_buff = buff(snapped, PRECISION, resolution=16)
-    relevant_lines, the_other_lines = sfilter_split(gdf, snapped_buff)
-
-    if max_distance and not len(relevant_lines):
-        if splitted_col:
-            return gdf.assign(**{splitted_col: 1})
-        return gdf
-
-    # need consistent coordinate dimensions later
-    # (doing it down here to not overwrite the original data)
-    relevant_lines.geometry = force_2d(relevant_lines.geometry)
-    snapped.geometry = force_2d(snapped.geometry)
-
-    # split the lines with buffer + difference, since shaply.split usually doesn't work
-    # relevant_lines["_idx"] = range(len(relevant_lines))
-    splitted = relevant_lines.overlay(snapped_buff, how="difference").explode(
-        ignore_index=True
-    )
-
-    # linearrings (maybe coded as linestrings) that were not split,
-    # do not have edges and must be added in the end
-    boundaries = splitted.geometry.boundary
-    circles = splitted[boundaries.is_empty]
-    splitted = splitted[~boundaries.is_empty]
-
-    if not len(splitted):
-        return pd.concat([the_other_lines, circles], ignore_index=True)
-
-    # the endpoints of the new lines are now sligtly off. Using get_k_nearest_neighbors
-    # to get the exact snapped point coordinates, . This will map the sligtly
-    # wrong line endpoints with the point the line was split by.
-
-    snapped["point_coords"] = [(geom.x, geom.y) for geom in snapped.geometry]
-
-    # get line endpoints as columns (source_coords and target_coords)
-    splitted = make_edge_coords_cols(splitted)
-
-    splitted_source = to_gdf(splitted["source_coords"], crs=gdf.crs)
-    splitted_target = to_gdf(splitted["target_coords"], crs=gdf.crs)
-
-    def get_nearest(splitted: GeoDataFrame, snapped: GeoDataFrame) -> pd.DataFrame:
-        """Find the nearest snapped point for each source and target of the lines."""
-        return get_k_nearest_neighbors(splitted, snapped, k=1).loc[
-            lambda x: x["distance"] <= PRECISION * 2
-        ]
-
-    # snapped = snapped.set_index("point_coords")
-    snapped.index = snapped.geometry
-    dists_source = get_nearest(splitted_source, snapped)
-    dists_target = get_nearest(splitted_target, snapped)
-
-    # neighbor_index: point coordinates as tuple
-    pointmapper_source: pd.Series = dists_source["neighbor_index"]
-    pointmapper_target: pd.Series = dists_target["neighbor_index"]
-
-    # now, we can replace the source/target coordinate with the coordinates of
-    # the snapped points.
-
-    splitted = _change_line_endpoint(
-        splitted,
-        indices=dists_source.index,
-        pointmapper=pointmapper_source,
-        change_what="first",
-    )  # i=0)
-
-    # same for the lines where the target was split, but change the last coordinate
-    splitted = _change_line_endpoint(
-        splitted,
-        indices=dists_target.index,
-        pointmapper=pointmapper_target,
-        change_what="last",
-    )  # , i=-1)
-
-    if splitted_col:
-        splitted[splitted_col] = 1
-
-    return pd.concat([the_other_lines, splitted, circles], ignore_index=True).drop(
-        ["source_coords", "target_coords"], axis=1
-    )
-
-
-def _change_line_endpoint(
-    gdf: GeoDataFrame,
-    indices: pd.Index,
-    pointmapper: pd.Series,
-    change_what: str | int,
-) -> GeoDataFrame:
-    """Modify the endpoints of selected lines in a GeoDataFrame based on an index mapping.
-
-    This function updates the geometry of specified line features within a GeoDataFrame,
-    changing either the first or last point of each line to new coordinates provided by a mapping.
-    It is typically used in scenarios where line endpoints need to be adjusted to new locations,
-    such as in network adjustments or data corrections.
-
-    Args:
-        gdf: A GeoDataFrame containing line geometries.
-        indices: An Index object identifying the rows in the GeoDataFrame whose endpoints will be changed.
-        pointmapper: A Series mapping from the index of lines to new point geometries.
-        change_what: Specifies which endpoint of the line to change. Accepts 'first' or 0 for the
-            starting point, and 'last' or -1 for the ending point.
-
-    Returns:
-        A GeoDataFrame with the specified line endpoints updated according to the pointmapper.
-
-    Raises:
-        ValueError: If `change_what` is not one of the accepted values ('first', 'last', 0, -1).
-    """
-    assert gdf.index.is_unique
-
-    if change_what == "first" or change_what == 0:
-        keep = "first"
-    elif change_what == "last" or change_what == -1:
-        keep = "last"
-    else:
-        raise ValueError(
-            f"change_what should be 'first' or 'last' or 0 or -1. Got {change_what}"
-        )
-
-    is_relevant = gdf.index.isin(indices)
-    relevant_lines = gdf.loc[is_relevant]
-
-    relevant_lines.geometry = extract_unique_points(relevant_lines.geometry)
-    relevant_lines = relevant_lines.explode(index_parts=False)
-
-    relevant_lines.loc[lambda x: ~x.index.duplicated(keep=keep), "geometry"] = (
-        relevant_lines.loc[lambda x: ~x.index.duplicated(keep=keep)]
-        .index.map(pointmapper)
-        .values
-    )
-
-    is_line = relevant_lines.groupby(level=0).size() > 1
-    relevant_lines_mapped = (
-        relevant_lines.loc[is_line].groupby(level=0)["geometry"].agg(LineString)
-    )
-
-    gdf.loc[relevant_lines_mapped.index, "geometry"] = relevant_lines_mapped
-
-    return gdf
+    return _split_lines_by_points_along_line(gdf, snapped, splitted_col=splitted_col)
 
 
 def cut_lines(

sgis/networkanalysis/finding_isolated_networks.py

@@ -91,6 +91,11 @@ def get_component_size(gdf: GeoDataFrame) -> GeoDataFrame:
     3          346
     Name: count, dtype: int64
     """
+    if not len(gdf):
+        gdf["component_index"] = None
+        gdf["component_size"] = None
+        return gdf
+
     gdf, _ = make_node_ids(gdf)
 
     edges = [
@@ -109,6 +114,7 @@ def get_component_size(gdf: GeoDataFrame) -> GeoDataFrame:
             for idx in component
         },
     ).transpose()
+
     mapper.columns = ["component_index", "component_size"]
 
     gdf["component_index"] = gdf["source"].map(mapper["component_index"])