mapreader 1.3.0__tar.gz → 1.3.2__tar.gz

{mapreader-1.3.0 → mapreader-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mapreader
-Version: 1.3.0
+Version: 1.3.2
 Summary: A computer vision pipeline for the semantic exploration of maps/images at scale
 Home-page: https://github.com/Living-with-machines/MapReader
 Download-URL: https://github.com/Living-with-machines/MapReader/archive/refs/heads/main.zip
@@ -29,7 +29,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: matplotlib<4.0.0,>=3.5.0
 Requires-Dist: numpy<2.0.0,>=1.21.5
-Requires-Dist: pandas<2.0.0,>=1.3.4
+Requires-Dist: pandas>=2.0.0
 Requires-Dist: pyproj<4.0.0,>=3.2.0
 Requires-Dist: azure-storage-blob<13.0.0,>=12.9.0
 Requires-Dist: aiohttp<4.0.0,>=3.8.1
@@ -53,10 +53,11 @@ Requires-Dist: simplekml<2.0.0,>=1.3.6
 Requires-Dist: versioneer>=0.28
 Requires-Dist: tqdm<5.0.0
 Requires-Dist: torchinfo<2.0.0
-Requires-Dist: openpyxl<4.0.0
+Requires-Dist: openpyxl<4.0.0,>=3.1.2
 Requires-Dist: geopandas<1.0.0
 Requires-Dist: pyogrio>=0.7.2
 Requires-Dist: cartopy>=0.22.0
+Requires-Dist: joblib>=1.4.0
 Provides-Extra: dev
 Requires-Dist: pytest<9.0.0; extra == "dev"
 Requires-Dist: pytest-cov<6.0.0,>=4.1.0; extra == "dev"

{mapreader-1.3.0 → mapreader-1.3.2}/mapreader/__init__.py

@@ -31,3 +31,8 @@ from . import _version
 __version__ = _version.get_versions()["version"]
 
 from mapreader.load import geo_utils
+
+import mapreader
+
+def print_version():
+    print(mapreader.__version__)

mapreader-1.3.2/mapreader/__main__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from . import print_version
+
+if __name__ == "__main__":
+    print_version()

{mapreader-1.3.0 → mapreader-1.3.2}/mapreader/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-04-29T15:52:44+0100",
+ "date": "2024-05-15T11:10:51+0100",
  "dirty": false,
  "error": null,
- "full-revisionid": "f256ae9b73ae9dd14d8bd72b61a86088f8f1905a",
- "version": "1.3.0"
+ "full-revisionid": "282b8733c2c83412b979c2c234adda1432691862",
+ "version": "1.3.2"
 }
 '''  # END VERSION_JSON
 

{mapreader-1.3.0 → mapreader-1.3.2}/mapreader/annotate/annotator.py

@@ -27,7 +27,7 @@ _CENTER_LAYOUT = widgets.Layout(
 )
 
 
-class Annotator(pd.DataFrame):
+class Annotator:
     """
     Annotator class for annotating patches with labels.
 
@@ -227,28 +227,29 @@ class Annotator(pd.DataFrame):
                 delimiter=delimiter,
             )
 
-        # initiate as a DataFrame
-        super().__init__(patch_df)
-
         ## pixel_bounds = x0, y0, x1, y1
-        self["min_x"] = self.pixel_bounds.apply(lambda x: x[0])
-        self["min_y"] = self.pixel_bounds.apply(lambda x: x[1])
-        self["max_x"] = self.pixel_bounds.apply(lambda x: x[2])
-        self["max_y"] = self.pixel_bounds.apply(lambda x: x[3])
+        patch_df["min_x"] = patch_df["pixel_bounds"].apply(lambda x: x[0])
+        patch_df["min_y"] = patch_df["pixel_bounds"].apply(lambda x: x[1])
+        patch_df["max_x"] = patch_df["pixel_bounds"].apply(lambda x: x[2])
+        patch_df["max_y"] = patch_df["pixel_bounds"].apply(lambda x: x[3])
 
         # Sort by sortby column if provided
         if isinstance(sortby, str):
-            if sortby in self.columns:
+            if sortby in patch_df.columns:
                 self._sortby = sortby
                 self._ascending = ascending
             else:
-                raise ValueError(f"[ERROR] {sortby} is not a column in the DataFrame.")
+                raise ValueError(
+                    f"[ERROR] {sortby} is not a column in the patch DataFrame."
+                )
         elif sortby is not None:
             raise ValueError("[ERROR] ``sortby`` must be a string or None.")
         else:
             self._sortby = None
             self._ascending = True
 
+        self.patch_df = patch_df
+
         self._labels = labels
         self.label_col = label_col
         self.patch_paths_col = patch_paths_col
@@ -299,6 +300,9 @@ class Annotator(pd.DataFrame):
         # Setup queue
         self._queue = []
 
+    def __len__(self):
+        return len(self.patch_df)
+
     @staticmethod
     def _load_dataframes(
         patch_paths: str | None = None,
@@ -499,7 +503,7 @@ class Annotator(pd.DataFrame):
 
             return True
 
-        queue_df = self.copy(deep=True)
+        queue_df = self.patch_df.copy(deep=True)
         queue_df = queue_df[queue_df[self.label_col].isna()]  # only unlabelled
         queue_df["eligible"] = queue_df.apply(check_eligibility, axis=1)
 
@@ -562,20 +566,20 @@ class Annotator(pd.DataFrame):
 
         ix = self._queue[self.current_index]
 
-        min_x = self.at[ix, "min_x"]
-        min_y = self.at[ix, "min_y"]
+        min_x = self.patch_df.at[ix, "min_x"]
+        min_y = self.patch_df.at[ix, "min_y"]
 
         # cannot assume all patches are same size
         try:
-            height, width, _ = self.at[ix, "shape"]
+            height, width, _ = self.patch_df.at[ix, "shape"]
         except KeyError:
-            im_path = self.at[ix, self.patch_paths_col]
+            im_path = self.patch_df.at[ix, self.patch_paths_col]
             im = Image.open(im_path)
             height = im.height
             width = im.width
 
-        current_parent = self.at[ix, "parent_id"]
-        parent_frame = self.query(f"parent_id=='{current_parent}'")
+        current_parent = self.patch_df.at[ix, "parent_id"]
+        parent_frame = self.patch_df.query(f"parent_id=='{current_parent}'")
 
         deltas = list(range(-self.surrounding, self.surrounding + 1))
         y_and_x = list(
@@ -770,7 +774,7 @@ class Annotator(pd.DataFrame):
 
         ix = self._queue[self.current_index]
 
-        img_path = self.at[ix, self.patch_paths_col]
+        img_path = self.patch_df.at[ix, self.patch_paths_col]
 
         self.render()
         return self.previous_index, self.current_index, img_path
@@ -794,7 +798,7 @@ class Annotator(pd.DataFrame):
 
         ix = self._queue[self.current_index]
 
-        img_path = self.at[ix, self.patch_paths_col]
+        img_path = self.patch_df.at[ix, self.patch_paths_col]
 
         self.render()
         return self.previous_index, self.current_index, img_path
@@ -826,7 +830,7 @@ class Annotator(pd.DataFrame):
                 # disable skip button when at last example
                 button.disabled = self.current_index >= len(self) - 1
             elif button.description != "submit":
-                if self.at[ix, self.label_col] == button.description:
+                if self.patch_df.at[ix, self.label_col] == button.description:
                     button.icon = "check"
                 else:
                     button.icon = ""
@@ -842,8 +846,8 @@ class Annotator(pd.DataFrame):
             else:
                 display(image.convert("RGB"))
             add_ins = []
-            if "url" in self.loc[ix].keys():
-                url = self.at[ix, "url"]
+            if "url" in self.patch_df.loc[ix].keys():
+                url = self.patch_df.at[ix, "url"]
                 text = f'<p><a href="{url}" target="_blank">Click to see entire map.</a></p>'
                 add_ins += [widgets.HTML(text)]
 
@@ -881,7 +885,7 @@ class Annotator(pd.DataFrame):
         PIL.Image
             A PIL.Image object of the image at the given index.
         """
-        image_path = self.at[ix, self.patch_paths_col]
+        image_path = self.patch_df.at[ix, self.patch_paths_col]
         image = Image.open(image_path)
 
         if self.resize_to is not None:
@@ -907,7 +911,7 @@ class Annotator(pd.DataFrame):
         """
         # ix = self.iloc[self.current_index].name
         ix = self._queue[self.current_index]
-        self.at[ix, self.label_col] = annotation
+        self.patch_df.at[ix, self.label_col] = annotation
         if self.auto_save:
             self._auto_save()
         self._next_example()
@@ -950,7 +954,9 @@ class Annotator(pd.DataFrame):
             A dataframe containing the labelled images and their associated
             label index.
         """
-        filtered_df = self[self[self.label_col].notna()].copy(deep=True)
+        filtered_df = self.patch_df[self.patch_df[self.label_col].notna()].copy(
+            deep=True
+        )
 
         # force image_id to be index (incase of integer index)
         # TODO: Force all indices to be integers so this is not needed
@@ -973,8 +979,8 @@ class Annotator(pd.DataFrame):
 
     @property
     def filtered(self) -> pd.DataFrame:
-        _filter = ~self[self.label_col].isna()
-        return self[_filter]
+        _filter = ~self.patch_df[self.label_col].isna()
+        return self.patch_df[_filter]
 
     def render_complete(self):
         """

{mapreader-1.3.0 → mapreader-1.3.2}/mapreader/classify/classifier.py

@@ -1367,7 +1367,7 @@ Use ``add_criterion`` to define one."
     def _initialize_model(
         self,
         model_name: str,
-        pretrained: bool | None = True,
+        weights: str | None = "DEFAULT",
         last_layer_num_classes: str | int | None = "default",
     ) -> tuple[Any, int, bool]:
         """
@@ -1378,8 +1378,9 @@ Use ``add_criterion`` to define one."
         ----------
         model_name : str
             Name of a PyTorch model. See https://pytorch.org/vision/0.8/models.html for options.
-        pretrained : bool, optional
-            Use pretrained version, by default ``True``
+        weights : str, optional
+            Weights to load into the model. If ``"DEFAULT"``, loads the default weights for the chosen model.
+            By default, ``"DEFAULT"``.
         last_layer_num_classes : str or int, optional
             Number of elements in the last layer. If ``"default"``, sets it to
             the number of classes. By default, ``"default"``.
@@ -1408,8 +1409,7 @@ Use ``add_criterion`` to define one."
 
         # Initialize these variables which will be set in this if statement.
         # Each of these variables is model specific.
-        model_dw = models.__getattribute__(model_name)
-        model_dw = model_dw(pretrained)
+        model_dw = models.get_model(model_name, weights=weights)
         input_size = (224, 224)
         is_inception = False
 

{mapreader-1.3.0 → mapreader-1.3.2}/mapreader/classify/load_annotations.py

@@ -395,7 +395,9 @@ Please check your image paths and update them if necessary.'
                 )
                 iter_ids.append(annots2review.iloc[image_idx].name)
                 # Add to reviewed
-                self.reviewed = self.reviewed.append(annots2review.iloc[image_idx])
+                self.reviewed = pd.concat(
+                    [self.reviewed, annots2review.iloc[image_idx : image_idx + 1]]
+                )
                 try:
                     self.reviewed.drop_duplicates(subset=[deduplicate_col])
                 except Exception:
@@ -416,9 +418,14 @@ Please check your image paths and update them if necessary.'
             ]:
                 list_input_ids = user_input_ids.split(",")
                 print(
-                    f"[INFO] Options for labels (or create a new label):{list(self.annotations[self.label_col].unique())}"
+                    f"[INFO] Options for labels:{list(self.annotations[self.label_col].unique())}"
                 )
                 input_label = input("Enter new label:  ")
+                if input_label not in list(self.annotations[self.label_col].unique()):
+                    print(
+                        f'[ERROR] Label "{input_label}" not found in the annotations. Please enter a valid label.'
+                    )
+                    continue
 
                 for input_id in list_input_ids:
                     input_id = int(input_id)

{mapreader-1.3.0 → mapreader-1.3.2}/mapreader/download/sheet_downloader.py

@@ -533,7 +533,7 @@ class SheetDownloader:
         """
         self.merger = TileMerger(output_folder=f"{path_save}/")
 
-    def _check_map_sheet_exists(self, feature: dict) -> bool:
+    def _check_map_sheet_exists(self, feature: dict, metadata_fname) -> bool:
         """
         Checks if a map sheet is already saved.
 
@@ -544,51 +544,87 @@ class SheetDownloader:
         Returns
         -------
         bool
-            True if file exists, False if not.
+            img_path if file exists, False if not.
         """
-        map_name = str("map_" + feature["properties"]["IMAGE"])
         path_save = self.merger.output_folder
-        if os.path.exists(f"{path_save}{map_name}.png"):
+
+        try:
+            # get image id with same coords in metadata
+            existing_metadata_df = pd.read_csv(
+                f"{path_save}{metadata_fname}", sep=",", index_col=0
+            )
+        except FileNotFoundError:
+            return False
+
+        polygon = get_polygon_from_grid_bb(feature["grid_bb"])
+        if str(polygon.bounds) in existing_metadata_df["coordinates"].values:
+            image_id = existing_metadata_df[
+                existing_metadata_df["coordinates"] == str(polygon.bounds)
+            ].iloc[0]["name"]
+        else:
+            return False  # coordinates not in metadata means image doesn't exist
+
+        if os.path.exists(f"{path_save}{image_id}"):
             try:
-                mpimg.imread(f"{path_save}{map_name}.png")
-                print(
-                    f'[INFO] "{path_save}{map_name}.png" already exists. Skipping download.'
-                )
-                return True
+                # check image is valid
+                mpimg.imread(f"{path_save}{image_id}")
+                return image_id
             except OSError:
                 return False
         return False
 
-    def _download_map(self, feature: dict, download_in_parallel: bool = True) -> bool:
+    def _download_map(
+        self,
+        feature: dict,
+        existing_id: str | bool,
+        download_in_parallel: bool = True,
+        overwrite: bool = False,
+    ) -> str | bool:
         """
         Downloads a single map sheet and saves as png file.
 
         Parameters
         ----------
         feature : dict
+            The feature for which to download the map sheet.
+        existing_id : str | bool
+            The existing image id if the map sheet already exists.
+        download_in_parallel : bool, optional
+            Whether to download tiles in parallel, by default ``True``.
+        overwrite : bool, optional
+            Whether to overwrite existing maps, by default ``False``.
 
         Returns
         -------
-        bool
-            True if map was downloaded successfully, False if not.
+        str or bool
+            image path if map was downloaded successfully, False if not.
         """
-        map_name = str("map_" + feature["properties"]["IMAGE"])
         self.downloader.download_tiles(
             feature["grid_bb"], download_in_parallel=download_in_parallel
         )
-        success = self.merger.merge(feature["grid_bb"], map_name)
-        if success:
-            print(f'[INFO] Downloaded "{map_name}.png"')
+
+        if existing_id is False:
+            map_name = f"map_{feature['properties']['IMAGE']}"
+        else:
+            map_name = existing_id[:-4]  # remove file extension (assuming .png)
+
+        img_path = self.merger.merge(
+            feature["grid_bb"], file_name=map_name, overwrite=overwrite
+        )
+
+        if img_path is not False:
+            print(f'[INFO] Downloaded "{img_path}"')
         else:
-            print(f'[WARNING] Download of "{map_name}.png" was unsuccessful.')
+            print(f'[WARNING] Download of "{img_path}" was unsuccessful.')
 
         shutil.rmtree(DEFAULT_TEMP_FOLDER)
-        return success
+        return img_path
 
     def _save_metadata(
         self,
         feature: dict,
         out_filepath: str,
+        img_path: str,
         metadata_to_save: dict | None = None,
         **kwargs: dict | None,
     ) -> None:
@@ -602,6 +638,8 @@ class SheetDownloader:
             The feature for which to extract the metadata from
         out_filepath : str
             The path to save metadata csv.
+        img_path : str
+            The path to the downloaded map sheet.
         metadata_to_save : dict, optional
             A dictionary containing column names (str) and metadata keys (str or list) to save to metadata csv.
             Multilayer keys should be passed as a list, i.e. ["key1","key2"] will search for ``self.features[i]["key1"]["key2"]``.
@@ -631,7 +669,7 @@ class SheetDownloader:
         metadata_dict = {col: None for col in metadata_cols}
 
         # get default metadata
-        metadata_dict["name"] = str("map_" + feature["properties"]["IMAGE"] + ".png")
+        metadata_dict["name"] = os.path.basename(img_path)
         metadata_dict["url"] = str(feature["properties"]["IMAGEURL"])
         if not self.published_dates:
             date_col = kwargs.get("date_col", None)
@@ -705,16 +743,25 @@ class SheetDownloader:
         """
 
         for feature in tqdm(features):
-            if not overwrite:
-                if self._check_map_sheet_exists(feature):
-                    continue
-            success = self._download_map(
-                feature, download_in_parallel=download_in_parallel
+            existing_id = self._check_map_sheet_exists(feature, metadata_fname)
+            if (
+                not overwrite and existing_id is not False
+            ):  # if map already exists and overwrite is False then skip
+                print(f'[INFO] "{existing_id}" already exists. Skipping download.')
+                continue
+            img_path = self._download_map(
+                feature,
+                existing_id,
+                download_in_parallel=download_in_parallel,
+                overwrite=overwrite,
             )
-            if success:
+            if img_path is not False:
                 metadata_path = f"{path_save}/{metadata_fname}"
                 self._save_metadata(
-                    feature=feature, out_filepath=metadata_path, **kwargs
+                    feature=feature,
+                    out_filepath=metadata_path,
+                    img_path=img_path,
+                    **kwargs,
                 )
 
     def download_all_map_sheets(
@@ -1242,7 +1289,6 @@ Try passing coordinates (min_x, max_x, min_y, max_y) instead or leave blank to a
                 pass
 
             for feature in features:
-                text_id = feature["wfs_id_no"]
                 coords = np.array(feature["geometry"]["coordinates"][0][0])
 
                 # Plot coordinates
@@ -1255,6 +1301,7 @@ Try passing coordinates (min_x, max_x, min_y, max_y) instead or leave blank to a
                 )
 
                 if add_id:
+                    text_id = feature["wfs_id_no"]
                     plt.text(
                         np.mean(coords[:, 0]) - 0.15,
                         np.mean(coords[:, 1]) - 0.05,

{mapreader-1.3.0 → mapreader-1.3.2}/mapreader/download/tile_merging.py

@@ -142,7 +142,12 @@ class TileMerger:
         tile_size = img_size[0]
         return tile_size
 
-    def merge(self, grid_bb: GridBoundingBox, file_name: str | None = None) -> bool:
+    def merge(
+        self,
+        grid_bb: GridBoundingBox,
+        file_name: str | None = None,
+        overwrite: bool = False,
+    ) -> str | bool:
         """Merges cells contained within GridBoundingBox.
 
         Parameters
@@ -151,11 +156,13 @@ class TileMerger:
             GridBoundingBox containing tiles to merge
         file_name : Union[str, None], optional
             Name to use when saving map
-
+            If None, default name will be used, by default None
+        overwrite : bool, optional
+            Whether or not to overwrite existing files, by default False
         Returns
         -------
-        bool
-            True if file has successfully downloaded, False if not.
+        str or bool
+            out path if file has successfully downloaded, False if not.
         """
         os.makedirs(self.output_folder, exist_ok=True)
 
@@ -191,11 +198,18 @@ class TileMerger:
             file_name = self._get_output_name(grid_bb)
 
         out_path = f"{self.output_folder}{file_name}.{self.img_output_format[0]}"
+        if not overwrite:
+            i = 1
+            while os.path.exists(out_path):
+                out_path = (
+                    f"{self.output_folder}{file_name}_{i}.{self.img_output_format[0]}"
+                )
+                i += 1
         merged_image.save(out_path, self.img_output_format[1])
-        success = True if os.path.exists(out_path) else False
-        if success:
-            logger.info(f"Merge successful! The image has been stored at '{out_path}'")
-        else:
+        success = out_path if os.path.exists(out_path) else False
+        if success is False:
             logger.warning(f"Merge unsuccessful! '{out_path}' not saved.")
+        else:
+            logger.info(f"Merge successful! The image has been stored at '{out_path}'")
 
         return success

{mapreader-1.3.0 → mapreader-1.3.2}/mapreader/load/images.py

@@ -1392,7 +1392,6 @@ See https://pillow.readthedocs.io/en/stable/handbook/concepts.html#modes for mor
                 min_x, min_y, max_x, max_y = self.patches[patch]["pixel_bounds"]
                 if width != max_x - min_x:
                     width = max_x - min_x
-                img = img.crop((0, 0, width, height))
                 if height != max_y - min_y:
                     height = max_y - min_y
                 img = img.crop((0, 0, width, height))
@@ -1400,22 +1399,24 @@ See https://pillow.readthedocs.io/en/stable/handbook/concepts.html#modes for mor
                 bands = img.getbands()
 
                 if calc_mean:
-                    if all(f"mean_pixel_{band}" in patch_keys for band in bands):
+                    if "mean_pixel" in patch_keys:
                         calc_mean = False
                 if calc_std:
-                    if all(f"std_pixel_{band}" in patch_keys for band in bands):
+                    if "std_pixel" in patch_keys:
                         calc_std = False
 
                 img_stat = ImageStat.Stat(img)
 
                 if calc_mean:
                     img_mean = img_stat.mean
+                    self.patches[patch]["mean_pixel"] = np.mean(img_mean) / 255
                     for i, band in enumerate(bands):
                         # Calculate mean pixel values
                         self.patches[patch][f"mean_pixel_{band}"] = img_mean[i] / 255
 
                 if calc_std:
                     img_std = img_stat.stddev
+                    self.patches[patch]["std_pixel"] = np.mean(img_std) / 255
                     for i, band in enumerate(bands):
                         # Calculate std pixel values
                         self.patches[patch][f"std_pixel_{band}"] = img_std[i] / 255