samgis_core 2.0.2__tar.gz → 3.0.1__tar.gz

{samgis_core-2.0.2 → samgis_core-3.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: samgis_core
-Version: 2.0.2
+Version: 3.0.1
 Summary: SamGIS CORE
 License: MIT
 Author: alessandro trinca tornidor
@@ -16,6 +16,8 @@ Requires-Dist: numpy (==1.25.2) ; python_version >= "3.10" and python_version <
 Requires-Dist: numpy (>=1.26,<2.0) ; python_version >= "3.11" and python_version < "3.12"
 Requires-Dist: onnxruntime (>=1.18.1,<2.0.0)
 Requires-Dist: pillow (>=10.4.0,<11.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: structlog (>=24.4.0,<25.0.0)
 Project-URL: Source, https://gitlab.com/aletrn/samgis_core
 Description-Content-Type: text/markdown
 

{samgis_core-2.0.2 → samgis_core-3.0.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "samgis_core"
-version = "2.0.2"
+version = "3.0.1"
 description = "SamGIS CORE"
 authors = ["alessandro trinca tornidor <alessandro@trinca.tornidor.com>"]
 license = "MIT license"
@@ -8,26 +8,29 @@ readme = "README.md"
 
 [metadata]
 name = "samgis_core"
-version = "2.0.2"
+version = "3.0.1"
 
 [tool.poetry.urls]
 Source = "https://gitlab.com/aletrn/samgis_core"
 
 [tool.poetry.dependencies]
+bson = "^0.5.10"
+loguru = "^0.7.2"
 numpy = [
     {version = "1.25.2", python = "~3.10"},
     {version = "^1.26", python = "~3.11"}
 ]
 pillow = "^10.4.0"
 python = ">=3.10, <3.12"
+python-dotenv = "^1.0.1"
 onnxruntime = "^1.18.1"
-loguru = "^0.7.2"
-bson = "^0.5.10"
+structlog = "^24.4.0"
 
 [tool.poetry.group.test]
 optional = true
 
 [tool.poetry.group.test.dependencies]
+mpld3 = "^0.5.10"
 pytest = "^8.2.2"
 pytest-cov = "^5.0.0"
 

samgis_core-3.0.1/samgis_core/__init__.py

@@ -0,0 +1,18 @@
+"""Get machine learning predictions from geodata raster images"""
+import os
+from pathlib import Path
+
+import structlog
+from dotenv import load_dotenv
+
+from samgis_core.utilities import session_logger
+
+
+load_dotenv()
+PROJECT_ROOT_FOLDER = Path(globals().get("__file__", "./_")).absolute().parent.parent
+PROJECT_MODEL_FOLDER = Path(PROJECT_ROOT_FOLDER / "machine_learning_models")
+MODEL_FOLDER = os.getenv("MODEL_FOLDER", PROJECT_MODEL_FOLDER)
+LOG_JSON_FORMAT = bool(os.getenv("LOG_JSON_FORMAT", False))
+log_level = os.getenv("LOG_LEVEL", "INFO")
+session_logger.setup_logging(json_logs=LOG_JSON_FORMAT, log_level=log_level)
+app_logger = structlog.stdlib.get_logger(__name__)

{samgis_core-2.0.2 → samgis_core-3.0.1}/samgis_core/prediction_api/sam_onnx2.py

@@ -62,7 +62,7 @@ class SegmentAnythingONNX2:
             encoder_model_path, providers=providers
         )
         self.encoder_input_name = self.encoder_session.get_inputs()[0].name
-        app_logger.info("encoder_input_name:", self.encoder_input_name)
+        app_logger.info(f"encoder_input_name:{self.encoder_input_name}.")
         self.decoder_session = InferenceSession(
             decoder_model_path, providers=providers
         )

samgis_core-3.0.1/samgis_core/utilities/create_folders_if_not_exists.py

@@ -0,0 +1,59 @@
+import json
+import logging
+import os
+from pathlib import Path
+
+
+def stats_pathname(pathname: Path | str):
+    current_pathname = Path(pathname)
+    return current_pathname.is_dir()
+
+
+def create_folder_if_not_exists(pathname: Path | str):
+    current_pathname = Path(pathname)
+    try:
+        print(f"Pathname exists? {current_pathname.exists()}, That's a folder? {current_pathname.is_dir()}...")
+        logging.info(f"Pathname exists? {current_pathname.exists()}, That's a folder? {current_pathname.is_dir()}...")
+        current_pathname.unlink(missing_ok=True)
+    except PermissionError as pe:
+        print(f"permission denied on removing pathname before folder creation:{pe}.")
+        logging.error(f"permission denied on removing pathname before folder creation:{pe}.")
+    except IsADirectoryError as errdir:
+        print(f"that's a directory:{errdir}.")
+        logging.error(f"that's a directory:{errdir}.")
+
+    print(f"Creating pathname: {current_pathname} ...")
+    logging.info(f"Creating pathname: {current_pathname} ...")
+    current_pathname.mkdir(mode=0o770, parents=True, exist_ok=True)
+
+    print(f"assertion: pathname exists and is a folder: {current_pathname} ...")
+    logging.info(f"assertion: pathname exists and is a folder: {current_pathname} ...")
+    assert current_pathname.is_dir()
+
+
+def folders_creation(folders_map: dict | str = None, ignore_errors: bool = True):
+    enforce_validation_with_getenv = folders_map is None
+    if enforce_validation_with_getenv:
+        folders_map = os.getenv("FOLDERS_MAP")
+    try:
+        folders_dict = folders_map if isinstance(folders_map, dict) else json.loads(folders_map)
+        for folder_env_ref, folder_env_path in folders_dict.items():
+            logging.info(f"folder_env_ref:{folder_env_ref}, folder_env_path:{folder_env_path}.")
+            create_folder_if_not_exists(folder_env_path)
+            print("========")
+            if enforce_validation_with_getenv:
+                assert os.getenv(folder_env_ref) == folder_env_path
+    except (json.JSONDecodeError, TypeError) as jde:
+        logging.error(f"jde:{jde}.")
+        msg = "double check your variables, e.g. for misspelling like 'FOLDER_MAP'"
+        msg += "instead than 'FOLDERS_MAP', or invalid json values."
+        logging.error(msg)
+        for k_env, v_env in dict(os.environ).items():
+            logging.info(f"{k_env}, v_env:{v_env}.")
+        if not ignore_errors:
+            raise TypeError(jde)
+
+
+if __name__ == '__main__':
+    folders_creation()
+

samgis_core-3.0.1/samgis_core/utilities/session_logger.py

@@ -0,0 +1,129 @@
+import logging
+import sys
+
+import structlog
+from structlog.types import EventDict, Processor
+
+
+# https://github.com/hynek/structlog/issues/35#issuecomment-591321744
+def rename_event_key(_, __, event_dict: EventDict) -> EventDict:
+    """
+    Log entries keep the text message in the `event` field, but Datadog
+    uses the `message` field. This processor moves the value from one field to
+    the other.
+    See https://github.com/hynek/structlog/issues/35#issuecomment-591321744
+    """
+    event_dict["message"] = event_dict.pop("event")
+    return event_dict
+
+
+def drop_color_message_key(_, __, event_dict: EventDict) -> EventDict:
+    """
+    Uvicorn logs the message a second time in the extra `color_message`, but we don't
+    need it. This processor drops the key from the event dict if it exists.
+    """
+    event_dict.pop("color_message", None)
+    return event_dict
+
+
+def setup_logging(json_logs: bool = False, log_level: str = "INFO"):
+    timestamper = structlog.processors.TimeStamper(fmt="iso")
+
+    shared_processors: list[Processor] = [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_logger_name,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        structlog.stdlib.ExtraAdder(),
+        drop_color_message_key,
+        timestamper,
+        structlog.processors.StackInfoRenderer(),
+        # adapted from https://www.structlog.org/en/stable/standard-library.html
+        # If the "exc_info" key in the event dict is either true or a
+        # sys.exc_info() tuple, remove "exc_info" and render the exception
+        # with traceback into the "exception" key.
+        structlog.processors.format_exc_info,
+        # If some value is in bytes, decode it to a Unicode str.
+        structlog.processors.UnicodeDecoder(),
+        # Add callsite parameters.
+        structlog.processors.CallsiteParameterAdder(
+            {
+                structlog.processors.CallsiteParameter.FUNC_NAME,
+                structlog.processors.CallsiteParameter.LINENO,
+            }
+        ),
+        # Render the final event dict as JSON.
+    ]
+
+    if json_logs:
+        # We rename the `event` key to `message` only in JSON logs, as Datadog looks for the
+        # `message` key but the pretty ConsoleRenderer looks for `event`
+        shared_processors.append(rename_event_key)
+        # Format the exception only for JSON logs, as we want to pretty-print them when
+        # using the ConsoleRenderer
+        shared_processors.append(structlog.processors.format_exc_info)
+
+    structlog.configure(
+        processors=shared_processors
+        + [
+            # Prepare event dict for `ProcessorFormatter`.
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        cache_logger_on_first_use=True,
+    )
+
+    log_renderer: structlog.types.Processor
+    if json_logs:
+        log_renderer = structlog.processors.JSONRenderer()
+    else:
+        log_renderer = structlog.dev.ConsoleRenderer()
+
+    formatter = structlog.stdlib.ProcessorFormatter(
+        # These run ONLY on `logging` entries that do NOT originate within
+        # structlog.
+        foreign_pre_chain=shared_processors,
+        # These run on ALL entries after the pre_chain is done.
+        processors=[
+            # Remove _record & _from_structlog.
+            structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+            log_renderer,
+        ],
+    )
+
+    handler = logging.StreamHandler()
+    # Use OUR `ProcessorFormatter` to format all `logging` entries.
+    handler.setFormatter(formatter)
+    root_logger = logging.getLogger()
+    root_logger.addHandler(handler)
+    root_logger.setLevel(log_level.upper())
+
+    for _log in ["uvicorn", "uvicorn.error"]:
+        # Clear the log handlers for uvicorn loggers, and enable propagation
+        # so the messages are caught by our root logger and formatted correctly
+        # by structlog
+        logging.getLogger(_log).handlers.clear()
+        logging.getLogger(_log).propagate = True
+
+    # Since we re-create the access logs ourselves, to add all information
+    # in the structured log (see the `logging_middleware` in main.py), we clear
+    # the handlers and prevent the logs to propagate to a logger higher up in the
+    # hierarchy (effectively rendering them silent).
+    logging.getLogger("uvicorn.access").handlers.clear()
+    logging.getLogger("uvicorn.access").propagate = False
+
+    def handle_exception(exc_type, exc_value, exc_traceback):
+        """
+        Log any uncaught exception instead of letting it be printed by Python
+        (but leave KeyboardInterrupt untouched to allow users to Ctrl+C to stop)
+        See https://stackoverflow.com/a/16993115/3641865
+        """
+        if issubclass(exc_type, KeyboardInterrupt):
+            sys.__excepthook__(exc_type, exc_value, exc_traceback)
+            return
+
+        root_logger.error(
+            "Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback)
+        )
+
+    sys.excepthook = handle_exception

{samgis_core-2.0.2 → samgis_core-3.0.1}/samgis_core/utilities/type_hints.py

@@ -6,6 +6,9 @@ from PIL.Image import Image
 from numpy import ndarray
 
 
+class ListStr(list[str]): pass
+
+
 class DictStrInt(dict[str, int]): pass
 
 

samgis_core-2.0.2/samgis_core/__init__.py

@@ -1,9 +0,0 @@
-"""Get machine learning predictions from geodata raster images"""
-from pathlib import Path
-
-from samgis_core.utilities.fastapi_logger import setup_logging
-
-
-app_logger = setup_logging(debug=True)
-PROJECT_ROOT_FOLDER = Path(globals().get("__file__", "./_")).absolute().parent.parent
-MODEL_FOLDER = Path(PROJECT_ROOT_FOLDER / "machine_learning_models")

samgis_core-2.0.2/samgis_core/utilities/fastapi_logger.py

@@ -1,26 +0,0 @@
-import loguru
-
-
-format_string = "{time} - {level} - {file} - {function} - ({extra[request_id]}) {message} "
-
-
-def setup_logging(debug: bool = False, formatter: str = format_string) -> loguru.logger:
-    """
-    Create a logging instance with log string formatter.
-
-    Args:
-        debug: logging debug argument
-        formatter: log string formatter
-
-    Returns:
-        Logger
-
-    """
-    import sys
-
-    logger = loguru.logger
-    logger.remove()
-    level_logger = "DEBUG" if debug else "INFO"
-    logger.add(sys.stdout, format=formatter, level=level_logger)
-    logger.info(f"type_logger:{type(logger)}, logger:{logger}.")
-    return logger