grants-shared 0.1.0__py3-none-any.whl

grants_shared/logs/__init__.py

@@ -0,0 +1,31 @@
+"""Module for initializing logging configuration for the application.
+
+There are two formatters for the log messages: human-readable and JSON.
+The formatter that is used is determined by the environment variable
+LOG_FORMAT. If the environment variable is not set, the JSON formatter
+is used by default. See grants_shared.logs.formatters for more information.
+
+The logger also adds a PII mask filter to the root logger. See
+grants_shared.logs.pii for more information.
+
+Usage:
+    import grants_shared.logs
+
+    with grants_shared.logs.init("program name"):
+        ...
+
+Once the module has been initialized, the standard logging module can be
+used to log messages:
+
+Example:
+    import logging
+
+    logger = logging.getLogger(__name__)
+    logger.info("message")
+"""
+
+import grants_shared.logs.config as config
+
+
+def init(program_name: str) -> config.LoggingContext:
+    return config.LoggingContext(program_name)

grants_shared/logs/audit.py

@@ -0,0 +1,129 @@
+#
+# Application-level audit logging.
+#
+# See https://docs.python.org/3/library/audit_events.html
+# https://docs.python.org/3/library/sys.html#sys.addaudithook
+# https://www.python.org/dev/peps/pep-0578/
+#
+import collections
+import logging
+import sys
+from collections.abc import Hashable, Sequence
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+AUDIT = 32
+logging.addLevelName(AUDIT, "AUDIT")
+
+
+def init() -> None:
+    """Initialize the audit logging module to start
+    logging security audit events."""
+    sys.addaudithook(handle_audit_event)
+
+
+def handle_audit_event(event_name: str, args: tuple[Any, ...]) -> None:
+    # Define events to log and the arguments to log for each event.
+    # For more information about these events and what they mean, see https://peps.python.org/pep-0578/#suggested-audit-hook-locations
+    # For the full list of auditable events, see https://docs.python.org/3/library/audit_events.html
+    # Define this variable locally so it can't be modified by other modules.
+
+    EVENTS_TO_LOG = {
+        # Detect dynamic execution of code objects. This only occurs for explicit
+        # calls, and is not raised for normal function invocation.
+        "exec": ("code_object",),
+        # Detect when a file is about to be opened. path and mode are the usual
+        # parameters to open if available, while flags is provided instead of
+        # mode in some cases.
+        "open": ("path", "mode", "flags"),
+        # Detect when a signal is sent to a process.
+        "os.kill": ("pid", "sig"),
+        # Detect when a file is renamed.
+        "os.rename": ("src", "dst", "src_dir_fd", "dst_dir_fd"),
+        # Detect when a subprocess is started.
+        "subprocess.Popen": ("executable", "args", "cwd", "_"),
+        # Detect access to network resources. The address is unmodified from the original call.
+        "socket.connect": ("socket", "address"),
+        "socket.getaddrinfo": ("host", "port", "family", "type", "protocol"),
+        # Detect when new audit hooks are being added.
+        "sys.addaudithook": (),
+        # Detects URL requests.
+        # Don't log data or headers because they may contain sensitive information.
+        "urllib.Request": ("url", "_", "_", "method"),
+    }
+
+    if event_name not in EVENTS_TO_LOG:
+        return
+
+    arg_names = EVENTS_TO_LOG[event_name]
+    log_audit_event(event_name, args, arg_names)
+
+
+# Set the audit hook to be traceable so that coverage module can track calls to it
+# The coverage module relies on Python's trace hooks
+# (See https://coverage.readthedocs.io/en/7.1.0/howitworks.html#execution)
+# According to the docs for sys.addaudithook, the audit hook is only traced if the callable
+# has a __cantrace__ member that is set to a true value.
+# (See https://docs.python.org/3/library/sys.html#sys.addaudithook)
+handle_audit_event.__cantrace__ = True  # type: ignore
+
+
+def log_audit_event(event_name: str, args: Sequence[Any], arg_names: Sequence[str]) -> None:
+    """Log a message but only log recently repeated messages at intervals."""
+    extra = {
+        f"audit.args.{arg_name}": arg
+        for arg_name, arg in zip(arg_names, args, strict=True)
+        if arg_name != "_"
+    }
+
+    key = (event_name, repr(args))
+    if key not in audit_message_count:
+        count = 1
+    else:
+        count = audit_message_count[key] + 1
+    audit_message_count[key] = count
+
+    if count > 100 and count % 100 != 0:
+        return
+
+    if count > 10 and count % 10 != 0:
+        return
+
+    extra["count"] = count
+
+    logger.log(AUDIT, event_name, extra=extra)
+
+
+class LeastRecentlyUsedDict(collections.OrderedDict):
+    """A dict with a maximum size, evicting the least recently written key when full.
+
+    Getting a key that is not present returns a default value of 0.
+
+    Setting a key marks it as most recently used and removes the oldest key if full.
+
+    May be useful for tracking the count of items where limited memory usage is needed even if
+    the set of items can be unlimited.
+
+    Based on the example at
+    https://docs.python.org/3/library/collections.html#ordereddict-examples-and-recipes
+    """
+
+    def __init__(self, maxsize: int = 128, *args: Any, **kwargs: Any) -> None:
+        self.maxsize = maxsize
+        super().__init__(*args, **kwargs)
+
+    def __getitem__(self, key: Hashable) -> int:
+        if key in self:
+            return super().__getitem__(key)
+        return 0
+
+    def __setitem__(self, key: Hashable, value: int) -> None:
+        if key in self:
+            self.move_to_end(key)
+        super().__setitem__(key, value)
+        if self.maxsize < len(self):
+            self.popitem(last=False)
+
+
+audit_message_count = LeastRecentlyUsedDict()

grants_shared/logs/config.py

@@ -0,0 +1,165 @@
+import contextlib
+import logging
+import os
+import platform
+import pwd
+import sys
+from typing import Any, cast
+
+from pydantic_settings import SettingsConfigDict
+
+import grants_shared.logs.audit
+import grants_shared.logs.formatters as formatters
+import grants_shared.logs.pii as pii
+from grants_shared.util.env_config import PydanticBaseEnvConfig
+
+logger = logging.getLogger(__name__)
+
+_original_argv = tuple(sys.argv)
+
+
+class HumanReadableFormatterConfig(PydanticBaseEnvConfig):
+    message_width: int = formatters.HUMAN_READABLE_FORMATTER_DEFAULT_MESSAGE_WIDTH
+
+
+class LoggingConfig(PydanticBaseEnvConfig):
+    model_config = SettingsConfigDict(env_prefix="log_", env_nested_delimiter="__")
+
+    format: str = "json"
+    level: str = "INFO"
+    enable_audit: bool = False
+    human_readable_formatter: HumanReadableFormatterConfig = HumanReadableFormatterConfig()
+
+    # Specify logging_level_overrides formatted as "<logger>=<level>" like "newrelic=INFO,something.else=ERROR"
+    level_overrides: str | None = None
+
+
+class LoggingContext(contextlib.AbstractContextManager[None]):
+    """
+    A context manager for handling setting up the logging stream.
+
+    To help facillitate being able to test logging, we need to be able
+    to easily create temporary output streams and then tear them down.
+
+    When this context manager is torn down, the stream handler created
+    with it will be removed.
+
+    For example:
+    ```py
+    import logging
+
+    logger = logging.getLogger(__name__)
+
+    with LoggingContext("example_program_name"):
+        # This log message will go to stdout
+        logger.info("example log message")
+
+    # This log message won't go to stdout as the
+    # handler will have been removed
+    logger.info("example log message")
+    ```
+    Note that any other handlers added to the root logger won't be affected
+    and calling this multiple times before exit would result in duplicate logs.
+    """
+
+    def __init__(self, program_name: str) -> None:
+        self._configure_logging()
+        log_program_info(program_name)
+
+    def __enter__(self) -> None:
+        pass
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        # Remove the console handler to stop logs from being sent to stdout
+        # This is useful in the test suite, since multiple tests may initialize
+        # separate duplicate handlers. This allows for easier cleanup for each
+        # of those tests.
+        logging.root.removeHandler(self.console_handler)
+
+    def _configure_logging(self) -> None:
+        """Configure logging for the application.
+
+        Configures the root module logger to log to stdout.
+        Adds a PII mask filter to the root logger.
+        Also configures log levels third party packages.
+        """
+        config = LoggingConfig()
+
+        # Loggers can be configured using config functions defined
+        # in logging.config or by directly making calls to the main API
+        # of the logging module (see https://docs.python.org/3/library/logging.config.html)
+        # We opt to use the main API using functions like `addHandler` which is
+        # non-destructive, i.e. it does not overwrite any existing handlers.
+        # In contrast, logging.config.dictConfig() would overwrite any existing loggers.
+        # This is important during testing, since fixtures like `caplog` add handlers that would
+        # get overwritten if we call logging.config.dictConfig() during the scope of the test.
+        self.console_handler = logging.StreamHandler(sys.stdout)
+        formatter = get_formatter(config)
+        self.console_handler.setFormatter(formatter)
+        self.console_handler.addFilter(pii.mask_pii)
+        logging.root.addHandler(self.console_handler)
+        logging.root.setLevel(config.level)
+
+        if config.enable_audit:
+            grants_shared.logs.audit.init()
+
+        # Configure loggers for third party packages
+        logging.getLogger("alembic").setLevel(logging.INFO)
+        logging.getLogger("werkzeug").setLevel(logging.WARN)
+        logging.getLogger("sqlalchemy.pool").setLevel(logging.INFO)
+        logging.getLogger("sqlalchemy.dialects.postgresql").setLevel(logging.INFO)
+
+        # Allow an env var to override logging config, mostly for development purposes
+        # Parsing string formatted like "logger1=INFO,logger2=ERROR"
+        if config.level_overrides is not None:
+            for override in config.level_overrides.split(","):
+                logger_override, level_override = override.split("=")
+                logging.getLogger(logger_override).setLevel(level_override)
+
+
+def get_formatter(config: LoggingConfig) -> logging.Formatter:
+    """Return the formatter used by the root logger.
+
+    The formatter is determined by the environment variable LOG_FORMAT. If the
+    environment variable is not set, the JSON formatter is used by default.
+    """
+    if config.format == "human-readable":
+        return get_human_readable_formatter(config.human_readable_formatter)
+    return formatters.JsonFormatter()
+
+
+def log_program_info(program_name: str) -> None:
+    logger.info(
+        "start %s: %s %s %s, hostname %s, pid %i, user %i(%s)",
+        program_name,
+        platform.python_implementation(),
+        platform.python_version(),
+        platform.system(),
+        platform.node(),
+        os.getpid(),
+        os.getuid(),
+        pwd.getpwuid(os.getuid()).pw_name,
+        extra={
+            "hostname": platform.node(),
+            "cpu_count": os.cpu_count(),
+            # If mypy is run on a mac, it will throw a module has no attribute error, even though
+            # we never actually access it with the conditional.
+            #
+            # However, we can't just silence this error, because on linux (e.g. CI/CD) that will
+            # throw an unused “type: ignore” comment error. Casting to Any instead ensures this
+            # passes regardless of where mypy is being run
+            "cpu_usable": (
+                len(cast(Any, os).sched_getaffinity(0))
+                if "sched_getaffinity" in dir(os)
+                else "unknown"
+            ),
+        },
+    )
+    logger.info("invoked as: %s", " ".join(_original_argv))
+
+
+def get_human_readable_formatter(
+    config: HumanReadableFormatterConfig,
+) -> formatters.HumanReadableFormatter:
+    """Return the human readable formatter used by the root logger."""
+    return formatters.HumanReadableFormatter(message_width=config.message_width)

grants_shared/logs/decodelog.py

@@ -0,0 +1,156 @@
+#
+# Make JSON logs easier to read when developing or troubleshooting.
+#
+# Expects JSON log lines or `docker-compose log` output on stdin and outputs plain text lines on
+# stdout.
+#
+# This module intentionally has no dependencies outside the standard library so that it can be run
+# as a script outside the virtual environment if needed.
+#
+# mypy: disallow-untyped-defs
+
+import datetime
+import json
+import sys
+from collections.abc import Mapping
+
+RED = "\033[31m"
+GREEN = "\033[32m"
+BLUE = "\033[34m"
+ORANGE = "\033[38;5;208m"
+RESET = "\033[0m"
+NO_COLOUR = ""
+
+DEFAULT_MESSAGE_WIDTH = 50
+
+output_dates = None
+
+
+def main() -> None:
+    """Main entry point when used as a script."""
+    for line in sys.stdin:
+        processed = process_line(line)
+        if processed is not None:
+            sys.stdout.write(processed)
+            sys.stdout.write("\r\n")
+
+
+def process_line(line: str) -> str | None:
+    """Process a line of the log and return the reformatted line."""
+    line = line.rstrip()
+    if line and line[0] == "{":
+        # JSON format
+        return decode_json_line(line)
+    elif "| {" in line:
+        # `docker-compose logs ...` format
+        return decode_json_line(line[line.find("| {") + 2 :])
+    # Anything else is left alone
+    return line
+
+
+def decode_json_line(line: str) -> str | None:
+    """Decode a JSON log line and return the reformatted line."""
+    try:
+        data = json.loads(line)
+    except json.decoder.JSONDecodeError:
+        return line
+
+    name = data.pop("name", "-")
+    level = data.pop("levelname", "-")
+    func_name = data.pop("funcName", "-")
+    created = datetime.datetime.fromtimestamp(
+        float(data.pop("created", 0)), tz=datetime.timezone.utc
+    )
+    message = data.pop("message", "-")
+
+    if level == "AUDIT":
+        return None
+
+    return format_line(created, name, func_name, level, message, data)
+
+
+def format_line(
+    created: datetime.datetime,
+    logger_name: str,
+    func_name: str,
+    level: str,
+    message: str,
+    extra: Mapping[str, str],
+    message_width: int = DEFAULT_MESSAGE_WIDTH,
+) -> str:
+    """Format log fields as a coloured string."""
+    logger_name_color = color_for_name(logger_name)
+    level_color = color_for_level(level)
+    return f"{format_datetime(created)}  {colorize(logger_name.ljust(36), logger_name_color)} {func_name:<28} {colorize(level.ljust(8), level_color)} {colorize(message.ljust(message_width), level_color)} {colorize(format_extra(extra), BLUE)}"
+
+
+def colorize(text: str, color: str) -> str:
+    return f"{color}{text}{RESET}"
+
+
+def color_for_name(name: str) -> str:
+    if name.startswith("src"):
+        return GREEN
+    elif name.startswith("sqlalchemy"):
+        return ORANGE
+    return NO_COLOUR
+
+
+def color_for_level(level: str) -> str:
+    if level in ("WARNING", "ERROR", "CRITICAL"):
+        return RED
+    return NO_COLOUR
+
+
+def format_datetime(created: datetime.datetime) -> str:
+    global output_dates
+    if output_dates is None:
+        # Check first line - if over 10h ago, output dates as well as time.
+        output_dates = 36000 < (datetime.datetime.now() - created).total_seconds()
+    if output_dates:
+        return created.isoformat(timespec="milliseconds")
+    else:
+        return created.time().isoformat(timespec="milliseconds")
+
+
+EXCLUDE_EXTRA = {
+    "args",
+    "created",
+    "entity.guid",
+    "entity.name",
+    "entity.type",
+    "exc_info",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "message",
+    "module",
+    "msecs",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "span.id",
+    "thread",
+    "threadName",
+    "trace.id",
+    "traceId",
+    "deploy_github_ref",
+    "deploy_github_sha",
+}
+
+
+def format_extra(data: Mapping[str, str]) -> str:
+    return " ".join(
+        "%s=%s" % (key, value)
+        for key, value in data.items()
+        if key not in EXCLUDE_EXTRA and value is not None
+    )
+
+
+if __name__ == "__main__":
+    main()

grants_shared/logs/flask_logger.py

@@ -0,0 +1,268 @@
+"""Module for adding standard logging functionality to a Flask app.
+
+This module configures an application's logger to add extra data
+to all log messages. Flask application context data such as the
+app name and request context data such as the request method, request url
+rule, and query parameters are added to the log record.
+
+This module also configures the Flask application to log every
+non-404 request.
+
+Usage:
+    import grants_shared.logs.flask_logger as flask_logger
+
+    logger = logging.getLogger(__name__)
+    app = create_app()
+
+    flask_logger.init_app(logger, app)
+"""
+
+import logging
+import os
+import sys
+import time
+import uuid
+
+import flask
+import newrelic.api.time_trace
+
+from grants_shared.util.deploy_metadata import get_deploy_metadata_config
+
+logger = logging.getLogger(__name__)
+EXTRA_LOG_DATA_ATTR = "extra_log_data"
+
+_GLOBAL_LOG_CONTEXT: dict = {}
+
+
+def init_general_logging(app_logger: logging.Logger, app_name: str, app_domain: str) -> None:
+    """Initialize logging that doesn't depend on a Flask app
+
+    If possible, use init_app instead which is called when we
+    create a flask app, this is only necessary for scripts that
+    aren't possible to run via Flask like our Alembic migrations
+    """
+
+    # Need to add filters to each of the handlers rather than to the logger itself, since
+    # messages are passed directly to the ancestor loggers’ handlers bypassing any filters
+    # set on the ancestors.
+    # See https://docs.python.org/3/library/logging.html#logging.Logger.propagate
+    for handler in app_logger.handlers:
+        handler.addFilter(_add_global_context_info_to_log_record)
+        handler.addFilter(_add_request_context_info_to_log_record)
+        handler.addFilter(_add_new_relic_context_to_log_record)
+        handler.addFilter(_add_error_info_to_log_record)
+
+    deploy_metadata = get_deploy_metadata_config()
+
+    # Add some metadata to all log messages globally
+    add_extra_data_to_global_logs(
+        {
+            "app.name": app_name,
+            "app_name": "api",
+            "app_domain": app_domain,
+            "run_mode": get_run_mode(),
+            "environment": os.environ.get("ENVIRONMENT"),
+            "deploy_github_ref": deploy_metadata.deploy_github_ref,
+            "deploy_github_sha": deploy_metadata.deploy_github_sha,
+            "deploy_whoami": deploy_metadata.deploy_whoami,
+        }
+    )
+
+    app_logger.info("initialized flask logger")
+
+
+def init_app(app_logger: logging.Logger, app: flask.Flask, app_domain: str) -> None:
+    """Initialize the Flask app logger.
+
+    Adds Flask app context data and Flask request context data
+    to every log record using log filters.
+    See https://docs.python.org/3/howto/logging-cookbook.html#using-filters-to-impart-contextual-information
+
+    Also configures the app to log every non-404 request using the given logger.
+
+    Usage:
+        import grants_shared.logs.flask_logger as flask_logger
+
+        logger = logging.getLogger(__name__)
+        app = create_app()
+
+        flask_logger.init_app(logger, app)
+    """
+
+    # Add request context data to every log record for the current request
+    # such as request id, request method, request path, and the matching Flask request url rule
+    app.before_request(
+        lambda: add_extra_data_to_current_request_logs(_get_request_context_info(flask.request))
+    )
+
+    app.before_request(_track_request_start_time)
+    app.before_request(_log_start_request)
+    app.after_request(_log_end_request)
+
+    init_general_logging(app_logger, app.name, app_domain)
+
+
+def add_extra_data_to_current_request_logs(
+    data: dict[str, str | int | float | bool | uuid.UUID | None],
+) -> None:
+    """Add data to every log record for the current request."""
+    if not flask.has_request_context():
+        return
+
+    extra_log_data = getattr(flask.g, EXTRA_LOG_DATA_ATTR, {})
+    extra_log_data.update(data)
+    setattr(flask.g, EXTRA_LOG_DATA_ATTR, extra_log_data)
+
+
+def add_extra_data_to_global_logs(data: dict[str, str | int | float | bool | None]) -> None:
+    """Add metadata to all logs for the rest of the lifecycle of this app process"""
+    global _GLOBAL_LOG_CONTEXT
+    _GLOBAL_LOG_CONTEXT.update(data)
+
+
+def _track_request_start_time() -> None:
+    """Store the request start time in flask.g"""
+    flask.g.request_start_time = time.perf_counter()
+
+
+def _log_start_request() -> None:
+    """Log the start of a request.
+
+    This function handles the Flask's before_request event.
+    See https://tedboy.github.io/flask/interface_src.application_object.html#flask.Flask.before_request
+
+    Additional info about the request will be in the `extra` field
+    added by `_add_request_context_info_to_log_record`
+    """
+    logger.info("start request")
+
+
+def _log_end_request(response: flask.Response) -> flask.Response:
+    """Log the end of a request.
+
+    This function handles the Flask's after_request event.
+    See https://tedboy.github.io/flask/interface_src.application_object.html#flask.Flask.after_request
+
+    Additional info about the request will be in the `extra` field
+    added by `_add_request_context_info_to_log_record`
+    """
+
+    logger.info(
+        "end request",
+        extra={
+            "response.status_code": response.status_code,
+            "response.content_length": response.content_length,
+            "response.content_type": response.content_type,
+            "response.mimetype": response.mimetype,
+            "response.time_ms": (time.perf_counter() - flask.g.request_start_time) * 1000,
+        },
+    )
+    return response
+
+
+def _add_request_context_info_to_log_record(record: logging.LogRecord) -> bool:
+    """Add request context data to the log record.
+
+    If there is no request context, then do not add any data.
+    """
+    if not flask.has_request_context():
+        return True
+
+    if flask.request is None:
+        raise Exception("")
+
+    extra_log_data: dict[str, str] = getattr(flask.g, EXTRA_LOG_DATA_ATTR, {})
+    record.__dict__.update(extra_log_data)
+
+    return True
+
+
+def _add_global_context_info_to_log_record(record: logging.LogRecord) -> bool:
+    global _GLOBAL_LOG_CONTEXT
+    record.__dict__ |= _GLOBAL_LOG_CONTEXT
+
+    return True
+
+
+def _get_request_context_info(request: flask.Request) -> dict:
+    internal_request_id = str(uuid.uuid4())
+    flask.g.internal_request_id = internal_request_id
+
+    data = {
+        "request.id": request.headers.get("x-amzn-requestid", ""),
+        "request.method": request.method,
+        "request.path": request.path,
+        "request.url_rule": str(request.url_rule),
+        # This ID is used to group all logs for a given request
+        # and is returned in the API response for any 4xx/5xx scenarios
+        "request.internal_id": internal_request_id,
+    }
+
+    # Add query parameter data in the format request.query.<key> = <value>
+    # For example, the query string ?foo=bar&baz=qux would be added as
+    # request.query.foo = bar and request.query.baz = qux
+    # PII should be kept out of the URL, as URLs are logged in access logs.
+    # With that assumption, it is safe to log query parameters.
+    for key, value in request.args.items():
+        data[f"request.query.{key}"] = value
+    return data
+
+
+def _add_new_relic_context_to_log_record(record: logging.LogRecord) -> bool:
+    """Add New Relic tracing info to our log record."""
+
+    # This is not the recommended way of implementing this, but the alternatives
+    # either change the structure of our logging to not be JSON, or would
+    # entirely replace the formatter we have for outputting logs.
+    #
+    # The NewRelicContextFormatter calls this function internally when it
+    # creates the output object.
+    #
+    # This sets the following fields:
+    # entity.type
+    # entity.name
+    # entity.guid
+    # hostname
+    # span.id
+    # trace.id
+    newrelic_metadata = newrelic.api.time_trace.get_linking_metadata()
+
+    record.__dict__ |= newrelic_metadata
+
+    return True
+
+
+def _add_error_info_to_log_record(record: logging.LogRecord) -> bool:
+    """Add a shorter form of the error message to our log record."""
+    exc_info = getattr(record, "exc_info", None)
+    # exc_info is a 3-part tuple with the class, error obj, and traceback
+    if exc_info and len(exc_info) == 3:
+        # Add the exception class name to the logs, check that it
+        # is a class just in case there is some code path that sets this different.
+        if isinstance(exc_info[0], type):
+            record.__dict__["exc_info_cls"] = exc_info[0].__name__
+        # If the error were `raise ValueError("example")`, the
+        # value of this would be "ValueError('example')"
+        record.__dict__["exc_info_short"] = repr(exc_info[1])
+
+    return True
+
+
+def get_run_mode() -> str:
+    # We want to indicate whether the app was run as an API service
+    # or as a CLI - use the argv of the command we ran it with
+    # to determine that.
+    # CLI commands are always of the form "/path/to/flask <blueprint name> <task name> <commands>"
+    #
+    # The API service can be started either as
+    #  "/path/to/flask --app src.app run ..."         --> When run locally
+    #  "/api/.venv/bin/gunicorn src.app:create_app()" --> When run non-locally
+    #
+    # So we check for pieces that only appear in the API commands
+
+    _original_argv = " ".join(sys.argv)
+    run_mode = "cli"
+    if "gunicorn" in _original_argv or "--app" in _original_argv:
+        run_mode = "service"
+
+    return run_mode

grants_shared/logs/formatters.py

@@ -0,0 +1,62 @@
+"""Log formatters for the API.
+
+This module defines two formatters, JsonFormatter for machine-readable logs to
+be used in production, and HumanReadableFormatter for human readable logs to
+be used used during development.
+
+See https://docs.python.org/3/library/logging.html#formatter-objects
+"""
+
+import json
+import logging
+from datetime import datetime
+
+import grants_shared.logs.decodelog as decodelog
+from grants_shared.util.json_util import json_encoder
+
+
+class JsonFormatter(logging.Formatter):
+    """A logging formatter which formats each line as JSON."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        # logging.Formatter.format adds the `message` attribute to the LogRecord
+        # see https://github.com/python/cpython/blob/main/Lib/logging/__init__.py#L690-L720
+        super().format(record)
+
+        # New Relic automatically maps log messages of certain names over
+        # the "message" field in their system. This includes mapping "msg"
+        # which is the unformatted version of a log message. This means the
+        # formatted message doesn't make it into New Relic
+        # https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#json-logs
+        #
+        # To work around this, we copy the message field to a new field name
+        record.formatted_msg = getattr(record, "message", None)
+
+        return json.dumps(record.__dict__, separators=(",", ":"), default=json_encoder)
+
+
+HUMAN_READABLE_FORMATTER_DEFAULT_MESSAGE_WIDTH = decodelog.DEFAULT_MESSAGE_WIDTH
+
+
+class HumanReadableFormatter(logging.Formatter):
+    """A logging formatter which formats each line
+    as color-code human readable text
+    """
+
+    message_width: int
+
+    def __init__(self, message_width: int = HUMAN_READABLE_FORMATTER_DEFAULT_MESSAGE_WIDTH):
+        super().__init__()
+        self.message_width = message_width
+
+    def format(self, record: logging.LogRecord) -> str:
+        message = super().format(record)
+        return decodelog.format_line(
+            datetime.fromtimestamp(record.created),
+            record.name,
+            record.funcName,
+            record.levelname,
+            message,
+            record.__dict__,
+            message_width=self.message_width,
+        )

grants_shared/logs/pii.py

@@ -0,0 +1,97 @@
+"""Mask PII from log records.
+
+This module defines a filter that can be attached to a logger to mask PII
+from log records. The filter is applied to all log records, and masks PII
+that looks like social security numbers.
+
+You can add the filter to a handler:
+
+Example:
+    import logging
+    import grants_shared.logs.pii as pii
+
+    handler = logging.StreamHandler()
+    handler.addFilter(pii.mask_pii)
+    logger = logging.getLogger(__name__)
+    logger.addHandler(handler)
+
+Or you can add the filter directly to a logger.
+If adding the filter directly to a logger, take note that the filter
+will not be called for child loggers.
+See https://docs.python.org/3/library/logging.html#logging.Logger.propagate
+
+Example:
+    import logging
+    import grants_shared.logs.pii as pii
+
+    logger = logging.getLogger(__name__)
+    logger.addFilter(pii.mask_pii)
+"""
+
+import logging
+import re
+from typing import Any
+
+
+def mask_pii(record: logging.LogRecord) -> bool:
+    # Loop through all entries in the record's __dict__
+    # attribute and mask any things that look like PII.
+    # We will mask positional args separately below.
+    record.__dict__ |= {
+        key: _mask_pii_for_key(key, value)
+        for key, value in record.__dict__.items()
+        if key != "args"  # Handle positional "args" separately
+    }
+
+    # record.__dict__["args"] will contain positional arguments to logging calls.
+    # For example, a call like logger.info("%s %s", "foo", "bar") will result in a LogRecord
+    # with record.__dict__["args"] == ("foo", "bar")
+    # We want to mask the PII on each argument separately rather than trying to do a PII regex
+    # match on the entire args tuple.
+    args = record.__dict__["args"]
+    record.__dict__["args"] = tuple(map(_mask_pii, args))
+    return True
+
+
+# Regular expression to match a tax identifier (SSN), 9 digits with optional dashes.
+# Matches between word boundaries (\b), except when:
+#  - Preceded by word character and dash (e.g. "ip-10-11-12-134")
+#  - Preceded by or followed by a decimal point (for floating point numbers)
+TIN_RE = re.compile(
+    r"""
+        \b          # word boundary
+        (?<!\w-)    # not preceded by word character and dash
+        (?<!\.)     # not preceded by decimal point
+        (\d-?){8}   # digit then optional dash, 8 times
+        \d          # last digit
+        \b          # word boundary
+        (?!\.\d)    # not followed by decimal point and digit (for decimal numbers)
+    """,
+    re.ASCII | re.VERBOSE,
+)
+
+ALLOW_NO_MASK = {
+    "account_key",
+    "count",
+    "created",
+    "hostname",
+    "process",
+    "thread",
+}
+
+
+def _mask_pii_for_key(key: str, value: Any | None) -> Any | None:
+    """
+    Mask the given value if it has the pattern of a tax identifier
+    unless its key is one of the allowed values to avoid masking
+    something that looks like an SSN but is known to be safe (like a timestamp)
+    """
+    if key in ALLOW_NO_MASK:
+        return value
+    return _mask_pii(value)
+
+
+def _mask_pii(value: Any | None) -> Any | None:
+    if TIN_RE.search(str(value)):
+        return TIN_RE.sub("*********", str(value))
+    return value

grants_shared/util/datetime_util.py

@@ -0,0 +1,99 @@
+import re
+import zoneinfo
+from datetime import date, datetime, timezone
+
+import pytz
+
+
+def utcnow() -> datetime:
+    """Current time in UTC tagged with timezone info marking it as UTC, unlike datetime.utcnow().
+
+    See https://docs.python.org/3/library/datetime.html#datetime.datetime.utcnow
+    """
+    return datetime.now(timezone.utc)
+
+
+def adjust_timezone(timestamp: datetime, timezone_str: str) -> datetime:
+    """
+    Utility method for converting a datetime object
+    between different timezones. The string passed in
+    can be anything recognized by the pytz library
+
+    Details on how to find all the potential timezone
+    names can be found in http://pytz.sourceforge.net/#helpers
+    but a few that are likely useful include:
+    * UTC
+    * US/Eastern
+    * US/Central
+    * US/Mountain
+    * US/Pacific
+    """
+    new_timezone = pytz.timezone(timezone_str)
+    return timestamp.astimezone(new_timezone)
+
+
+def make_timezone_aware(timestamp: datetime, timezone_str: str) -> datetime:
+    new_timezone = zoneinfo.ZoneInfo(timezone_str)
+    return timestamp.replace(tzinfo=new_timezone)
+
+
+def get_now_us_eastern_datetime() -> datetime:
+    """
+    Return the current time in the eastern time zone. DST is handled based on the local time.
+    For information on handling Daylight Savings Time, refer to this documentation on now() vs. utcnow():
+    http://pytz.sourceforge.net/#problems-with-localtime
+    """
+
+    # Note that this uses Eastern time (not UTC)
+    tz = pytz.timezone("America/New_York")
+    return datetime.now(tz)
+
+
+def get_now_us_eastern_date() -> date:
+    # We get the datetime and truncate it to the date portion
+    # as there aren't any direct date methods that take in a timezone
+    return get_now_us_eastern_datetime().date()
+
+
+def datetime_str_to_date(datetime_str: str | None) -> date | None:
+    if not datetime_str:
+        return None
+    return datetime.fromisoformat(datetime_str).date()
+
+
+def parse_grants_gov_date(date_str: str | None) -> date | None:
+    """
+    Parse a date string from grants.gov SOAP API response.
+
+    Grants.gov returns dates in formats like:
+    - "2025-09-16-04:00" (with timezone suffix)
+    - "2025-09-16" (standard ISO format)
+
+    This function strips any timezone suffix and returns a date object.
+
+    Args:
+        date_str: Date string from grants.gov API
+
+    Returns:
+        date object or None if date_str is None/empty
+
+    Raises:
+        ValueError: If date_str cannot be parsed as a valid date
+    """
+    if not date_str or not date_str.strip():
+        return None
+
+    # Strip timezone suffix if present (e.g., "-04:00" or "+05:00")
+    # The pattern matches a hyphen or plus sign followed by HH:MM at the end of string
+    cleaned_date_str = re.sub(r"[+-]\d{2}:\d{2}$", "", date_str.strip())
+
+    try:
+        # Parse the cleaned date string
+        return datetime.fromisoformat(cleaned_date_str).date()
+    except ValueError as e:
+        raise ValueError(f"Could not parse date string '{date_str}': {e}") from e
+
+
+def from_timestamp(timestamp: int) -> datetime:
+    """Convert an epoch timestamp (in milliseconds) into a datetime object in timezone aware UTC."""
+    return datetime.fromtimestamp(timestamp / 1000.0, timezone.utc)

grants_shared/util/deploy_metadata.py

@@ -0,0 +1,72 @@
+import re
+import typing
+from datetime import datetime
+
+from pydantic_settings import SettingsConfigDict
+
+import grants_shared.util.datetime_util as datetime_util
+from grants_shared.util.env_config import PydanticBaseEnvConfig
+
+# We expect release notes to be formatted as:
+# YYYY-MM-DD-#
+# However we don't always put leading zeroes, so all of the following
+# would be valid release versions:
+# 2024.11.27-1
+# 2024.11.5-1
+# 2024.4.30-1
+RELEASE_NOTE_REGEX = re.compile(
+    r"""
+    ^[0-9]{4}         # Exactly 4 leading digits
+    (?:\.[0-9]{1,2})  # Period followed by 1-2 digits
+    (?:\.[0-9]{1,2})  # Period followed by 1-2 digits
+    (?:\-[0-9]{1,2})$ # Ends with a dash and 1-2 digits
+    """,
+    re.ASCII | re.VERBOSE,
+)
+
+
+class DeployMetadataConfig(PydanticBaseEnvConfig):
+    model_config = SettingsConfigDict(extra="allow")
+
+    # We don't want these values being None to break
+    # any of our system, so allow them to be None
+    deploy_github_ref: str | None = None  # DEPLOY_GITHUB_REF
+    deploy_github_sha: str | None = None  # DEPLOY_GITHUB_SHA
+    deploy_timestamp: datetime | None = None  # DEPLOY_TIMESTAMP
+    deploy_whoami: str | None = None  # DEPLOY_WHOAMI
+
+    def model_post_init(self, _context: typing.Any) -> None:
+        """Run after __init__ sets above values from env vars"""
+
+        if self.deploy_github_ref and RELEASE_NOTE_REGEX.match(self.deploy_github_ref):
+            self.release_notes = (
+                f"https://github.com/HHS/simpler-grants-gov/releases/tag/{self.deploy_github_ref}"
+            )
+        else:
+            self.release_notes = "https://github.com/HHS/simpler-grants-gov/releases"
+
+        if self.deploy_github_sha:
+            self.deploy_commit = (
+                f"https://github.com/HHS/simpler-grants-gov/commit/{self.deploy_github_sha}"
+            )
+        else:
+            self.deploy_commit = "https://github.com/HHS/simpler-grants-gov"
+
+        if self.deploy_timestamp:
+            self.deploy_datetime_est = datetime_util.adjust_timezone(
+                self.deploy_timestamp, "US/Eastern"
+            )
+        else:
+            # Just put when the API started up as a fallback
+            self.deploy_datetime_est = datetime_util.get_now_us_eastern_datetime()
+
+
+_deploy_metadata_config: DeployMetadataConfig | None = None
+
+
+def get_deploy_metadata_config() -> DeployMetadataConfig:
+    global _deploy_metadata_config
+    if _deploy_metadata_config is None:
+        _deploy_metadata_config = DeployMetadataConfig()
+
+    return _deploy_metadata_config

grants_shared/util/env_config.py

@@ -0,0 +1,16 @@
+import os
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+import grants_shared
+
+# TODO - this wouldn't make sense when used as a package - fix?
+env_file = os.path.join(
+    os.path.dirname(os.path.dirname(grants_shared.__file__)),
+    "config",
+    "%s.env" % os.getenv("ENVIRONMENT", "local"),
+)
+
+
+class PydanticBaseEnvConfig(BaseSettings):
+    model_config = SettingsConfigDict(env_file=env_file)

grants_shared/util/json_util.py

@@ -0,0 +1,60 @@
+from collections.abc import Callable
+from datetime import date, datetime
+from decimal import Decimal
+from enum import Enum
+from typing import Any
+from uuid import UUID
+
+
+# identity returns an unmodified object
+def identity[T](obj: T) -> T:
+    return obj
+
+
+# Mapping of types to functions for conversion
+# when writing logs to JSON
+ENCODERS_BY_TYPE: dict[type[Any], Callable[[Any], Any]] = {
+    # JSONEncoder handles these properly already:
+    # https://docs.python.org/3/library/json.html#json.JSONEncoder
+    str: identity,
+    int: identity,
+    float: identity,
+    bool: identity,
+    list: identity,
+    datetime: lambda d: d.isoformat(),
+    date: lambda d: d.isoformat(),
+    Enum: lambda e: e.value,
+    set: lambda s: list(s),
+    # The fallback below would do these,
+    # but making it explicit that these
+    # types are supported for logging.
+    Decimal: str,
+    UUID: str,
+    Exception: str,
+}
+
+
+def json_encoder(obj: Any) -> Any:
+    """
+    Handle conversion of various types when logs
+    are serialized into JSON. If not specified
+    will attempt to convert using str() on the object
+    """
+
+    _type = type(obj)
+    encode = ENCODERS_BY_TYPE.get(_type, str)
+
+    """
+    The recommended approach from the JSON docs
+    is to call the default method from JSONEncoder
+    to allow it to error anything not defined, we
+    choose not to do that as we want to give a best
+    effort for every value to be serialized for the logs
+    https://docs.python.org/3/library/json.html
+
+    If a field you are trying to log doesn't make sense
+    to format as a string then please add it above, but be
+    aware that the format needs to be parseable by whatever
+    tools you are using to ingest logs and metrics.
+    """
+    return encode(obj)

grants_shared/util/local.py

@@ -0,0 +1,31 @@
+import logging
+import os
+
+from dotenv import load_dotenv
+
+logger = logging.getLogger(__name__)
+
+
+def load_local_env_vars(env_file: str = "local.env") -> None:
+    """
+    Load environment variables from the local.env so
+    that they can be fetched with `os.getenv()` or with
+    other utils that pull env vars.
+
+    https://pypi.org/project/python-dotenv/
+
+    NOTE: any existing env vars will not be overriden by this
+    """
+    environment = os.getenv("ENVIRONMENT", None)
+
+    # If the environment is explicitly local or undefined
+    # we'll use the dotenv file, otherwise we'll skip
+    # Should never run if not local development
+    if environment is None or environment == "local":
+        load_dotenv(env_file)
+
+
+def error_if_not_local() -> None:
+    if (env := os.getenv("ENVIRONMENT")) != "local":
+        logger.error("Environment %s is not local - cannot run operation", env)
+        raise Exception("Local-only process called when environment was set to non-local")

grants_shared-0.1.0.dist-info/METADATA

@@ -0,0 +1,67 @@
+Metadata-Version: 2.4
+Name: grants-shared
+Version: 0.1.0
+Summary: Shared code used by the Simpler Grants.gov repo
+Author-email: Nava Engineering <engineering@navapbc.com>
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: <3.15,>=3.14
+Description-Content-Type: text/markdown
+Requires-Dist: apiflask<4,>=3.1.0
+Requires-Dist: marshmallow<4,>=3.20.1
+Requires-Dist: pydantic<3,>=2.13.3
+Requires-Dist: pydantic-settings<3,>=2.14.0
+Requires-Dist: sqlalchemy[mypy]<3,>=2.0.49
+Requires-Dist: psycopg[binary]<4,>=3.3.4
+Requires-Dist: botocore<2,>=1.43.3
+Requires-Dist: boto3<2,>=1.43.3
+Requires-Dist: smart-open<8,>=7.6.0
+Requires-Dist: pytz<2027,>=2026.2
+Requires-Dist: pyjwt[crypto]<3,>=2.12.1
+Requires-Dist: jsonschema[format-nongpl]<5,>=4.26.0
+Requires-Dist: jsonpath-ng<2,>=1.8.0
+Requires-Dist: jsonref<2,>=1.1.0
+Requires-Dist: pandas<3,>=2.0.3
+Requires-Dist: pandas-stubs<3,>=2.0.3
+Requires-Dist: newrelic<13,>=12.1.0
+Requires-Dist: python-dotenv<2,>=1.2.2
+
+# Grants Shared
+
+This repo exists to contain the shared code used by the backend
+of simpler.grants.gov which is made up of multiple backend services.
+
+This code is not meant to be used outside of the [Simpler Grants](https://github.com/HHS/simpler-grants-gov) system. 
+
+[License](https://github.com/HHS/simpler-grants-gov/blob/main/LICENSE.md)
+
+## Installation
+TODO - this code isn't yet in PyPi, so this won't actually work yet.
+Will update instructions more thoroughly once it is available.
+
+```shell
+# Using pip
+pip install grants_shared
+
+# Using poetry
+poetry add grants_shared
+
+# Using uv
+uv add grants_shared
+```
+
+## Usage
+Guidance on common commands and running the application will come in later
+versions as we're still getting this setup, but a few basic commands to get you started.
+
+```shell
+# Build the docker image
+make build
+
+# Run tests
+make test
+
+# Formatting and linting
+make format
+make lint
+```

grants_shared-0.1.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+grants_shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+grants_shared/logs/__init__.py,sha256=7lgGlUEwJpjyPl0TWkAi3PFxt6Wq4flIBVLue2E1bJM,914
+grants_shared/logs/audit.py,sha256=gK8baMR5s4Q1iamnvMoTuZwR8aFRREcBHC5GOdB8CvA,4699
+grants_shared/logs/config.py,sha256=Wd3x08SJEP05pfCXleBZ_ajDTxkJxKcV5cr8NOOgAog,6457
+grants_shared/logs/decodelog.py,sha256=SMAqS2TyMtyDNMhhBQUPVMXtLo4KxosBhW4XD-nxrXk,3999
+grants_shared/logs/flask_logger.py,sha256=Z9QDPilE_EEv_u4VbcYYmnggQsH_Se8-JNzp-r0nukg,9454
+grants_shared/logs/formatters.py,sha256=0tGFFAekLO_nuYf_SLahEHvq-73SiKVBC4gVnY-SbUo,2231
+grants_shared/logs/pii.py,sha256=5PAlWjNKQWUMkLPJiewRDbfgVLvFEv71Hg5H8XTtgFs,3131
+grants_shared/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+grants_shared/util/datetime_util.py,sha256=h9o_w916oP1nJQTY4oTJqbxyYccpBeEEYI2xXuPNA9s,3229
+grants_shared/util/deploy_metadata.py,sha256=l6CnVJZe9VSrUJI1_EhXgr_JGtcvuNV2sWQsqRnMnEg,2534
+grants_shared/util/env_config.py,sha256=heyDp8bUj-lfTAtCOrEZOQd_x2prN20mUBpdeAyIHY8,416
+grants_shared/util/json_util.py,sha256=01BpkldEQATDChIt5MQQgdDWjF38KtiwegmAPqsK34o,1760
+grants_shared/util/local.py,sha256=n4uO2-oSH1C5xmuAHun777VJbW493rmujYOCL4B0uSk,982
+grants_shared-0.1.0.dist-info/METADATA,sha256=H59-cQQe8ipflBw2gFBYZsRUAIIz9XzqNXYusY3WwYU,1934
+grants_shared-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+grants_shared-0.1.0.dist-info/top_level.txt,sha256=5xACZ9ZOtI4Vg-LU8zhoCSc55ITPn3bFQ-36oIilRWM,14
+grants_shared-0.1.0.dist-info/RECORD,,

grants_shared-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

grants_shared-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+grants_shared