grants-shared 0.1.0__tar.gz

grants_shared-0.1.0/PKG-INFO

@@ -0,0 +1,67 @@
+Metadata-Version: 2.4
+Name: grants-shared
+Version: 0.1.0
+Summary: Shared code used by the Simpler Grants.gov repo
+Author-email: Nava Engineering <engineering@navapbc.com>
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: <3.15,>=3.14
+Description-Content-Type: text/markdown
+Requires-Dist: apiflask<4,>=3.1.0
+Requires-Dist: marshmallow<4,>=3.20.1
+Requires-Dist: pydantic<3,>=2.13.3
+Requires-Dist: pydantic-settings<3,>=2.14.0
+Requires-Dist: sqlalchemy[mypy]<3,>=2.0.49
+Requires-Dist: psycopg[binary]<4,>=3.3.4
+Requires-Dist: botocore<2,>=1.43.3
+Requires-Dist: boto3<2,>=1.43.3
+Requires-Dist: smart-open<8,>=7.6.0
+Requires-Dist: pytz<2027,>=2026.2
+Requires-Dist: pyjwt[crypto]<3,>=2.12.1
+Requires-Dist: jsonschema[format-nongpl]<5,>=4.26.0
+Requires-Dist: jsonpath-ng<2,>=1.8.0
+Requires-Dist: jsonref<2,>=1.1.0
+Requires-Dist: pandas<3,>=2.0.3
+Requires-Dist: pandas-stubs<3,>=2.0.3
+Requires-Dist: newrelic<13,>=12.1.0
+Requires-Dist: python-dotenv<2,>=1.2.2
+
+# Grants Shared
+
+This repo exists to contain the shared code used by the backend
+of simpler.grants.gov which is made up of multiple backend services.
+
+This code is not meant to be used outside of the [Simpler Grants](https://github.com/HHS/simpler-grants-gov) system. 
+
+[License](https://github.com/HHS/simpler-grants-gov/blob/main/LICENSE.md)
+
+## Installation
+TODO - this code isn't yet in PyPi, so this won't actually work yet.
+Will update instructions more thoroughly once it is available.
+
+```shell
+# Using pip
+pip install grants_shared
+
+# Using poetry
+poetry add grants_shared
+
+# Using uv
+uv add grants_shared
+```
+
+## Usage
+Guidance on common commands and running the application will come in later
+versions as we're still getting this setup, but a few basic commands to get you started.
+
+```shell
+# Build the docker image
+make build
+
+# Run tests
+make test
+
+# Formatting and linting
+make format
+make lint
+```

grants_shared-0.1.0/README.md

@@ -0,0 +1,39 @@
+# Grants Shared
+
+This repo exists to contain the shared code used by the backend
+of simpler.grants.gov which is made up of multiple backend services.
+
+This code is not meant to be used outside of the [Simpler Grants](https://github.com/HHS/simpler-grants-gov) system. 
+
+[License](https://github.com/HHS/simpler-grants-gov/blob/main/LICENSE.md)
+
+## Installation
+TODO - this code isn't yet in PyPi, so this won't actually work yet.
+Will update instructions more thoroughly once it is available.
+
+```shell
+# Using pip
+pip install grants_shared
+
+# Using poetry
+poetry add grants_shared
+
+# Using uv
+uv add grants_shared
+```
+
+## Usage
+Guidance on common commands and running the application will come in later
+versions as we're still getting this setup, but a few basic commands to get you started.
+
+```shell
+# Build the docker image
+make build
+
+# Run tests
+make test
+
+# Formatting and linting
+make format
+make lint
+```

grants_shared-0.1.0/pyproject.toml

@@ -0,0 +1,228 @@
+[project]
+name = "grants-shared"
+version = "0.1.0"
+description = "Shared code used by the Simpler Grants.gov repo"
+readme = "README.md"
+authors = [{ name = "Nava Engineering", email = "engineering@navapbc.com" }]
+requires-python = ">=3.14,<3.15"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.14",
+]
+
+dependencies = [
+    "apiflask>=3.1.0,<4",
+    "marshmallow>=3.20.1,<4",
+    "pydantic>=2.13.3,<3",
+    "pydantic-settings>=2.14.0,<3",
+    "sqlalchemy[mypy]>=2.0.49,<3",
+    "psycopg[binary]>=3.3.4,<4",
+    "botocore>=1.43.3,<2",
+    "boto3>=1.43.3,<2",
+    "smart-open>=7.6.0,<8",
+    "pytz>=2026.2,<2027",
+    "pyjwt[crypto]>=2.12.1,<3",
+    "jsonschema[format-nongpl]>=4.26.0,<5",
+    "jsonpath-ng>=1.8.0,<2",
+    "jsonref>=1.1.0,<2",
+    "pandas>=2.0.3,<3",
+    "pandas-stubs>=2.0.3,<3",
+    "newrelic>=12.1.0,<13",
+    "python-dotenv>=1.2.2,<2",
+]
+
+[dependency-groups]
+dev = [
+    "black>=26.3.1,<27",
+    "isort>=8.0.1,<9",
+    "moto[s3]>=5.2.0,<6",
+    "mypy>=1.20.2,<2",
+    "coverage>=7.13.5,<8",
+    "faker>=40.15.0,<41",
+    "factory-boy>=3.3.3,<4",
+    "bandit>=1.9.4,<2",
+    "pytest>=9.0.3,<10",
+    "ruff>=0.15.12,<16",
+    "freezegun>=1.5.5,<2",
+    "debugpy>=1.8.20,<2",
+    "types-requests>=2.33.0.20260503",
+]
+
+
+[tool.hatch.build.targets.sdist]
+include = ["src"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/grants_shared"]
+
+[tool.black]
+line-length = 100
+
+[tool.isort]
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+line_length = 100
+
+[tool.ruff]
+line-length = 100
+# Some rules are considered preview-only, this allows them
+# assuming we enabled them below
+preview = true
+
+target-version = "py314"
+
+[tool.ruff.lint]
+# See: https://docs.astral.sh/ruff/rules/ for all possible rules
+select = [
+    "B", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
+    "C",
+    "E", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
+    "F", # https://docs.astral.sh/ruff/rules/#pyflakes-f
+    "W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
+    "UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up
+    "RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
+    "PT", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt
+    "TID251", # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid
+    "T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20
+]
+ignore = [
+    # too many leading '#' for block comment, we can format our comments however we want
+    "E266",
+    # Ignore line-too-long errors, assume the formatter handles that appropriately
+    "E501",
+    # Ignore rules regarding unecessary list / generator usage which complains about [e for e in MyEnum] #
+    "C4",
+    # Ignore rule that flags functions with many branches - sometimes we just have a lot of
+    # business rules that make sense to aggregate in one place.
+    "C901",
+    # Ruff suggests not doing .encode("utf-8") and leaving utf-8 (the default out),
+    # but nothing wrong with being explicit and clear
+    "UP012",
+    # Ruff suggests using datetime.UTC over datetime.timezone.utc - but other timezones
+    # would need to be specified the latter way, seems odd to intentionally be inconsistent
+    "UP017",
+    # Ruff suggests using f-strings over .format - this one is a good recommendation
+    # we just have a few too many to refactor at this time.
+    "UP031",
+    # Ruff doesn't like array concatenation like [1, 2, 3] + [4, 5, 6], but it's intuitive where used.
+    "RUF005",
+    # Ruff doesn't like str() in f-strings, but instead
+    # recommends conversion flags (eg. f"{a!r}") which is less well known
+    "RUF010",
+    # Ruff thinks our SQLAlchemy models and factories have ClassVars
+    # but those classes don't use those variables in the problematic way it wants to avoid
+    "RUF012",
+    # Ruff wants __all__ sorted, not against it, but we'd want our
+    # formatter to handle that first to avoid it being tedious work
+    "RUF022",
+    # Ruff doesn't like when you use variables with _ prefixes in functions
+    # saying to avoid shadowing other params to do my_var_, but while
+    # that follows PEP8 formatting, it's not generally what I've seen
+    "RUF052",
+    # Ruff doesn't want any code in __init__ files, but some of our
+    # libraries and patterns need a very small amount of code configured
+    # in these files.
+    "RUF067",
+]
+
+# These are characters we are allowing to be confusing
+# for RUF001 which recommends not using certain characters
+# that look like each other.
+# https://docs.astral.sh/ruff/rules/ambiguous-unicode-character-string/#ambiguous-unicode-character-string-ruf001
+allowed-confusables = [
+    # endash (not a regular dash) - we need this in our email formatting
+    "–",
+    # right quotation mark (not a simple "'") - used in email formatting
+    "’"
+]
+
+[tool.ruff.lint.per-file-ignores]
+# These are rules that are excluded from just our unit tests
+# but still run for the rest of our code.
+"tests/*" = [
+    # Ruff suggests changing how iterables are merged, but it complicates test setup
+    "RUF005",
+    # Ruff wants to avoid "arg: int = None" - we shouldn't do this
+    # but our tests are only partially typed
+    "RUF013",
+    # Ruff suggests making any match statement in pytest.raises
+    # a proper regex pattern, but we just use it for finding strings in most cases
+    "RUF043",
+    # Ruff doesn't like "x, y = get_tuple()" and not using one of the values
+    # but the value might be used by a developer doing debugging
+    "RUF059",
+    # Ruff recommends the first parameter of a parametrized test be a tuple
+    # We have a lot of tests that don't do that, can circle back to this later
+    "PT006",
+    # Ruff recommends having pytest.raises() be a specific error
+    # but sometimes we just want to verify an error was raised at all
+    "PT011",
+    # Ruff recommends not doing "assert x == 1 and y == 2" in tests
+    # but the few places we do this are kept simple and help better organize complex scenarios
+    "PT018"
+]
+
+[tool.ruff.lint.flake8-tidy-imports.banned-api]
+# For these libraries, we do use them, but have our own derived versions we want
+# to use instead as we've rewritten fundamental components like error messaging.
+"marshmallow.validate".msg = "Do not import marshmallow.validate directly - instead use grants_shared.api.schemas.extension.validators as we've rewritten the error messaging"
+"apiflask.validators".msg = "Do not import apiflask.validators directly - instead use grants_shared.api.schemas.extension.validators as we've rewritten the error messaging"
+"apiflask.fields".msg = "Do not import apiflask.fields directly - instead use grants_shared.api.schemas.extension.fields as we've rewritten the error messaging"
+"apiflask.Schema".msg = "Do not import apiflask.Schema directly - instead use grants_shared.api.schemas.extension"
+
+[tool.mypy]
+# https://mypy.readthedocs.io/en/stable/config_file.html
+color_output = true
+error_summary = true
+pretty = true
+show_error_codes = true
+show_column_numbers = true
+show_error_context = true
+
+namespace_packages = true
+ignore_missing_imports = true
+warn_unused_configs = true
+
+check_untyped_defs = true
+disallow_incomplete_defs = true
+disallow_untyped_defs = true
+no_implicit_optional = true
+strict_equality = true
+warn_no_return = true
+warn_redundant_casts = true
+warn_unreachable = true
+warn_unused_ignores = true
+
+plugins = ["pydantic.mypy"]
+
+[tool.bandit]
+# Ignore audit logging test file since test audit logging requires a lot of operations that trigger bandit warnings
+exclude_dirs = ["./tests/grants_shared/logs/test_audit.py"]
+
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+
+markers = [
+  "audit: mark a test as a security audit log test, to be run isolated from other tests",
+]
+
+[tool.coverage.run]
+omit = [
+  # Decodelog is only used for formatting logs locally
+  "src/grants_shared/logs/decodelog.py"
+]
+
+[tool.coverage.report]
+fail_under = 80
+
+exclude_lines = [
+  # Exclude abstract & overloaad methods from
+  # code coverage reports as they won't ever directly run
+  "@abc.abstractmethod",
+  "@abstractmethod",
+  "@typing.overload",
+]

grants_shared-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

grants_shared-0.1.0/src/grants_shared/logs/__init__.py

@@ -0,0 +1,31 @@
+"""Module for initializing logging configuration for the application.
+
+There are two formatters for the log messages: human-readable and JSON.
+The formatter that is used is determined by the environment variable
+LOG_FORMAT. If the environment variable is not set, the JSON formatter
+is used by default. See grants_shared.logs.formatters for more information.
+
+The logger also adds a PII mask filter to the root logger. See
+grants_shared.logs.pii for more information.
+
+Usage:
+    import grants_shared.logs
+
+    with grants_shared.logs.init("program name"):
+        ...
+
+Once the module has been initialized, the standard logging module can be
+used to log messages:
+
+Example:
+    import logging
+
+    logger = logging.getLogger(__name__)
+    logger.info("message")
+"""
+
+import grants_shared.logs.config as config
+
+
+def init(program_name: str) -> config.LoggingContext:
+    return config.LoggingContext(program_name)

grants_shared-0.1.0/src/grants_shared/logs/audit.py

@@ -0,0 +1,129 @@
+#
+# Application-level audit logging.
+#
+# See https://docs.python.org/3/library/audit_events.html
+# https://docs.python.org/3/library/sys.html#sys.addaudithook
+# https://www.python.org/dev/peps/pep-0578/
+#
+import collections
+import logging
+import sys
+from collections.abc import Hashable, Sequence
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+AUDIT = 32
+logging.addLevelName(AUDIT, "AUDIT")
+
+
+def init() -> None:
+    """Initialize the audit logging module to start
+    logging security audit events."""
+    sys.addaudithook(handle_audit_event)
+
+
+def handle_audit_event(event_name: str, args: tuple[Any, ...]) -> None:
+    # Define events to log and the arguments to log for each event.
+    # For more information about these events and what they mean, see https://peps.python.org/pep-0578/#suggested-audit-hook-locations
+    # For the full list of auditable events, see https://docs.python.org/3/library/audit_events.html
+    # Define this variable locally so it can't be modified by other modules.
+
+    EVENTS_TO_LOG = {
+        # Detect dynamic execution of code objects. This only occurs for explicit
+        # calls, and is not raised for normal function invocation.
+        "exec": ("code_object",),
+        # Detect when a file is about to be opened. path and mode are the usual
+        # parameters to open if available, while flags is provided instead of
+        # mode in some cases.
+        "open": ("path", "mode", "flags"),
+        # Detect when a signal is sent to a process.
+        "os.kill": ("pid", "sig"),
+        # Detect when a file is renamed.
+        "os.rename": ("src", "dst", "src_dir_fd", "dst_dir_fd"),
+        # Detect when a subprocess is started.
+        "subprocess.Popen": ("executable", "args", "cwd", "_"),
+        # Detect access to network resources. The address is unmodified from the original call.
+        "socket.connect": ("socket", "address"),
+        "socket.getaddrinfo": ("host", "port", "family", "type", "protocol"),
+        # Detect when new audit hooks are being added.
+        "sys.addaudithook": (),
+        # Detects URL requests.
+        # Don't log data or headers because they may contain sensitive information.
+        "urllib.Request": ("url", "_", "_", "method"),
+    }
+
+    if event_name not in EVENTS_TO_LOG:
+        return
+
+    arg_names = EVENTS_TO_LOG[event_name]
+    log_audit_event(event_name, args, arg_names)
+
+
+# Set the audit hook to be traceable so that coverage module can track calls to it
+# The coverage module relies on Python's trace hooks
+# (See https://coverage.readthedocs.io/en/7.1.0/howitworks.html#execution)
+# According to the docs for sys.addaudithook, the audit hook is only traced if the callable
+# has a __cantrace__ member that is set to a true value.
+# (See https://docs.python.org/3/library/sys.html#sys.addaudithook)
+handle_audit_event.__cantrace__ = True  # type: ignore
+
+
+def log_audit_event(event_name: str, args: Sequence[Any], arg_names: Sequence[str]) -> None:
+    """Log a message but only log recently repeated messages at intervals."""
+    extra = {
+        f"audit.args.{arg_name}": arg
+        for arg_name, arg in zip(arg_names, args, strict=True)
+        if arg_name != "_"
+    }
+
+    key = (event_name, repr(args))
+    if key not in audit_message_count:
+        count = 1
+    else:
+        count = audit_message_count[key] + 1
+    audit_message_count[key] = count
+
+    if count > 100 and count % 100 != 0:
+        return
+
+    if count > 10 and count % 10 != 0:
+        return
+
+    extra["count"] = count
+
+    logger.log(AUDIT, event_name, extra=extra)
+
+
+class LeastRecentlyUsedDict(collections.OrderedDict):
+    """A dict with a maximum size, evicting the least recently written key when full.
+
+    Getting a key that is not present returns a default value of 0.
+
+    Setting a key marks it as most recently used and removes the oldest key if full.
+
+    May be useful for tracking the count of items where limited memory usage is needed even if
+    the set of items can be unlimited.
+
+    Based on the example at
+    https://docs.python.org/3/library/collections.html#ordereddict-examples-and-recipes
+    """
+
+    def __init__(self, maxsize: int = 128, *args: Any, **kwargs: Any) -> None:
+        self.maxsize = maxsize
+        super().__init__(*args, **kwargs)
+
+    def __getitem__(self, key: Hashable) -> int:
+        if key in self:
+            return super().__getitem__(key)
+        return 0
+
+    def __setitem__(self, key: Hashable, value: int) -> None:
+        if key in self:
+            self.move_to_end(key)
+        super().__setitem__(key, value)
+        if self.maxsize < len(self):
+            self.popitem(last=False)
+
+
+audit_message_count = LeastRecentlyUsedDict()

grants_shared-0.1.0/src/grants_shared/logs/config.py

@@ -0,0 +1,165 @@
+import contextlib
+import logging
+import os
+import platform
+import pwd
+import sys
+from typing import Any, cast
+
+from pydantic_settings import SettingsConfigDict
+
+import grants_shared.logs.audit
+import grants_shared.logs.formatters as formatters
+import grants_shared.logs.pii as pii
+from grants_shared.util.env_config import PydanticBaseEnvConfig
+
+logger = logging.getLogger(__name__)
+
+_original_argv = tuple(sys.argv)
+
+
+class HumanReadableFormatterConfig(PydanticBaseEnvConfig):
+    message_width: int = formatters.HUMAN_READABLE_FORMATTER_DEFAULT_MESSAGE_WIDTH
+
+
+class LoggingConfig(PydanticBaseEnvConfig):
+    model_config = SettingsConfigDict(env_prefix="log_", env_nested_delimiter="__")
+
+    format: str = "json"
+    level: str = "INFO"
+    enable_audit: bool = False
+    human_readable_formatter: HumanReadableFormatterConfig = HumanReadableFormatterConfig()
+
+    # Specify logging_level_overrides formatted as "<logger>=<level>" like "newrelic=INFO,something.else=ERROR"
+    level_overrides: str | None = None
+
+
+class LoggingContext(contextlib.AbstractContextManager[None]):
+    """
+    A context manager for handling setting up the logging stream.
+
+    To help facillitate being able to test logging, we need to be able
+    to easily create temporary output streams and then tear them down.
+
+    When this context manager is torn down, the stream handler created
+    with it will be removed.
+
+    For example:
+    ```py
+    import logging
+
+    logger = logging.getLogger(__name__)
+
+    with LoggingContext("example_program_name"):
+        # This log message will go to stdout
+        logger.info("example log message")
+
+    # This log message won't go to stdout as the
+    # handler will have been removed
+    logger.info("example log message")
+    ```
+    Note that any other handlers added to the root logger won't be affected
+    and calling this multiple times before exit would result in duplicate logs.
+    """
+
+    def __init__(self, program_name: str) -> None:
+        self._configure_logging()
+        log_program_info(program_name)
+
+    def __enter__(self) -> None:
+        pass
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        # Remove the console handler to stop logs from being sent to stdout
+        # This is useful in the test suite, since multiple tests may initialize
+        # separate duplicate handlers. This allows for easier cleanup for each
+        # of those tests.
+        logging.root.removeHandler(self.console_handler)
+
+    def _configure_logging(self) -> None:
+        """Configure logging for the application.
+
+        Configures the root module logger to log to stdout.
+        Adds a PII mask filter to the root logger.
+        Also configures log levels third party packages.
+        """
+        config = LoggingConfig()
+
+        # Loggers can be configured using config functions defined
+        # in logging.config or by directly making calls to the main API
+        # of the logging module (see https://docs.python.org/3/library/logging.config.html)
+        # We opt to use the main API using functions like `addHandler` which is
+        # non-destructive, i.e. it does not overwrite any existing handlers.
+        # In contrast, logging.config.dictConfig() would overwrite any existing loggers.
+        # This is important during testing, since fixtures like `caplog` add handlers that would
+        # get overwritten if we call logging.config.dictConfig() during the scope of the test.
+        self.console_handler = logging.StreamHandler(sys.stdout)
+        formatter = get_formatter(config)
+        self.console_handler.setFormatter(formatter)
+        self.console_handler.addFilter(pii.mask_pii)
+        logging.root.addHandler(self.console_handler)
+        logging.root.setLevel(config.level)
+
+        if config.enable_audit:
+            grants_shared.logs.audit.init()
+
+        # Configure loggers for third party packages
+        logging.getLogger("alembic").setLevel(logging.INFO)
+        logging.getLogger("werkzeug").setLevel(logging.WARN)
+        logging.getLogger("sqlalchemy.pool").setLevel(logging.INFO)
+        logging.getLogger("sqlalchemy.dialects.postgresql").setLevel(logging.INFO)
+
+        # Allow an env var to override logging config, mostly for development purposes
+        # Parsing string formatted like "logger1=INFO,logger2=ERROR"
+        if config.level_overrides is not None:
+            for override in config.level_overrides.split(","):
+                logger_override, level_override = override.split("=")
+                logging.getLogger(logger_override).setLevel(level_override)
+
+
+def get_formatter(config: LoggingConfig) -> logging.Formatter:
+    """Return the formatter used by the root logger.
+
+    The formatter is determined by the environment variable LOG_FORMAT. If the
+    environment variable is not set, the JSON formatter is used by default.
+    """
+    if config.format == "human-readable":
+        return get_human_readable_formatter(config.human_readable_formatter)
+    return formatters.JsonFormatter()
+
+
+def log_program_info(program_name: str) -> None:
+    logger.info(
+        "start %s: %s %s %s, hostname %s, pid %i, user %i(%s)",
+        program_name,
+        platform.python_implementation(),
+        platform.python_version(),
+        platform.system(),
+        platform.node(),
+        os.getpid(),
+        os.getuid(),
+        pwd.getpwuid(os.getuid()).pw_name,
+        extra={
+            "hostname": platform.node(),
+            "cpu_count": os.cpu_count(),
+            # If mypy is run on a mac, it will throw a module has no attribute error, even though
+            # we never actually access it with the conditional.
+            #
+            # However, we can't just silence this error, because on linux (e.g. CI/CD) that will
+            # throw an unused “type: ignore” comment error. Casting to Any instead ensures this
+            # passes regardless of where mypy is being run
+            "cpu_usable": (
+                len(cast(Any, os).sched_getaffinity(0))
+                if "sched_getaffinity" in dir(os)
+                else "unknown"
+            ),
+        },
+    )
+    logger.info("invoked as: %s", " ".join(_original_argv))
+
+
+def get_human_readable_formatter(
+    config: HumanReadableFormatterConfig,
+) -> formatters.HumanReadableFormatter:
+    """Return the human readable formatter used by the root logger."""
+    return formatters.HumanReadableFormatter(message_width=config.message_width)