pjdev-sn-sdk 4.6.4__py3-none-any.whl

pjdev_sn_sdk/__about__.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2026-present Chris O'Neill <chris@purplejay.io>
+#
+# SPDX-License-Identifier: MIT
+__version__ = "4.6.4"

pjdev_sn_sdk/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2026-present Chris O'Neill <chris@purplejay.io>
+#
+# SPDX-License-Identifier: MIT

pjdev_sn_sdk/api_utilities.py

@@ -0,0 +1,161 @@
+import asyncio
+import time
+from functools import wraps
+from typing import Optional, List, Any
+
+import httpx
+from httpx import HTTPStatusError
+from loguru import logger
+
+
+async def log_server_addr(response):
+    network_stream_ext = response.extensions["network_stream"]
+    logger.debug(f"Client IP, Port: {network_stream_ext.get_extra_info('client_addr')}")
+    logger.debug(f"Server IP, Port: {network_stream_ext.get_extra_info('server_addr')}")
+
+
+async def log_request_headers(request):
+    header_keys = [k for k in request.headers.keys()]
+    logger.debug("Request event hook - Headers: " + " | ".join(header_keys))
+    logger.debug(
+        f"Request event hook: {request.method} {request.url} - Waiting for response"
+    )
+
+
+async def log_response_headers(response):
+    request = response.request
+    header_keys = [k for k in response.headers.keys()]
+    logger.debug("Request event hook - Headers: " + " | ".join(header_keys))
+    logger.debug(
+        f"Response event hook: Content-Type -> {response.headers.get('content-type')}"
+    )
+    logger.debug(
+        f"Response event hook: Content-encoding -> {response.headers.get('content-encoding')}"
+    )
+    logger.debug(
+        f"Response event hook: transfer-encoding -> {response.headers.get('transfer-encoding')}"
+    )
+    logger.debug(f"Response event hook: Server -> {response.headers.get('server')}")
+    logger.debug(
+        f"Response event hook: {request.method} {request.url} - Status {response.status_code}"
+    )
+
+
+def get_error_message(eg: ExceptionGroup) -> str:
+    error_messages = []
+    for e in eg.exceptions:
+        if isinstance(e, httpx.HTTPStatusError):
+            error_messages.append(f"{e.response.status_code} -> " + e.response.text)
+        else:
+            error_messages.append(str(e))
+    error = " | ".join(f"{e}" for e in error_messages)
+    logger.error(error)
+
+    return error
+
+
+def retry_http(max_attempts: int, delay_seconds: int):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            attempts = 0
+            while attempts < max_attempts:
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    attempts += 1
+                    logger.warning(e)
+                    logger.warning(f"Attempt {attempts}/{max_attempts} failed: {e}")
+                    time.sleep(delay_seconds**attempts)
+            raise Exception(f"Failed after {max_attempts} attempts")
+
+        return wrapper
+
+    return decorator
+
+
+def async_retry_http(
+    max_attempts: int,
+    delay_seconds: int,
+    default_value: Optional[Any] = None,
+    status_codes_to_ignore: Optional[List[int]] = None,
+):
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            attempts = 0
+            exceptions = []
+
+            while attempts < max_attempts:
+                try:
+                    return await func(*args, **kwargs)
+                except HTTPStatusError as e:
+                    logger.warning(e)
+                    logger.warning(
+                        f"{e.response.status_code}: {e.response.reason_phrase}"
+                    )
+                    logger.warning(f"{e.response.text}")
+                    exceptions.append(e)
+                    if (
+                        status_codes_to_ignore
+                        and e.response.status_code in status_codes_to_ignore
+                    ):
+                        break
+                except Exception as e:
+                    logger.warning("unexpected exception")
+                    logger.warning(e)
+                    exceptions.append(e)
+
+                attempts += 1
+                if attempts == max_attempts:
+                    break
+                logger.warning(f"Attempt {attempts}/{max_attempts} failed")
+                total_delay = delay_seconds**attempts
+                logger.warning(f"Retrying in {total_delay} seconds...")
+                await asyncio.sleep(total_delay)
+            if default_value is None:
+                raise ExceptionGroup(
+                    f"Failed after {max_attempts} attempts", exceptions
+                )
+            logger.error(f"Failed after {max_attempts} attempts")
+            return default_value
+
+        return wrapper
+
+    return decorator
+
+
+def record_time(log: bool = False):
+    def decorator(func):
+        if asyncio.iscoroutinefunction(func):
+
+            @wraps(func)
+            async def wrap_func(*args, **kwargs):
+                t1 = time.time()
+                result = await func(*args, **kwargs)
+                t2 = time.time()
+                if log:
+                    logger.info(
+                        f"Function {func.__name__!r} executed in {(t2 - t1):.2f}s"
+                    )
+                return result
+
+            return wrap_func
+
+        else:
+
+            @wraps(func)
+            def wrap_func(*args, **kwargs):
+                t1 = time.time()
+                result = func(*args, **kwargs)
+                t2 = time.time()
+                if log:
+                    logger.info(
+                        f"Function {func.__name__!r} executed in {(t2 - t1):.2f}s"
+                    )
+
+                return result
+
+            return wrap_func
+
+    return decorator

pjdev_sn_sdk/config_service.py

@@ -0,0 +1,19 @@
+from pathlib import Path
+from typing import Optional
+
+from pjdev_sn_sdk.models import Config
+
+__ctx = {}
+
+
+def get_config() -> Config:
+    return __ctx["config"]
+
+
+async def init(env_path: Optional[Path] = None) -> None:
+    Config.model_config.update(env_file=env_path)
+    __ctx["config"] = Config()
+
+    if __ctx["config"].output_path is None:
+        __ctx["config"].output_path = env_path / "output"
+

pjdev_sn_sdk/control_parsing_service.py

@@ -0,0 +1,98 @@
+import re
+from typing import Optional
+
+from loguru import logger
+
+
+def extract_control(ap: Optional[str]) -> Optional[str]:
+    """
+    Extract the NIST 800-53 control_id from an assessment procedure (ap) value.
+
+    What it does:
+    - Accepts a wide variety of AP formats (case-insensitive).
+    - Removes [r5] revision markers (wherever they appear).
+    - Parses the control family (e.g., AC) and control number (e.g., 2), removing leading zeros.
+    - Captures only the immediate numeric enhancement in parentheses (e.g., (3)).
+    - Ignores objective letters like (a), (b), roman numerals, dot suffixes like .01, and other notes.
+
+    Examples:
+    - "AC-2(3)(a)"        -> "AC-02(03)"
+    - "[r5] ac-02 (03)"   -> "AC-02(03)"
+    - "PM-16 (01) (b)"    -> "PM-16(01)"
+    - "RA-5(5)(A)(1)"     -> "RA-05(05)"
+    - "SC-7 (12).01(a)"   -> "SC-07(12)"
+    - "AU-3"              -> "AU-03"
+    - "invalid"           -> None
+
+    :param ap: The assessment procedure string.
+    :return: The related control_id (e.g., 'AC-2' or 'AC-2(3)') or None if it can't be parsed.
+    """
+    if not ap:
+        return None
+
+    s = ap.strip()
+    if not s:
+        return None
+
+    # Remove revision markers like [r5] anywhere in the string (case-insensitive).
+    s = re.sub(r"\[\s*r?\s*5\s*\]", "", s, flags=re.IGNORECASE)
+
+    # Match the base control family and number (e.g., AC-2, CM-06, etc.)
+    # NOTE: Use a negative lookahead to avoid requiring a word boundary after the number.
+    # This allows inputs like "CA-05b" to match the base "CA-05" while excluding trailing digits.
+    base_match = re.search(r"(?i)\b([A-Z]{2})\s*-\s*0*([0-9]{1,3})(?!\d)", s)
+
+    if not base_match:
+        logger.debug(f"Could not find control family and number in AP: {ap}")
+        return None
+
+    family = base_match.group(1).upper()
+    number = str(int(base_match.group(2)))  # strip any leading zeros by int conversion
+    control_id = f"{family}-{'0' if len(number) == 1 else ''}{number}"
+
+    # Starting right after the base match, capture immediate numeric parentheses like (03) -> (3).
+    idx = base_match.end()
+    paren_token = re.compile(r"\s*\(\s*([^)]+)\s*\)")
+
+    while True:
+        m = paren_token.match(s, idx)
+        if not m:
+            break
+
+        token = m.group(1)
+
+        # Only accept purely numeric tokens as part of the control identifier.
+        # Stop at the first non-numeric token (e.g., letters like (a), roman numerals, etc.).
+        if re.fullmatch(r"0*\d+", token):
+            sub_number = f"{int(token)}"
+            control_id += f"({'0' if len(sub_number) == 1 else ''}{sub_number})"  # normalize 0-padded numbers
+            idx = m.end()
+            # Continue in case there is another numeric enhancement directly chained.
+            continue
+        else:
+            break
+
+    return control_id
+
+
+def strip_leading_zeros_after_dash_or_dot(s: str) -> str:
+    return re.sub(r"(?<=[-.(])0+(?=\d)", "", s)
+
+
+def extract_control_requirement(ap: Optional[str]) -> Optional[str]:
+    if not ap:
+        return None
+
+    control_id = extract_control(ap=ap)
+    if not control_id:
+        return None
+
+    sn_control_id = strip_leading_zeros_after_dash_or_dot(control_id)
+    stripped_ap = strip_leading_zeros_after_dash_or_dot(ap).upper()
+
+    part = stripped_ap.removeprefix(sn_control_id).removeprefix(".").split(".")[0]
+
+    if not part:
+        return None
+
+    return f"{sn_control_id}.{part.strip().lower()}"

pjdev_sn_sdk/models.py

@@ -0,0 +1,123 @@
+import enum
+from pathlib import Path
+
+from pydantic import BaseModel, computed_field, BeforeValidator, Field, ConfigDict
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from datetime import datetime
+from typing import Dict, List, Optional, Annotated
+
+class ServiceNowEnvironment(enum.StrEnum):
+    PJ = "pj"
+    SubProd1 = "subprod1"
+    Dev = "dev"
+    Test = "test"
+    PreProd = "preprod"
+    Prod = "prod"
+
+
+class SnowTables(BaseModel):
+    auth_boundary_table_name: str = "sn_irm_cont_auth_authorization_boundary"
+    auth_pack_table_name: str = "sn_irm_cont_auth_pack"
+    info_type_definition_table_name: str = (
+        "sn_irm_cont_auth_information_type_definition"
+    )
+    info_type_table_name: str = "sn_irm_cont_auth_information_type"
+    baseline_control_table_name: str = "sn_irm_cont_auth_baseline_control_objective"
+    nist_control_table_name: str = "sn_grc_content"
+    control_table_name: str = "sn_compliance_control"
+    control_overlay_table_name: str = "sn_compliance_policy"
+    control_objective_table_name: str = "sn_compliance_policy_statement"
+    poam_issue_rating_table_name: str = "sn_grc_issue_rating"
+    poam_table_name: str = "sn_grc_issue"
+    poam_control_m2m_table_name: str = "sn_grc_m2m_issue_item"
+    auth_pack_poam_m2m_table_name: str = "sn_irm_cont_auth_m2m_issue_auth_pack"
+    milestone_table_name: str = "sn_irm_cont_auth_milestone_task"
+    cmdb_ci_table_name: str = "cmdb_ci"
+    cmdb_hardware: str = "cmdb_ci_hardware"
+    control_parts_table: str = "sn_compliance_policy_stmt_requirement"
+    control_objective_control_part_m2m_table: str = (
+        "sn_compliance_m2m_policy_stmt_policy_stmt_rqmt"
+    )
+    item_generation_action_event_queue_table: str = "sn_grc_item_generation_action_event_queue"
+
+
+snow_tables = SnowTables()
+
+
+class Config(BaseSettings):
+    scripted_api_base_path: str = "/api/sn_irm_cont_auth/auth_pack"
+    http_retry_max_count = 1000
+    environment: ServiceNowEnvironment = ServiceNowEnvironment.SubProd1
+    api_key: Optional[str] = None
+    api_username: Optional[str] = None
+    api_password: Optional[str] = None
+    output_path: Optional[Path] = None
+
+    model_config = SettingsConfigDict(
+        env_prefix="SN_",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+    @computed_field
+    def sn_api_url(self) -> str:
+        match self.environment:
+            case ServiceNowEnvironment.PreProd:
+                return "https://vaoitpreprod.servicenowservices.com"
+            case ServiceNowEnvironment.SubProd1:
+                return "https://vasubprod1.servicenowservices.com"
+            case ServiceNowEnvironment.Prod:
+                return "https://yourit.va.gov"
+            case ServiceNowEnvironment.Dev:
+                return "https://yourit-dev.va.gov"
+            case ServiceNowEnvironment.Test:
+                return "https://vaoittest.servicenowservices.com"
+            case ServiceNowEnvironment.PJ:
+                return "https://dev190624.service-now.com"
+
+
+class RMFStep(enum.Enum):
+    Prepare = 1
+    Categorize = 8
+    Select = 9
+    Implement = 10
+    Assess = 11
+    Authorize = 12
+    Monitor = 13
+
+
+class SnowBase(BaseModel):
+    sys_id: str
+
+    @classmethod
+    def get_fields(cls) -> List[str]:
+        fields = []
+        for k, fi in cls.model_fields.items():
+            if fi.alias:
+                fields.append(fi.alias)
+            else:
+                fields.append(k)
+        return fields
+
+    @classmethod
+    def get_fields_query_param_value(cls) -> str:
+        return ",".join(cls.get_fields())
+
+    @classmethod
+    def get_df_column_rename_map(cls) -> Dict[str, str]:
+        original_cols = cls.get_fields()
+        cols = [k for k in cls.model_fields.keys()]
+        return dict(zip(original_cols, cols))
+
+
+class SnowAuthPack(SnowBase):
+    name: str
+    number: str
+    step: int
+    authorization_date: Optional[str] = None
+    next_authorization_date: Optional[str] = None
+    authorization_comments: Optional[str] = None
+    authorization_decision: Optional[str] = None
+    ongoing_authorization: Optional[str] = None
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True, strict=False)

pjdev_sn_sdk/sn_api_service.py

@@ -0,0 +1,628 @@
+from contextlib import asynccontextmanager
+from pathlib import Path
+import ssl
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Tuple,
+    Optional,
+    AsyncIterator,
+)
+
+from httpx import AsyncClient
+from loguru import logger
+
+import httpx
+import asyncio
+
+from pjdev_sn_sdk.api_utilities import async_retry_http, log_server_addr, log_response_headers, log_request_headers
+from pjdev_sn_sdk.config_service import get_config
+
+
+async def check_for_202(r: httpx.Response) -> None:
+    if r.status_code == 202:
+        logger.warning(
+            f"202 status code recieved for {r.request.method.upper()} {r.url.path}"
+        )
+        await asyncio.sleep(5)
+        raise httpx.HTTPError("Too many requests")
+
+
+class SnApiKeyAuth(httpx.Auth):
+    def __init__(self, token):
+        self.token = token
+
+    def auth_flow(self, request):
+        request.headers["x-sn-apikey"] = self.token
+        yield request
+
+
+def get_http_client() -> httpx.AsyncClient:
+    config = get_config()
+    if not config.api_key:
+        if not config.api_username or not config.api_password:
+            raise ValueError("Must set username and password")
+        auth = httpx.BasicAuth(
+            username=config.api_username, password=config.api_password
+        )
+    else:
+        auth = SnApiKeyAuth(config.api_key)
+
+    return httpx.AsyncClient(
+        base_url=config.sn_api_url,
+        auth=auth,
+        verify=ssl.create_default_context(),
+        timeout=None,
+        event_hooks={
+            "response": [log_server_addr, log_response_headers],
+            "request": [log_request_headers],
+        },
+    )
+
+
+@asynccontextmanager
+async def http_client() -> AsyncIterator[httpx.AsyncClient]:
+    async with get_http_client() as _client:
+        yield _client
+
+
+@async_retry_http(
+    max_attempts=get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 405, 404],
+)
+async def add_table_row(
+        table_name: str,
+        payload: Dict[str, Any],
+        fields: Optional[str] = None,
+        client: Optional[AsyncClient] = None,
+) -> Dict[str, Any]:
+    async def _exec(_client: AsyncClient) -> Dict[str, Any]:
+        params = {}
+        if fields:
+            params["sysparm_fields"] = fields
+        r = await _client.post(
+            f"/api/now/table/{table_name}", json=payload, params=params
+        )
+        r.raise_for_status()
+        await check_for_202(r=r)
+
+        json_data = r.json()
+        logger.info(f"Added row to {table_name}")
+        return json_data["result"]
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+@async_retry_http(
+    max_attempts=get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 405, 404],
+)
+async def delete_table_row(
+        table_name: str, sys_id: str, client: Optional[AsyncClient] = None
+) -> None:
+    async def _exec(_client: AsyncClient) -> None:
+        r = await _client.delete(f"/api/now/table/{table_name}/{sys_id}")
+        if not r.is_success and r.status_code != 404:
+            r.raise_for_status()
+        await check_for_202(r=r)
+        if r.status_code != 404:
+            logger.info(f"Removed row {sys_id} from {table_name}")
+
+    if not client:
+        async with http_client() as _client:
+            await _exec(_client)
+    else:
+        await _exec(client)
+
+
+async def get_table_rows(
+        table_name: str,
+        query: Optional[str] = None,
+        fields: Optional[str] = None,
+        order_by_fields: Optional[List[str]] = None,
+        sysparm_limit: Optional[int] = None,
+        paginate: bool = False,
+        extra_parms: Optional[Dict] = None,
+        concurrency: Optional[int] = None,
+        process_page_data_func: Optional[Callable[[List[Any]], None]] = None,
+        get_display_values: Optional[bool] = False,
+) -> Tuple[List[Any], int]:
+    url = f"/api/now/table/{table_name}"
+    params: Dict[str, Any] = {}
+
+    # Build query locally (avoid confusing in-place += on the function arg).
+    full_query = query
+    if order_by_fields:
+        order_by_query = "^".join(f"ORDERBY{f}" for f in order_by_fields)
+        full_query = f"{full_query}^{order_by_query}" if full_query else order_by_query
+
+    if full_query:
+        params["sysparm_query"] = full_query
+
+    if fields:
+        params["sysparm_fields"] = fields
+
+    if get_display_values:
+        params["sysparm_display_value"] = True
+
+    if extra_parms:
+        params = {**params, **extra_parms}
+
+    if sysparm_limit is not None:
+        page_size = sysparm_limit
+    elif paginate:
+        page_size = 5000
+    else:
+        page_size = 5000
+
+    params["sysparm_limit"] = page_size
+
+    async with http_client() as client:
+        first_result = await client.get(url, params=params)
+        first_result.raise_for_status()
+        await check_for_202(r=first_result)
+
+    try:
+        first_rows = first_result.json()["result"]
+    except Exception:
+        logger.exception(f"Unexpected response payload for table: {table_name}")
+        raise
+
+    total_count_str = first_result.headers.get("X-Total-Count")
+    if not total_count_str:
+        logger.warning(
+            f"X-Total-Count header was not present in request to table: {table_name}"
+        )
+        # Best-effort: if header is missing, keep count consistent with returned rows.
+        return first_rows, len(first_rows)
+
+    try:
+        total_count = int(total_count_str.strip())
+    except (ValueError, AttributeError):
+        logger.warning(
+            f"Could not parse X-Total-Count={total_count_str!r} for table: {table_name}"
+        )
+        return first_rows, len(first_rows)
+
+    if total_count <= 0:
+        return [], 0
+
+    if not paginate:
+        return first_rows, total_count
+
+    # Paginate remaining pages.
+    concurrency_value = concurrency or 1
+
+    sem = asyncio.Semaphore(value=concurrency_value)
+    logger.info(
+        f"total records for {table_name}: {total_count} (page_size={page_size}, concurrency={concurrency_value})"
+    )
+
+    if process_page_data_func:
+        process_page_data_func(first_rows)
+        logger.info("Finished processing data for initial page")
+
+    @async_retry_http(
+        max_attempts=get_config().http_retry_max_count,
+        delay_seconds=1,
+        status_codes_to_ignore=[400, 401, 403, 405, 404],
+    )
+    async def __run(offset: int) -> List[Any]:
+        async with sem:
+            local_params = {**params, "sysparm_offset": offset}
+            logger.info(f"Offset: {offset}")
+            async with get_http_client() as _client:
+                r = await _client.get(url, params=local_params)
+                r.raise_for_status()
+                await check_for_202(r=r)
+                offset_data = r.json()["result"]
+
+            if total_count - offset >= page_size and len(offset_data) < page_size:
+                raise Exception(
+                    f"Did not receive entire payload: {len(offset_data)} rows"
+                )
+
+            if process_page_data_func:
+                process_page_data_func(offset_data)
+                logger.info(f"Finished processing data for offset {offset} ")
+
+            return offset_data
+
+    # We already fetched the first page (offset 0). Start from the next offset.
+    offsets = range(page_size, total_count, page_size)
+    results_list = await asyncio.gather(*(__run(off) for off in offsets))
+
+    all_rows = [*first_rows, *(item for sub in results_list for item in sub)]
+    return all_rows, total_count
+
+
+@async_retry_http(
+    max_attempts=get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 405, 404],
+)
+async def patch_table_row(
+        table_name: str,
+        sys_id: str,
+        payload: Dict[str, Any],
+        fields: Optional[List[str]] = None,
+        client: Optional[AsyncClient] = None,
+) -> Dict[str, Any]:
+    async def _exec(_client: AsyncClient) -> Dict[str, Any]:
+        params = {}
+
+        if fields:
+            params["sysparm_fields"] = ",".join(fields)
+
+        r = await _client.patch(
+            f"/api/now/table/{table_name}/{sys_id}", json=payload, params=params
+        )
+        r.raise_for_status()
+        await check_for_202(r=r)
+
+        logger.info(f"Updated row to {table_name}")
+
+        return r.json()["result"]
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+@async_retry_http(
+    max_attempts=get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 405, 404],
+)
+async def move_auth_package_to_step(
+        target_step: int, auth_package_sys_id: str, client: Optional[AsyncClient] = None
+) -> Dict:
+    scripted_api_base_path = get_config().scripted_api_base_path
+
+    async def _exec(_client: AsyncClient) -> Dict:
+        r = await _client.patch(
+            f"{scripted_api_base_path}/{auth_package_sys_id}",
+            json={"target_step": target_step},
+        )
+        r.raise_for_status()
+        await check_for_202(r=r)
+        return r.json()["result"]
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+async def update_poam(
+        auth_package_sys_id: str,
+        poam_sys_id: str,
+        payload: Dict[str, Any],
+        client: Optional[AsyncClient] = None,
+) -> Dict:
+    config = get_config()
+
+    async def _exec(_client: AsyncClient) -> Dict:
+        r = await _client.patch(
+            f"{config.scripted_api_base_path}/{auth_package_sys_id}/poam/{poam_sys_id}",
+            json=payload,
+        )
+        r.raise_for_status()
+        await check_for_202(r=r)
+        return r.json()["result"]
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+@async_retry_http(
+    max_attempts=get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 405, 404],
+)
+async def create_poam(
+        auth_package_sys_id: str, payload: Dict, client: Optional[AsyncClient] = None
+) -> Dict:
+    config = get_config()
+
+    async def _exec(_client: AsyncClient) -> Dict:
+        r = await _client.post(
+            f"{config.scripted_api_base_path}/{auth_package_sys_id}/poam",
+            json=payload,
+        )
+        r.raise_for_status()
+        await check_for_202(r=r)
+        return r.json()["result"]
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+@async_retry_http(
+    max_attempts=get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 405, 404],
+)
+async def update_auth_package_authorize_info(
+        auth_package_sys_id: str,
+        payload: Dict[str, Any],
+        client: Optional[AsyncClient] = None,
+) -> Dict:
+    config = get_config()
+
+    async def _exec(_client: AsyncClient) -> Dict:
+        r = await _client.patch(
+            f"{config.scripted_api_base_path}/{auth_package_sys_id}/authorize",
+            json=payload,
+        )
+        r.raise_for_status()
+        await check_for_202(r=r)
+        return r.json()["result"]
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+@async_retry_http(
+    max_attempts=get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 405, 404],
+)
+async def update_control_status(
+        auth_pack_sys_id: str,
+        compliance_control_sys_id: str,
+        payload: Dict[str, Any],
+        client: Optional[AsyncClient] = None,
+) -> Dict:
+    config = get_config()
+
+    async def _exec(_client: AsyncClient) -> Dict:
+        r = await _client.patch(
+            f"{config.scripted_api_base_path}/{auth_pack_sys_id}/control/{compliance_control_sys_id}",
+            json=payload,
+        )
+        r.raise_for_status()
+        await check_for_202(r=r)
+        return r.json()["result"]
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+__sem = asyncio.Semaphore(10)
+
+
+@async_retry_http(
+    max_attempts=get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 404],
+)
+async def associate_file_reference_to_compliance_control(
+        compliance_control_sys_id: str, text: str, client: Optional[AsyncClient] = None
+) -> Dict[str, Any]:
+    config = get_config()
+
+    async def _exec(_client: AsyncClient) -> Dict[str, Any]:
+        payload = {"u_associated_artifacts": text}
+
+        async with __sem:
+            r = await _client.patch(
+                f"{config.scripted_api_base_path}/control_artifact_association/{compliance_control_sys_id}",
+                json=payload,
+            )
+            r.raise_for_status()
+            await check_for_202(r=r)
+        updated_control = r.json()["result"]
+
+        return updated_control["current_values"]
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+@async_retry_http(
+    max_attempts=get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 404],
+)
+async def get_attachment_sys_id(
+        table_name: str,
+        table_row_sys_id: str,
+        snow_filename: str,
+        client: Optional[AsyncClient] = None,
+) -> Optional[str]:
+    async def _exec(_client: AsyncClient) -> Optional[str]:
+        query_params = {
+            "table_name": table_name,
+            "table_sys_id": table_row_sys_id,
+            "file_name": snow_filename,
+        }
+        async with __sem:
+            r = await _client.get(
+                "api/now/attachment",
+                params=query_params,
+            )
+
+            r.raise_for_status()
+            await check_for_202(r=r)
+            json_data = r.json()["result"]
+            if len(json_data) > 0:
+                return json_data[0].get("sys_id")
+
+        return None
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+@async_retry_http(
+    max_attempts=get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 404],
+)
+async def get_attachments(
+        table_name: str,
+        table_row_sys_id: str,
+        client: Optional[AsyncClient] = None,
+) -> List[Dict[str, Any]]:
+    async def _exec(_client: AsyncClient) -> List[Dict[str, Any]]:
+        query = f'table_name="{table_name}"^table_sys_id="{table_row_sys_id}"'
+        async with __sem:
+            r = await _client.get(
+                "api/now/attachment",
+                params=f"sysparm_query={query}",
+            )
+
+            r.raise_for_status()
+            await check_for_202(r=r)
+            return r.json()["result"]
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+@async_retry_http(
+    max_attempts=get_config().get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 404],
+)
+async def delete_attachment(
+        attachment_id: str, client: Optional[AsyncClient] = None
+) -> None:
+    async def _exec(_client: AsyncClient) -> None:
+        r = await _client.delete(f"api/now/attachment/{attachment_id}")
+
+        r.raise_for_status()
+        await check_for_202(r=r)
+
+    if not client:
+        async with http_client() as _client:
+            await _exec(_client)
+    else:
+        await _exec(client)
+
+
+@async_retry_http(
+    max_attempts=3,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 404],
+)
+async def upload_artifact(
+        table_name: str,
+        table_row_sys_id: str,
+        filepath: Path,
+        snow_filename: str,
+        client: Optional[AsyncClient] = None,
+) -> Tuple[str, str]:
+    async def _exec(_client: AsyncClient) -> Tuple[str, str]:
+        generic_type = "application/octet-stream"
+        headers = {"Content-Type": generic_type, "Accept": "application/json"}
+        query_params = {
+            "table_name": table_name,
+            "table_sys_id": table_row_sys_id,
+            "file_name": snow_filename,
+        }
+        async with __sem:
+            with open(filepath, mode="rb") as file:
+                r = await _client.post(
+                    "api/now/attachment/file",
+                    headers=headers,
+                    params=query_params,
+                    content=file.read(),
+                )
+
+            if not r.is_success:
+                logger.warning(r.text)
+
+            r.raise_for_status()
+            await check_for_202(r=r)
+
+            json_data = r.json()["result"]
+            filename = json_data.get("file_name")
+            attachment_id = json_data.get("sys_id")
+
+            return f"{attachment_id}", filename
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+@async_retry_http(
+    max_attempts=get_config().http_retry_max_count,
+    delay_seconds=1,
+    status_codes_to_ignore=[400, 401, 403, 404],
+)
+async def download_artifact(
+        attachment_id: str, filename: str, client: Optional[AsyncClient] = None
+) -> Tuple[str, str]:
+    config = get_config()
+
+    async def _exec(_client: AsyncClient) -> Tuple[str, str]:
+        r = await _client.get(f"api/now/attachment/{attachment_id}/file")
+
+        if not r.is_success:
+            logger.warning(r.text)
+
+        r.raise_for_status()
+        await check_for_202(r=r)
+
+        with open(config.output_path / filename, mode="wb") as file:
+            for chunk in r.iter_bytes():
+                file.write(chunk)
+
+        return config.output_path / filename
+
+    if not client:
+        async with http_client() as _client:
+            return await _exec(_client)
+    else:
+        return await _exec(client)
+
+
+def get_attachment_url(attachment_id: str) -> str:
+    config = get_config()
+    return f"{config.sn_api_url}/nav_to.do?uri=sys_attachment.do?sys_id={attachment_id}"
+
+
+if __name__ == "__main__":
+    async def __test() -> None:
+        _data = await get_table_rows("sn_irm_cont_auth_pack", sysparm_limit=1)
+        logger.info(_data)
+
+
+    asyncio.run(__test())

pjdev_sn_sdk/sn_api_utilities.py

@@ -0,0 +1,188 @@
+import asyncio
+from datetime import datetime, UTC
+from typing import List, Dict, Optional
+
+from loguru import logger
+
+from pjdev_sn_sdk import sn_api_service, control_parsing_service
+from pjdev_sn_sdk.config_service import get_config
+from pjdev_sn_sdk.models import snow_tables, SnowAuthPack
+
+
+async def watch_for_queued_task_to_complete(auth_pack_sys_id: str) -> None:
+    table_name = snow_tables.item_generation_action_event_queue_table
+    fields = ["sys_id", "state", "source.name", "action", "table", "sys_created_on"]
+    current_state = None
+    max_retries = 1080  # 5400 total seconds (1.5 hours) in 5-second intervals
+    retry_count = 0
+    utc_timestamp = datetime.now(tz=UTC)
+    while current_state != "processed":
+        if retry_count >= max_retries:
+            raise Exception(
+                f"Timed out waiting for task to complete for {auth_pack_sys_id}"
+            )
+        retry_count += 1
+        await asyncio.sleep(5)
+        response, count = await sn_api_service.get_table_rows(
+            table_name=table_name,
+            query=f"source={auth_pack_sys_id}^ORDERBYDESCsys_created_on",
+            fields=",".join(fields),
+        )
+        if len(response) == 0:
+            continue
+        data = response[0]
+        created_on_str = data["sys_created_on"] + " +0000"
+        created_on_dt = datetime.strptime(created_on_str, "%Y-%m-%d %H:%M:%S %z")
+        current_state = data["state"]
+        # ensure we are not looking at a job that was processed by a previous run
+        if (
+            abs((utc_timestamp - created_on_dt).total_seconds()) > 60 * 3
+            and current_state == "processed"
+        ):
+            current_state = ""
+            utc_timestamp = datetime.now(tz=UTC)
+            continue
+        auth_pack_name = data["source.name"]
+        logger.info(f"State of task for {auth_pack_name}: {current_state}")
+
+
+async def mark_control_ids_as_common(
+    auth_pack_sys_id: str, control_ids: List[str]
+) -> None:
+    baseline_controls, baseline_count = await sn_api_service.get_table_rows(
+        table_name=snow_tables.baseline_control_table_name,
+        query=f"authorization_package.sys_id={auth_pack_sys_id}",
+        fields="sys_id,control_objective.reference,common_control,implementation_status,inherited_from,justification",
+    )
+
+    for control_id in control_ids:
+        b_control = next(
+            (
+                b
+                for b in baseline_controls
+                if control_parsing_service.extract_control(
+                    b["control_objective.reference"]
+                )
+                == control_parsing_service.extract_control(control_id)
+            ),
+            None,
+        )
+        if not b_control:
+            logger.warning(f"{control_id} not found in baseline")
+            continue
+
+        updated_sn_row = await sn_api_service.patch_table_row(
+            table_name=snow_tables.baseline_control_table_name,
+            payload={
+                "implementation_status": "need_to_implement",
+                "common_control": True,
+            },
+            fields=[
+                "implementation_status",
+                "sys_id",
+                "common_control",
+                "control_objective.reference",
+            ],
+            sys_id=b_control["sys_id"],
+        )
+
+        if (
+            updated_sn_row["implementation_status"] != "need_to_implement"
+            or updated_sn_row["common_control"] != "true"
+        ):
+            logger.error(f"{control_id} was not marked as n/a")
+
+
+async def associate_controls_to_poam(
+    auth_pack_sys_id: str,
+    poam_sys_id: str,
+    display_poam_id: str,
+    control_ids: List[str],
+) -> None:
+    controls, count = await sn_api_service.get_table_rows(
+        table_name=snow_tables.control_table_name,
+        paginate=True,
+        fields="sys_id,content.reference",
+        query=f"authorization_package.sys_id={auth_pack_sys_id}",
+    )
+
+    for control_id in control_ids:
+        sn_c = next(
+            (
+                c
+                for c in controls
+                if c["content.reference"].lower() == control_id.lower()
+            ),
+            None,
+        )
+
+        if not sn_c:
+            logger.warning(
+                f"Could not find control {control_id} in auth pack {auth_pack_sys_id}"
+            )
+            continue
+
+        logger.info(
+            f"Associating control {control_id} to poam {display_poam_id} ({poam_sys_id})"
+        )
+        payload = {"sn_grc_issue": poam_sys_id, "sn_grc_item": sn_c["sys_id"]}
+        existing_link, link_count = await sn_api_service.get_table_rows(
+            table_name=snow_tables.poam_control_m2m_table_name,
+            query=f"sn_grc_issue.sys_id={poam_sys_id}^sn_grc_item.sys_id={sn_c['sys_id']}",
+        )
+
+        if link_count == 0:
+            await sn_api_service.add_table_row(
+                table_name=snow_tables.poam_control_m2m_table_name,
+                payload=payload,
+            )
+        else:
+            logger.info(
+                f"Poam Control Link already exists for {display_poam_id} - {control_id}"
+            )
+
+    logger.info(f"Finished associating controls for {display_poam_id}")
+
+
+def save_existing_auth_pack_data(sn_auth_pack: Dict) -> SnowAuthPack:
+    auth_pack = SnowAuthPack.model_validate(sn_auth_pack)
+    config = get_config()
+
+    with open(
+        config.output_path / f"existing_data_{auth_pack.number}.json",
+        "w",
+        encoding="utf-8",
+    ) as f:
+        f.write(auth_pack.model_dump_json(indent=2))
+
+    return auth_pack
+
+
+def read_existing_auth_pack_data(sn_auth_pack_number: str) -> Optional[SnowAuthPack]:
+    config = get_config()
+    filepath = config.output_path / f"existing_data_{sn_auth_pack_number}.json"
+    if not filepath.exists():
+        return None
+    with open(filepath, "r", encoding="utf-8") as f:
+        json_str = f.read()
+
+        auth_pack = SnowAuthPack.model_validate_json(json_str)
+
+    return auth_pack
+
+
+if __name__ == "__main__":
+    ap_number = "testing"
+    test = {
+        "sys_id": "test sys id",
+        "name": "test",
+        "number": ap_number,
+        "step": 9,
+        "authorization_comments": "auth_pack.authorization_comments",
+        "next_authorization_date": "auth_pack.next_authorization_date",
+        "authorization_date": "auth_pack.authorization_date",
+        "authorization_decision": "auth_pack.authorization_decision",
+        "ongoing_authorization": "auth_pack.ongoing_authorization",
+    }
+    save_existing_auth_pack_data(test)
+    print(read_existing_auth_pack_data(ap_number))

pjdev_sn_sdk-4.6.4.dist-info/METADATA

@@ -0,0 +1,48 @@
+Metadata-Version: 2.4
+Name: pjdev-sn-sdk
+Version: 4.6.4
+Project-URL: Documentation, https://gitlab.purplejay.io/keystone/python/-/tree/main/pjdev-pydantic/README.md
+Project-URL: Issues, https://gitlab.purplejay.io/keystone/python/-/issues
+Project-URL: Source, https://gitlab.purplejay.io/keystone/python
+Author-email: Purple Jay LLC <developers@purplejay.io>
+License-Expression: MIT
+License-File: LICENSE.txt
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=3.12
+Requires-Dist: httpx
+Requires-Dist: loguru
+Requires-Dist: pydantic-settings>=2.13.1
+Requires-Dist: pydantic>=2.12.5
+Provides-Extra: dev
+Requires-Dist: ruff; extra == 'dev'
+Provides-Extra: test
+Requires-Dist: coverage; extra == 'test'
+Requires-Dist: pytest; extra == 'test'
+Requires-Dist: pytz; extra == 'test'
+Description-Content-Type: text/markdown
+
+# pjdev-sn-sdk
+
+[![PyPI - Version](https://img.shields.io/pypi/v/pjdev-sn-sdk.svg)](https://pypi.org/project/pjdev-sn-sdk)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pjdev-sn-sdk.svg)](https://pypi.org/project/pjdev-sn-sdk)
+
+-----
+
+## Table of Contents
+
+- [Installation](#installation)
+- [License](#license)
+
+## Installation
+
+```console
+pip install pjdev-sn-sdk
+```
+
+## License
+
+`pjdev-sn-sdk` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license.

pjdev_sn_sdk-4.6.4.dist-info/RECORD

@@ -0,0 +1,12 @@
+pjdev_sn_sdk/__about__.py,sha256=opYe2fsq9dxRdB87CIMtUyN0XYxM8oLw0JewyGRCu-8,129
+pjdev_sn_sdk/__init__.py,sha256=2UETghiXy-pnzHKsOef2J4B_g7SSzVWGI-dVTed7FYo,107
+pjdev_sn_sdk/api_utilities.py,sha256=XIy_cJWswaeacjEU5iHq0eD6w4N7GXklUQewEgmESRk,5294
+pjdev_sn_sdk/config_service.py,sha256=ZHTlyLCgvXZaLX7NWyt0Tvn6WsGBk3dwwkbeZXRRYRw,406
+pjdev_sn_sdk/control_parsing_service.py,sha256=fPi9QXhsMQT57XAhqtGyoxB-1-CpAD2jHUPd04KzDHM,3448
+pjdev_sn_sdk/models.py,sha256=o3kOKU0LORjCbuybhq7tc1IR0qZoCyeQzISn7SC1-kQ,4177
+pjdev_sn_sdk/sn_api_service.py,sha256=A_un_RsALpkVuxCWnquv_3mApjfucLUzyOj1wPpx7lI,18617
+pjdev_sn_sdk/sn_api_utilities.py,sha256=Cy6pTEvzWgw2EQi17Y6LE7Yo7fcnDrqs2SIBerLEFP0,6570
+pjdev_sn_sdk-4.6.4.dist-info/METADATA,sha256=5vGmsx4Vyim9aniWxJalNcQn1oQ8KoiA7-73qZi66GU,1567
+pjdev_sn_sdk-4.6.4.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+pjdev_sn_sdk-4.6.4.dist-info/licenses/LICENSE.txt,sha256=lsBCQyCWJBNssHxGJFROYAUl6CVjKJ52uHHydxCxr4c,1099
+pjdev_sn_sdk-4.6.4.dist-info/RECORD,,

pjdev_sn_sdk-4.6.4.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pjdev_sn_sdk-4.6.4.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2026-present Chris O'Neill <chris@purplejay.io>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.