clue-api 1.3.0.dev32__tar.gz → 1.3.0.dev34__tar.gz

{clue_api-1.3.0.dev32 → clue_api-1.3.0.dev34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clue-api
-Version: 1.3.0.dev32
+Version: 1.3.0.dev34
 Summary: Clue distributed enrichment service
 License: MIT
 License-File: LICENSE
@@ -30,7 +30,6 @@ Requires-Dist: flask (<3.0.0)
 Requires-Dist: flask-caching (>=2.1.0,<3.0.0)
 Requires-Dist: flask-cors (>=4.0.1,<7.0.0) ; extra == "server"
 Requires-Dist: gevent (>=24.2.1,<25.0.0)
-Requires-Dist: geventhttpclient (>=2.3.1,<3.0.0)
 Requires-Dist: gunicorn (>=22,<24)
 Requires-Dist: imgkit (>=1.2.3,<2.0.0)
 Requires-Dist: passlib (>=1.7.4,<2.0.0) ; extra == "server"

{clue_api-1.3.0.dev32 → clue_api-1.3.0.dev34}/clue/app.py

@@ -1,3 +1,7 @@
+from gevent import monkey
+
+monkey.patch_all()
+
 import os
 import sys
 from pathlib import Path

clue_api-1.3.0.dev34/clue/constants/env.py

@@ -0,0 +1,5 @@
+import os
+import sys
+
+DISABLE_CACHE = os.environ.get("DISABLE_CACHE", "false").lower() in ["true", "1"]
+TESTING = (os.environ.get("TESTING", "false").lower() in ["true", "1"]) or ("pytest" in sys.modules)

clue_api-1.3.0.dev34/clue/helper/discover.py

@@ -0,0 +1,51 @@
+from typing import Any
+
+import requests
+
+from clue.common.logging import get_logger
+from clue.config import config
+from clue.constants.env import TESTING
+
+logger = get_logger(__file__)
+
+
+def get_apps_list() -> list[dict[str, str]]:
+    """Get a list of apps from the discovery service
+
+    Returns:
+        list[dict[str, str]]: A list of other apps
+    """
+    apps: list[dict[str, Any]] = []
+
+    if TESTING:
+        return apps
+
+    if config.api.discover_url:
+        try:
+            resp = requests.get(config.api.discover_url, headers={"accept": "application/json"}, timeout=5)
+
+            if not resp.ok:
+                logger.warning(
+                    "Invalid response %s from server for apps discovery: %s", resp.status_code, config.api.discover_url
+                )
+                return apps
+
+            data = resp.json()
+            for app in data["applications"]["application"]:
+                url = app["instance"][0]["hostName"]
+
+                if "clue" not in url:
+                    apps.append(
+                        {
+                            "alt": app["instance"][0]["metadata"]["alternateText"],
+                            "name": app["name"],
+                            "img_d": app["instance"][0]["metadata"]["imageDark"],
+                            "img_l": app["instance"][0]["metadata"]["imageLight"],
+                            "route": url,
+                            "classification": app["instance"][0]["metadata"]["classification"],
+                        }
+                    )
+        except Exception:
+            logger.exception(f"Failed to get apps from discover URL: {config.api.discover_url}")
+
+    return sorted(apps, key=lambda k: k["name"])

{clue_api-1.3.0.dev32 → clue_api-1.3.0.dev34}/clue/remote/datatypes/__init__.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+import sys
 import time
 from datetime import datetime
 
@@ -10,6 +11,7 @@ from packaging.version import parse
 
 from clue.common.logging import get_logger
 from clue.common.uid import get_random_id
+from clue.constants.env import TESTING
 
 logger = get_logger(__file__)
 
@@ -91,7 +93,7 @@ def get_client(host, port, private, password=None):
 
     if password:
         logger.debug("Connecting to redis with password")
-    else:
+    elif "pytest" not in sys.modules and not TESTING:
         logger.warning("Connecting to redis without authentication.")
 
     ssl_kwargs = {}

{clue_api-1.3.0.dev32 → clue_api-1.3.0.dev34}/clue/services/lookup_service.py

@@ -7,17 +7,14 @@ import time
 from datetime import datetime, timedelta, timezone
 from hashlib import sha256
 from typing import Any, Optional
-from urllib.parse import urlparse
 
-import elasticapm
-from elasticapm.traces import Transaction, execution_context
+from elasticapm.traces import Transaction, capture_span, execution_context
 from flask import Request, request
 from gevent import Greenlet
 from gevent.pool import Pool
-from geventhttpclient import HTTPClient
-from geventhttpclient.response import HTTPResponse
 from pydantic import BaseModel, ValidationError
-from requests import Response
+from requests import Response, Session
+from requests.adapters import HTTPAdapter, Retry
 
 from clue.common.exceptions import (
     AuthenticationException,
@@ -35,32 +32,40 @@ from clue.models.selector import Selector
 from clue.services import auth_service, type_service, user_service
 
 logger = get_logger(__file__)
-CLIENTS: dict[str, HTTPClient] = {}
+CLIENTS: dict[str, Session] = {}
 
 
-def get_client(base_url: str, timeout: float) -> HTTPClient:
-    """Gets or creates an HTTPClient for the provided base_url.
+def get_client(base_url: str, timeout: float) -> Session:
+    """Gets or creates a requests session for the provided base_url.
 
     Args:
         base_url (str): The base url of the desired client.
         timeout (float): The connection and network timeout to use (is multiplied by 3).
 
     Returns:
-        HTTPClient: The HTTPClient instance matching the provided base_url.
+        Session: The requests Session instance matching the provided base_url.
     """
     client_hash = sha256(base_url.encode())
     client_hash.update(str(timeout).encode())
     client_key = client_hash.hexdigest()
 
     if client_key not in CLIENTS:
-        # Pool of 16 connections by default
-        CLIENTS[client_key] = HTTPClient.from_url(
-            base_url,
-            concurrency=math.floor(int(os.environ.get("EXECUTOR_THREADS", 32)) / 2),
-            connection_timeout=timeout * 3,
-            network_timeout=timeout * 3,
+        session = Session()
+        # Configure connection pool with HTTPAdapter
+        pool_connections = math.floor(int(os.environ.get("EXECUTOR_THREADS", 32)) / 2)
+
+        retry_strategy = Retry(
+            total=2, backoff_factor=0.5, status_forcelist=[429, 500, 502, 503, 504], allowed_methods=["GET", "POST"]
+        )
+        adapter = HTTPAdapter(
+            pool_connections=pool_connections, pool_maxsize=pool_connections, max_retries=retry_strategy
         )
 
+        session.mount("http://", adapter)
+        session.mount("https://", adapter)
+
+        CLIENTS[client_key] = session
+
     return CLIENTS[client_key]
 
 
@@ -162,7 +167,7 @@ def parse_query_params(request: Request, limit: int = 10, timeout: float = 5.0):
 
 def generate_params(
     limit: int, timeout: float, no_annotation: bool = False, include_raw: bool = False, no_cache: bool = False
-):
+) -> dict[str, str | int | float | bool]:
     """Generates HTTP request parameters for a call to a source.
 
     Args:
@@ -176,7 +181,7 @@ def generate_params(
         str: A string of HTTP params formatted so that it can be appended to a url
             (in the format "?param1=value1&param2=value2")
     """
-    params = {
+    params: dict[str, str | int | float | bool] = {
         "limit": limit,
         "max_timeout": max(timeout * 0.95, 0.5),
         "deadline": (datetime.now(timezone.utc) + timedelta(seconds=max(timeout * 0.95, 0.5))).timestamp(),
@@ -191,7 +196,7 @@ def generate_params(
     if no_cache:
         params["no_cache"] = True
 
-    return "?" + "&".join(f"{key}={val}" for key, val in params.items())
+    return params
 
 
 def process_exception(source_name: str, rsp: Response | None, exception: Exception):
@@ -252,7 +257,7 @@ def parse_response(source: ExternalSource, user: dict[str, Any], api_response: A
     Returns:
         list[QueryEntry]: The list of results contained in the response.
     """
-    with elasticapm.capture_span(source.name, "parsing"):
+    with capture_span(source.name, "parsing"):
         if isinstance(api_response, dict):
             api_response = [api_response]
 
@@ -294,7 +299,7 @@ def parse_bulk_response(
     if source.production:
         logger.debug(f"Skipping validation for production source {source.name}")
 
-    with elasticapm.capture_span(f"{source.name}-bulk", "parsing"):
+    with capture_span(f"{source.name}-bulk", "parsing"):
         for type in api_response:
             bulk_result.setdefault(type, {})
             for value in api_response[type]:
@@ -379,7 +384,7 @@ def query_external(
 
     finish_result = functools.partial(build_result, type_name, value, source)
 
-    with elasticapm.capture_span(query_external.__name__, span_type="greenlet"):
+    with capture_span(query_external.__name__, span_type="greenlet"):
         if type_name not in type_service.all_supported_types(user, access_token=access_token).get(source.name, {}):
             return finish_result(error="invalid_type")
 
@@ -404,17 +409,19 @@ def query_external(
         # perform the lookup, ensuring access controls are applied
         url = f"{source.url}/lookup/{type_name}/{value}/"
         response: Any = None
-        rsp: HTTPResponse | None = None
+        rsp: Response | None = None
         start = time.perf_counter()
         try:
-            with elasticapm.capture_span(url, "http"):
-                parsed_url = urlparse(url)
-                rsp = get_client(f"{parsed_url.scheme}://{parsed_url.netloc}", timeout).get(
-                    parsed_url.path + generate_params(limit, timeout, no_annotation, include_raw, no_cache),
+            with capture_span(url, "http"):
+                rsp = get_client(source.url, timeout).get(
+                    url,
+                    params=generate_params(limit, timeout, no_annotation, include_raw, no_cache),
                     headers=generate_headers(access_token, clue_access_token),
+                    timeout=(timeout, timeout * 3),
                 )
+                rsp.raise_for_status()
 
-            response = json.load(rsp)
+            response = rsp.json()
         except Exception as exception:
             return finish_result(
                 error=process_exception(source.name, rsp, exception),
@@ -571,7 +578,7 @@ def bulk_query_external(  # noqa: C901
     if apm_transaction:
         execution_context.set_transaction(apm_transaction)
 
-    with elasticapm.capture_span(bulk_query_external.__name__, span_type="greenlet"):
+    with capture_span(bulk_query_external.__name__, span_type="greenlet"):
         supported_types = type_service.all_supported_types(user, access_token=access_token).get(source.name, {})
         bulk_result: dict[str, dict[str, QueryResult]] = {}
 
@@ -617,22 +624,23 @@ def bulk_query_external(  # noqa: C901
         url = f"{source.url}/lookup/"
         response: Any = None
         start = time.perf_counter()
-        rsp: HTTPResponse | None = None
+        rsp: Response | None = None
         try:
-            with elasticapm.capture_span(url, "http"):
-                parsed_url = urlparse(url)
-
-                rsp = get_client(f"{parsed_url.scheme}://{parsed_url.netloc}", timeout * 1.1).post(
-                    parsed_url.path + generate_params(limit, timeout, no_annotation, include_raw, no_cache),
-                    body=json.dumps([entry.model_dump(exclude_none=True, exclude_unset=True) for entry in data]),
+            with capture_span(url, "http"):
+                rsp = get_client(source.url, timeout).post(
+                    url,
+                    params=generate_params(limit, timeout, no_annotation, include_raw, no_cache),
+                    json=[entry.model_dump(exclude_none=True, exclude_unset=True) for entry in data],
                     headers=generate_headers(access_token, clue_access_token),
+                    timeout=(timeout * 3, timeout * 3),
                 )
+                rsp.raise_for_status()
 
             if not rsp:
                 raise ClueRuntimeError(f"An error occurred when connecting to {source.name}.")  # noqa: TRY301
 
             logger.debug(f"{rsp.status_code}: {url}")
-            response = json.load(rsp)
+            response = rsp.json()
         except Exception as exception:
             error = process_exception(source.name, rsp, exception)
         finally:

{clue_api-1.3.0.dev32 → clue_api-1.3.0.dev34}/clue/services/type_service.py

@@ -7,6 +7,7 @@ from requests import exceptions
 
 from clue.common.logging import get_logger
 from clue.config import CLASSIFICATION, DEBUG, cache, config
+from clue.constants.env import DISABLE_CACHE
 from clue.constants.supported_types import SUPPORTED_TYPES
 from clue.helper.headers import generate_headers
 from clue.models.config import ExternalSource
@@ -57,7 +58,7 @@ def get_supported_types(source_url: str, access_token: str | None = None, obo_ac
     """
     url = f"{source_url}{'' if source_url.endswith('/') else '/'}types/"
 
-    if result := CACHE.get(url):
+    if not DISABLE_CACHE and (result := CACHE.get(url)):
         logger.info("Cache hit for url %s", url)
         return result
 

{clue_api-1.3.0.dev32 → clue_api-1.3.0.dev34}/pyproject.toml

@@ -8,7 +8,7 @@ line-length = 120
 # coverage.py settings #
 ########################
 [tool.coverage.run]
-omit = ["clue/gunicorn_config.py", "clue/patched.py", "test/utils/*"]
+omit = ["clue/gunicorn_config.py", "test/utils/*"]
 branch = true
 sigterm = true
 data_file = ".coverage.server"
@@ -76,9 +76,6 @@ line-length = 120
 indent-width = 4
 target-version = "py312"
 
-[tool.ruff.format]
-exclude = ["clue/patched.py"]
-
 [tool.ruff.lint]
 select = [
   "E",
@@ -112,7 +109,6 @@ ignore = [
   "TRY300",
 ]
 exclude = [
-  "clue/patched.py",
   "plugins/assemblyline_lookup/app.py",
   "plugins/**/test*.py",
 ]
@@ -146,7 +142,7 @@ log_cli_level = "WARN"
 [tool.poetry]
 package-mode = true
 name = "clue-api"
-version = "1.3.0.dev32"
+version = "1.3.0.dev34"
 description = "Clue distributed enrichment service"
 authors = ["Canadian Centre for Cyber Security <contact@cyber.gc.ca>"]
 license = "MIT"
@@ -184,7 +180,6 @@ flask-caching = "^2.1.0"
 gunicorn = ">=22,<24"
 gevent = "^24.2.1"
 pydantic = "^2.7.1"
-geventhttpclient = "^2.3.1"
 pydantic-settings = { extras = ["yaml"], version = "^2.3.4" }
 redis = { version = "^5.0.3" }
 beautifulsoup4 = "^4.13.3"
@@ -232,7 +227,7 @@ name = "pypi"
 priority = "supplemental"
 
 [tool.poetry.scripts]
-server = "clue.patched:main"
+server = "clue.app:main"
 test = "build_scripts.run_tests:main"
 last_success = "build_scripts.last_success:main"
 check_changes = "build_scripts.check_changes:main"
@@ -261,6 +256,7 @@ types-pyyaml = "^6.0.12.20240311"
 types-pytz = "^2024.1.0.20240203"
 types-redis = "^4.6.0.20241004"
 types-dateparser = "^1.2.0.20250208"
+types-requests = "^2.32.4.20250913"
 
 [build-system]
 requires = ["poetry-core"]

clue_api-1.3.0.dev32/clue/helper/discover.py

@@ -1,53 +0,0 @@
-import os
-import sys
-
-import geventhttpclient
-
-from clue.common.logging import get_logger
-from clue.config import config
-
-logger = get_logger(__file__)
-
-
-def get_apps_list() -> list[dict[str, str]]:
-    """Get a list of apps from the discovery service
-
-    Returns:
-        list[dict[str, str]]: A list of other apps
-    """
-    apps = []
-
-    if "pytest" in sys.modules or bool(os.getenv("SKIP_DISCOVERY", "")):
-        logger.info("Skipping discovery, running in a test environment")
-
-    if config.api.discover_url:
-        try:
-            resp = geventhttpclient.get(
-                config.api.discover_url,
-                headers={"accept": "application/json"},
-            )
-            if resp.ok:
-                data = resp.json()
-                for app in data["applications"]["application"]:
-                    try:
-                        url = app["instance"][0]["hostName"]
-
-                        if "clue" not in url:
-                            apps.append(
-                                {
-                                    "alt": app["instance"][0]["metadata"]["alternateText"],
-                                    "name": app["name"],
-                                    "img_d": app["instance"][0]["metadata"]["imageDark"],
-                                    "img_l": app["instance"][0]["metadata"]["imageLight"],
-                                    "route": url,
-                                    "classification": app["instance"][0]["metadata"]["classification"],
-                                }
-                            )
-                    except Exception:
-                        logger.exception(f"Failed to parse get app: {str(app)}")
-            else:
-                logger.warning(f"Invalid response from server for apps discovery: {config.api.discover_url}")
-        except Exception:
-            logger.exception(f"Failed to get apps from discover URL: {config.api.discover_url}")
-
-    return sorted(apps, key=lambda k: k["name"])