clue-api 1.0.1.dev67__tar.gz → 1.0.1.dev69__tar.gz

{clue_api-1.0.1.dev67 → clue_api-1.0.1.dev69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clue-api
-Version: 1.0.1.dev67
+Version: 1.0.1.dev69
 Summary: Clue distributed enrichment service
 License: MIT
 License-File: LICENSE

{clue_api-1.0.1.dev67 → clue_api-1.0.1.dev69}/pyproject.toml

@@ -129,9 +129,11 @@ suppress-none-returning = true
 "clue/app.py" = ["E402"]
 "clue/api/v1/auth.py" = ["TRY301"]
 "clue/common/classification.py" = ["D", "ANN", "C901", "TRY301", "T203"]
-"clue/plugin/interactive.py" = ["T201"]
 "clue/remote/datatypes/*" = ["D", "ANN", "C901"]
 "clue/security/__init__.py" = ["TRY301"]
+"plugin/interactive.py" = ["T201"]
+"plugin/create.py" = ["T201", "D103"]
+"plugin/commands.py" = ["T201", "D103"]
 "test/conftest.py" = ["E402"]
 
 ###################
@@ -147,7 +149,7 @@ log_cli_level = "WARN"
 [tool.poetry]
 package-mode = true
 name = "clue-api"
-version = "1.0.1.dev67"
+version = "1.0.1.dev69"
 description = "Clue distributed enrichment service"
 authors = ["Canadian Centre for Cyber Security <contact@cyber.gc.ca>"]
 license = "MIT"
@@ -239,7 +241,8 @@ last_success = "build_scripts.last_success:main"
 check_changes = "build_scripts.check_changes:main"
 type_check = "build_scripts.type_check:main"
 coverage_report = "build_scripts.coverage_reports:main"
-plugin = "clue.plugin.interactive:main"
+plugin = "plugin.interactive:main"
+create = "plugin.create:main"
 
 [tool.poetry.group.test.dependencies]
 pytest = "^8.1.1"

clue_api-1.0.1.dev67/clue/plugin/files/gunicorn_config.py

@@ -1,29 +0,0 @@
-import multiprocessing
-from os import environ as env
-
-# Port to bind to
-bind = f"{env.get('HOST')}:{int(env.get('PORT', 8000))}"
-
-# Number of processes to launch
-workers = int(env.get("WORKERS", multiprocessing.cpu_count()))
-
-# Number of concurrent handled connections
-threads = int(env.get("THREADS", 4))
-worker_connections = int(env.get("WORKER_CONNECTIONS", "1000"))
-
-# Recycle the process after X request randomized by the jitter
-max_requests = int(env.get("MAX_REQUESTS", "1000"))
-max_requests_jitter = int(env.get("MAX_REQUESTS_JITTER", "100"))
-
-# Connection timeouts
-graceful_timeout = int(env.get("GRACEFUL_TIMEOUT", "30"))
-timeout = int(env.get("TIMEOUT", "30"))
-
-# TLS/SSL Configuration
-certfile = env.get("CERTFILE")
-keyfile = env.get("KEYFILE")
-
-# Request Max Size Configuration
-limit_request_line = int(env.get("LIMIT_REQUEST_LINE", "4094"))
-limit_request_fields = int(env.get("LIMIT_REQUEST_FIELDS", "100"))
-limit_request_field_size = int(env.get("LIMIT_REQUEST_FIELD_SIZE", "8190"))

clue_api-1.0.1.dev67/clue/plugin/files/patched.py

@@ -1,5 +0,0 @@
-from gevent.monkey import patch_all
-
-patch_all()
-
-from app import app  # noqa: F401, E402

clue_api-1.0.1.dev67/clue/plugin/interactive.py

@@ -1,265 +0,0 @@
-import importlib
-import inspect
-import json
-import os
-import sys
-import textwrap
-from pathlib import Path
-from typing import Callable
-from urllib.parse import quote_plus
-
-from flask.testing import FlaskClient
-from termcolor import colored
-
-from clue.plugin import CluePlugin
-
-PLUGINS_PATH = Path(__file__).parent.parent.parent.parent / "plugins"
-sys.path.insert(0, str(PLUGINS_PATH))
-
-TESTABLE_FUNCTIONS = [
-    ("get_actions", None),
-    ("execute_action", "run_action"),
-    ("get_fetchers", None),
-    ("execute_fetcher", "run_fetcher"),
-    ("get_type_names", None),
-    ("lookup", "enrich"),
-    ("bulk_lookup", "enrich"),
-    ("liveness", None),
-    ("readyness", None),
-]
-
-
-def success(*messages: str):
-    "Print success message"
-    print(f"[{colored("success", "green")}]", *messages)
-
-
-def warn(*messages: str):
-    "Print error message"
-    print(f"[{colored("warn", "yellow")}]", *messages)
-
-
-def error(*messages: str):
-    "Print error message"
-    print(f"[{colored("error", "red")}]", *messages)
-
-
-def info(*messages: str):
-    "Print info message"
-    print(f"[{colored("info", "cyan")}]", *messages)
-
-
-class CustomTestClient(FlaskClient):
-    "Custom test client to inject authorization headers"
-
-    def open(self, *args, buffered=False, follow_redirects=False, **kwargs):
-        "Overriden open function to inject auth header"
-        headers = kwargs.setdefault("headers", {})
-
-        if "CLUE_ACCESS_TOKEN" in os.environ:
-            info("Clue access token in env, setting Authorization header")
-            headers["Authorization"] = f"Bearer {os.environ["CLUE_ACCESS_TOKEN"]}"
-        else:
-            warn("Missing access token, skipping authorization header.")
-
-        return super().open(*args, buffered=buffered, follow_redirects=follow_redirects, **kwargs)
-
-
-def filter_members(member, current_module):
-    "Get a filtered list of members exported by a given application"
-    member_module = inspect.getmodule(member)
-
-    if member_module is None:
-        return False
-
-    if member_module == current_module:
-        return True
-
-    if not member_module.__name__.startswith("clue"):
-        return False
-
-    return True
-
-
-def test_function(plugin: CluePlugin, fn_id: str, fn: Callable):  # noqa: C901
-    "test a function"
-    info(f"Executing test functionality for {fn_id}")
-
-    plugin.app.test_client_class = CustomTestClient
-
-    for rule in plugin.app.url_map.iter_rules():
-        if rule.endpoint != fn_id:
-            continue
-
-        if "GET" in (rule.methods or {}) and "<" not in rule.rule:
-            info("Simple endpoint detected. Running GET")
-            response = plugin.app.test_client().get(rule.rule)
-            info("Response:", json.dumps(response.json, indent=2) if response.json else response.data.decode())
-        elif "GET" in (rule.methods or {}):
-            kwargs: dict[str, str] = {}
-            info(f"{len(rule.arguments)} arguments are necessary. Supply them now:")
-            for argument in sorted(list(rule.arguments)):
-                kwargs[argument] = quote_plus(quote_plus(input(f"{argument}: ")))
-
-            with plugin.app.test_request_context():
-                path = plugin.app.url_for(fn_id, **kwargs)  # type: ignore[arg-type]
-                info(f"Making request to path {path}")
-
-                response = plugin.app.test_client().get(path)
-
-                if response.status_code > 299:
-                    error(
-                        (response.json or {}).get(
-                            "api_error_message", f"An unknown error occurred. Full response:\n{response.text}"
-                        )
-                    )
-                else:
-                    info("Response:", json.dumps(response.json, indent=2) if response.json else response.data.decode())
-        elif "POST" in (rule.methods or {}):
-            kwargs: dict[str, str] = {}
-            if "<" in rule.rule:
-                info(f"{len(rule.arguments)} arguments are necessary. Supply them now:")
-                for argument in sorted(list(rule.arguments)):
-                    kwargs[argument] = quote_plus(quote_plus(input(f"{argument}: ")))
-
-            info(
-                "Endpoint requires POST data. You can probide a JSON file for this data. "
-                f"Provide a path relative to {os.getcwd()} or an absolute path."
-            )
-            json_path = Path(os.getcwd()) / input("Path to JSON: ").strip()
-
-            if not json_path.exists() or json_path.is_dir():
-                error(f"Provided path {json_path} is invalid or is a directory.")
-                return
-
-            with json_path.open("r") as _data, plugin.app.test_request_context():
-                try:
-                    post_data = json.load(_data)
-                except json.JSONDecodeError:
-                    error(f"The file data in {json_path} is not valid JSON.")
-                    return
-
-                api_path = rule.rule if "<" not in rule.rule else plugin.app.url_for(fn_id, **kwargs)  # type: ignore[arg-type]
-
-                info(f"Submitting POST request to {api_path}:\n{post_data}")
-
-                response = plugin.app.test_client().post(
-                    api_path,
-                    data=json.dumps(post_data),
-                    headers={"Content-Type": "application/json"},
-                )
-
-                if response.status_code > 299:
-                    error(
-                        (response.json or {}).get(
-                            "api_error_message", f"An unknown error occurred. Full response:\n{response.text}"
-                        )
-                    )
-                else:
-                    info("Response:", json.dumps(response.json, indent=2) if response.json else response.data.decode())
-
-
-def main():  # noqa: C901
-    "main interactive loop"
-    os.environ["ENABLE_CACHE"] = "false"
-
-    plugin_name = None
-    if len(sys.argv) > 1:
-        plugin_name = sys.argv[1]
-
-        if not (PLUGINS_PATH / plugin_name).exists():
-            error(f"Plugin {plugin_name} does not exist.")
-            plugin_name = None
-
-    while plugin_name is None:
-        plugin_name = input("What plugin do you want to interact with?\n> ")
-        if not (Path(__file__).parent.parent.parent / "plugins" / plugin_name).exists():
-            error(f"Plugin {plugin_name} does not exist.")
-            plugin_name = None
-
-    try:
-        _module = importlib.import_module(f"{plugin_name}.app")
-        success(f"Initializing plugin {plugin_name} for interactivity")
-    except Exception:
-        error(f"Initializing plugin {plugin_name} for interactivity")
-        raise
-
-    plugin: CluePlugin | None = None
-    for key, member in inspect.getmembers(_module, predicate=lambda _m: filter_members(_m, _module)):
-        if isinstance(member, CluePlugin):
-            success(f"Plugin found exported as member {key}")
-            plugin = member
-            break
-
-    if plugin is None:
-        error("CluePlugin object is not exported from this module!")
-        return
-
-    plugin.cache = None
-
-    functions: list[tuple[str, Callable]] = []
-
-    for attribute in dir(plugin):
-        test_entry = next((entry for entry in TESTABLE_FUNCTIONS if entry[0] == attribute), None)
-        if test_entry is None:
-            continue
-
-        fn = plugin.__getattribute__(attribute)
-        if fn is None:
-            continue
-
-        if test_entry[1] is not None:
-            helper_fn = plugin.__getattribute__(test_entry[1])
-
-            if helper_fn is None:
-                continue
-
-        functions.append((attribute, fn))
-
-    choice: int | None = None
-
-    print(
-        textwrap.dedent("""
-        Clue Plugin Development Script
-
-        This script will help you test various aspects of your plugin interactively.
-        """),
-    )
-
-    if "CLUE_ACCESS_TOKEN" not in os.environ:
-        warn(
-            textwrap.dedent("""
-            Environment variable CLUE_ACCESS_TOKEN not set!
-
-            It is highly likely your plugin will not work if it connects to an external service.
-            """).strip()  # noqa: E501
-        )
-
-    while choice is None:
-        print("\nAvailable functions:")
-
-        for i in range(len(functions)):
-            print(f"{i + 1}) {' '.join(word.capitalize() for word in functions[i][0].split("_"))}")
-        print(f"{len(functions) + 1}) Quit")
-
-        action = input("\nEnter a selection: ")
-
-        try:
-            choice = int(action)
-
-            if choice > len(functions) + 1:
-                error(f"Invalid choice, choose option between 1 - {len(functions)}.")
-                choice = None
-        except ValueError:
-            error(f"Invalid integer, choose option between 1 - {len(functions)}.")
-
-        if choice is not None and choice <= len(functions):
-            test_function(plugin, *functions[choice - 1])
-            choice = None
-
-
-if __name__ == "__main__":
-    try:
-        main()
-    except KeyboardInterrupt:
-        print("\rExiting!" + " " * 80)