gulp-cli 1.1.4__tar.gz → 1.1.5__tar.gz

{gulp_cli-1.1.4/src/gulp_cli.egg-info → gulp_cli-1.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gulp-cli
-Version: 1.1.4
+Version: 1.1.5
 Summary: Command-line client for gULP
 Author-email: Mentat <info@mentat.is>
 Requires-Python: >=3.12

{gulp_cli-1.1.4 → gulp_cli-1.1.5}/docs/command-reference.md

@@ -1298,6 +1298,7 @@ gulp-cli enrich documents OPERATION_ID [OPTIONS]
 - `--plugin-params TEXT` — Plugin parameters JSON
 - `--flt TEXT` — Filter JSON
 - `--wait` — Wait for completion
+- `--fields TEXT` — { "field": "value" } pairs to set on enriched documents (JSON string): if "value" is `null` the enrichment is made on field's value in the document. either, enriched "value" is used.
 
 **Examples:**
 
@@ -1305,7 +1306,8 @@ gulp-cli enrich documents OPERATION_ID [OPTIONS]
 gulp-cli enrich documents my_op \
   --plugin my_enricher \
   --plugin-params '{"api_key":"..."}'  \
-  --flt '{"enriched":false}'
+  --flt '{"operation_ids":["my_op"],"source_ids":["security"]}' \
+  --fields '{"ip.destination": null}'
 ```
 
 ---
@@ -1327,12 +1329,13 @@ gulp-cli enrich single-id OPERATION_ID DOCUMENT_ID [OPTIONS]
 
 - `--plugin TEXT` — Plugin name (required)
 - `--plugin-params TEXT` — Plugin parameters JSON
+- `--fields TEXT` — { "field": "value" } pairs to set on enriched documents (JSON string): if "value" is `null` the enrichment is made on field's value in the document. either, enriched "value" is used.
 - `--wait` — Wait for completion
 
 **Examples:**
 
 ```bash
-gulp-cli enrich single-id my_op doc123 --plugin my_enricher
+gulp-cli enrich single-id my_op doc123 --plugin my_enricher --fields '{"host.os": null}' --wait
 ```
 
 ---

{gulp_cli-1.1.4 → gulp_cli-1.1.5}/docs/examples.md

@@ -45,6 +45,7 @@
     - [Run Rule on Specific Source](#run-rule-on-specific-source)
     - [Multiple Sigma Rules in Batch](#multiple-sigma-rules-in-batch)
   - [Enrichment \& Tagging Workflows](#enrichment--tagging-workflows)
+    - [use whois to enrich IP addresses with domain information](#use-whois-to-enrich-ip-addresses-with-domain-information)
     - [Tag Suspicious Events](#tag-suspicious-events)
     - [Apply Threat Level Classification](#apply-threat-level-classification)
     - [Track Analysis Progress](#track-analysis-progress)
@@ -613,6 +614,16 @@ done
 
 ## Enrichment & Tagging Workflows
 
+### use whois to enrich IP addresses with domain information
+
+```bash
+gulp-cli enrich documents incident-001 \
+  --plugin enrich_whois \
+  --flt '{"operation_ids":["network"]}' \
+  --fields '{"ip.destination": null}' \
+  --wait
+```
+
 ### Tag Suspicious Events
 
 ```bash

{gulp_cli-1.1.4 → gulp_cli-1.1.5}/src/gulp_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.1.4'
-__version_tuple__ = version_tuple = (1, 1, 4)
+__version__ = version = '1.1.5'
+__version_tuple__ = version_tuple = (1, 1, 5)
 
-__commit_id__ = commit_id = 'g8cbdcc79a'
+__commit_id__ = commit_id = 'g8a6d297b4'

{gulp_cli-1.1.4 → gulp_cli-1.1.5}/src/gulp_cli/cli.py

@@ -7,6 +7,7 @@ from gulp_cli.commands.auth import app as auth_app
 from gulp_cli.commands.collab import app as collab_app
 from gulp_cli.commands.context import app as context_app
 from gulp_cli.commands.db import app as db_app
+from gulp_cli.commands.enrich import app as enrich_app
 from gulp_cli.commands.enhance_map import app as enhance_map_app
 from gulp_cli.commands.glyph import app as glyph_app
 from gulp_cli.commands.ingest import app as ingest_app
@@ -93,6 +94,7 @@ app.add_typer(stats_app, name="stats")
 app.add_typer(storage_app, name="storage")
 app.add_typer(collab_app, name="collab")
 app.add_typer(db_app, name="db")
+app.add_typer(enrich_app, name="enrich")
 app.add_typer(acl_app, name="acl")
 app.add_typer(plugin_app, name="plugin")
 app.add_typer(mapping_app, name="mapping")
@@ -114,6 +116,7 @@ load_extensions(
         "storage": storage_app,
         "collab": collab_app,
         "db": db_app,
+        "enrich": enrich_app,
         "acl": acl_app,
         "plugin": plugin_app,
         "mapping": mapping_app,

gulp_cli-1.1.5/src/gulp_cli/commands/enrich.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import typer
+
+from gulp_cli.client import get_client
+from gulp_cli.output import print_result
+from gulp_cli.utils import comma_split, parse_json_option
+
+app = typer.Typer(help="Enrichment and tagging commands")
+
+
+@app.command("tag")
+def enrich_tag(
+    operation_id: str,
+    tags: list[str] = typer.Option(..., "--tag", help="Tag to add (repeatable)"),
+    flt: str | None = typer.Option(
+        None, "--flt", help="JSON object for GulpQueryFilter"
+    ),
+    wait: bool = typer.Option(False, "--wait", help="Wait for the async job to finish"),
+) -> None:
+    """Add tags to documents matching a filter."""
+
+    async def _run() -> None:
+        flt_parsed = parse_json_option(flt, field_name="flt")
+        async with get_client() as client:
+            result = await client.enrich.tag_documents(
+                operation_id=operation_id,
+                tags=tags,
+                flt=flt_parsed,
+                wait=wait,
+            )
+            print_result(result)
+
+    asyncio.run(_run())
+
+
+@app.command("untag")
+def enrich_untag(
+    operation_id: str,
+    tags: list[str] = typer.Option(..., "--tag", help="Tag to remove (repeatable)"),
+    flt: str | None = typer.Option(
+        None, "--flt", help="JSON object for GulpQueryFilter"
+    ),
+    wait: bool = typer.Option(False, "--wait", help="Wait for the async job to finish"),
+) -> None:
+    """Remove tags from documents matching a filter."""
+
+    async def _run() -> None:
+        flt_parsed = parse_json_option(flt, field_name="flt")
+        async with get_client() as client:
+            result = await client.enrich.untag_documents(
+                operation_id=operation_id,
+                tags=tags,
+                flt=flt_parsed,
+                wait=wait,
+            )
+            print_result(result)
+
+    asyncio.run(_run())
+
+
+@app.command("update")
+def enrich_update(
+    operation_id: str,
+    fields: str = typer.Option(..., "--fields", help="JSON object of field updates"),
+    flt: str | None = typer.Option(
+        None, "--flt", help="JSON object for GulpQueryFilter"
+    ),
+    wait: bool = typer.Option(False, "--wait", help="Wait for the async job to finish"),
+) -> None:
+    """Update fields on documents matching a filter."""
+
+    async def _run() -> None:
+        fields_parsed = parse_json_option(fields, field_name="fields")
+        if fields_parsed is None:
+            raise typer.BadParameter("--fields is required")
+        flt_parsed = parse_json_option(flt, field_name="flt")
+        async with get_client() as client:
+            result = await client.enrich.update_documents(
+                operation_id=operation_id,
+                fields=fields_parsed,
+                flt=flt_parsed,
+                wait=wait,
+            )
+            print_result(result)
+
+    asyncio.run(_run())
+
+
+@app.command("remove")
+def enrich_remove(
+    operation_id: str,
+    fields: str = typer.Option(
+        ..., "--fields", help="Comma-separated field names to remove"
+    ),
+    flt: str | None = typer.Option(
+        None, "--flt", help="JSON object for GulpQueryFilter"
+    ),
+    wait: bool = typer.Option(False, "--wait", help="Wait for the async job to finish"),
+) -> None:
+    """Remove enrichment markers or specific fields from documents matching a filter."""
+
+    async def _run() -> None:
+        flt_parsed = parse_json_option(flt, field_name="flt")
+        field_names = comma_split(fields)
+        async with get_client() as client:
+            result = await client.enrich.enrich_remove(
+                operation_id=operation_id,
+                fields=field_names or None,
+                flt=flt_parsed,
+                wait=wait,
+            )
+            print_result(result)
+
+    asyncio.run(_run())
+
+
+@app.command("documents")
+def enrich_documents(
+    operation_id: str,
+    plugin: str = typer.Option(..., "--plugin", help="Enrichment plugin name"),
+    fields: str | None = typer.Option(
+        None, "--fields", help="JSON object of fields to enrich"
+    ),
+    flt: str | None = typer.Option(
+        None, "--flt", help="JSON object for GulpQueryFilter"
+    ),
+    plugin_params: str | None = typer.Option(
+        None, "--plugin-params", help="JSON object for GulpPluginParameters"
+    ),
+    wait: bool = typer.Option(False, "--wait", help="Wait for the async job to finish"),
+) -> None:
+    """Run an enrichment plugin on documents matching a filter."""
+
+    async def _run() -> None:
+        fields_parsed = parse_json_option(fields, field_name="fields")
+        flt_parsed = parse_json_option(flt, field_name="flt")
+        plugin_params_parsed = parse_json_option(
+            plugin_params, field_name="plugin-params"
+        )
+        async with get_client() as client:
+            result = await client.enrich.enrich_documents(
+                operation_id=operation_id,
+                plugin=plugin,
+                fields=fields_parsed,
+                flt=flt_parsed,
+                plugin_params=plugin_params_parsed,
+                wait=wait,
+            )
+            print_result(result)
+
+    asyncio.run(_run())
+
+
+@app.command("single-id")
+def enrich_single_id(
+    operation_id: str,
+    doc_id: str,
+    plugin: str = typer.Option(..., "--plugin", help="Enrichment plugin name"),
+    fields: str | None = typer.Option(
+        None, "--fields", help="JSON object of fields to enrich"
+    ),
+    plugin_params: str | None = typer.Option(
+        None, "--plugin-params", help="JSON object for GulpPluginParameters"
+    ),
+) -> None:
+    """Enrich a single document by OpenSearch document ID."""
+
+    async def _run() -> None:
+        fields_parsed = parse_json_option(fields, field_name="fields")
+        plugin_params_parsed = parse_json_option(
+            plugin_params, field_name="plugin-params"
+        )
+        async with get_client() as client:
+            result = await client.enrich.enrich_single_id(
+                operation_id=operation_id,
+                doc_id=doc_id,
+                plugin=plugin,
+                fields=fields_parsed,
+                plugin_params=plugin_params_parsed,
+            )
+            print_result(result)
+
+    asyncio.run(_run())

{gulp_cli-1.1.4 → gulp_cli-1.1.5}/src/gulp_cli/commands/ingest.py

@@ -948,7 +948,10 @@ def ingest_file(
                 return
 
             if reset_operation:
-                await client.operations.delete(operation_id, force=True)
+                try:
+                    await client.operations.delete(operation_id, force=True)
+                except NotFoundError:
+                    pass
                 op = await client.operations.create(name=operation_id)
                 if not get_runtime_verbose():
                     print_warning(

{gulp_cli-1.1.4 → gulp_cli-1.1.5/src/gulp_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gulp-cli
-Version: 1.1.4
+Version: 1.1.5
 Summary: Command-line client for gULP
 Author-email: Mentat <info@mentat.is>
 Requires-Python: >=3.12

{gulp_cli-1.1.4 → gulp_cli-1.1.5}/src/gulp_cli.egg-info/SOURCES.txt

@@ -40,6 +40,7 @@ src/gulp_cli/commands/collab.py
 src/gulp_cli/commands/context.py
 src/gulp_cli/commands/db.py
 src/gulp_cli/commands/enhance_map.py
+src/gulp_cli/commands/enrich.py
 src/gulp_cli/commands/glyph.py
 src/gulp_cli/commands/ingest.py
 src/gulp_cli/commands/mapping.py
@@ -53,4 +54,5 @@ src/gulp_cli/commands/user_group.py
 src/gulp_cli/commands/users.py
 src/gulp_cli/extension/__init__.py
 src/gulp_cli/extension/query_sigma_zip.py
-src/gulp_cli/extension/story.py
+src/gulp_cli/extension/story.py
+tests/test_enrich_live.py

gulp_cli-1.1.5/tests/test_enrich_live.py

@@ -0,0 +1,302 @@
+import asyncio
+import json
+import os
+import shutil
+import subprocess
+import sys
+import uuid
+from pathlib import Path
+
+import pytest
+
+from gulp_sdk import GulpClient
+
+CLI_BIN = Path("/gulp/.venv/bin/gulp-cli")
+SAMPLE_EVTX = Path("/gulp/samples/win_evtx/Security_short_selected.evtx")
+
+
+def _unique(prefix: str) -> str:
+    return f"{prefix}_{uuid.uuid4().hex[:8]}"
+
+
+def _run_cli(*args: str, config_dir: Path) -> subprocess.CompletedProcess[str]:
+    env = os.environ.copy()
+    env["GULP_CLI_HOME"] = str(config_dir)
+    return subprocess.run(
+        [str(CLI_BIN), *args],
+        cwd="/gulp",
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+
+async def _setup_sample_docs() -> tuple[str, str]:
+    async with GulpClient("http://localhost:8080") as client:
+        await client.auth.login("admin", "admin")
+        operation_id = _unique("cli_enrich_op")
+        operation = await client.operations.create(operation_id)
+        ingest = await client.ingest.file(
+            operation_id=operation.id,
+            plugin_name="win_evtx",
+            file_path=str(SAMPLE_EVTX),
+            context_name="cli_enrich_context",
+            wait=True,
+        )
+        assert str(getattr(ingest, "status", "")).lower() in {"done", "pending"}
+
+        preview = await client.queries.query_raw(
+            operation_id=operation.id,
+            q=[{"query": {"match_all": {}}}],
+            q_options={"preview_mode": True, "limit": 1, "name": "cli_enrich_preview"},
+        )
+        docs = (preview.get("data") or {}).get("docs") or []
+        assert docs, "Expected at least one sample document after ingestion"
+        doc_id = docs[0].get("_id") or docs[0].get("id")
+        assert doc_id, "Could not determine document id for CLI enrichment test"
+        return operation.id, str(doc_id)
+
+
+def _load_saved_session(config_dir: Path) -> tuple[str, str]:
+    config = json.loads((config_dir / "config.json").read_text(encoding="utf-8"))
+    active_user = str(config.get("active_user") or "")
+    sessions = config.get("sessions") or {}
+    session = sessions.get(active_user) or next(iter(sessions.values()), None)
+    assert isinstance(session, dict), "Expected a saved CLI session for the live test"
+    url = str(session.get("url") or "").strip()
+    token = str(session.get("token") or "").strip()
+    assert url and token, "Saved CLI session is missing url or token"
+    return url, token
+
+
+async def _get_first_doc_id(operation_id: str, config_dir: Path) -> str:
+    url, token = _load_saved_session(config_dir)
+    async with GulpClient(url, token=token) as client:
+        preview = await client.queries.query_raw(
+            operation_id=operation_id,
+            q=[{"query": {"match_all": {}}}],
+            q_options={"preview_mode": True, "limit": 1, "name": "cli_enrich_preview"},
+        )
+        docs = (preview.get("data") or {}).get("docs") or []
+        assert docs, "Expected at least one sample document after CLI ingestion"
+        doc_id = docs[0].get("_id") or docs[0].get("id")
+        assert doc_id, "Could not determine document id for CLI enrichment test"
+        return str(doc_id)
+
+
+async def _cleanup_operation(operation_id: str) -> None:
+    async with GulpClient("http://localhost:8080") as client:
+        await client.auth.login("admin", "admin")
+        await client.operations.delete(operation_id)
+
+
+@pytest.mark.integration
+def test_enrich_cli_tag_update_remove_and_untag(tmp_path: Path) -> None:
+    config_dir = tmp_path / "cli-config"
+    config_dir.mkdir(parents=True, exist_ok=True)
+
+    login = _run_cli(
+        "auth",
+        "login",
+        "--url",
+        "http://localhost:8080",
+        "--username",
+        "admin",
+        "--password",
+        "admin",
+        config_dir=config_dir,
+    )
+    assert login.returncode == 0, login.stderr or login.stdout
+
+    operation_id = _unique("cli_enrich_op")
+
+    ingest_result = _run_cli(
+        "ingest",
+        "file",
+        operation_id,
+        "win_evtx",
+        str(SAMPLE_EVTX),
+        "--context-name",
+        "test_context",
+        "--reset-operation",
+        "--batch-size",
+        "128",
+        "--wait",
+        config_dir=config_dir,
+    )
+    assert ingest_result.returncode == 0, ingest_result.stderr or ingest_result.stdout
+
+    doc_id = asyncio.run(_get_first_doc_id(operation_id, config_dir))
+
+    try:
+        tag_result = _run_cli(
+            "enrich",
+            "tag",
+            operation_id,
+            "--tag",
+            "cli_enrich_tag",
+            "--flt",
+            json.dumps({"operation_ids": [operation_id]}),
+            "--wait",
+            config_dir=config_dir,
+        )
+        assert tag_result.returncode == 0, tag_result.stderr or tag_result.stdout
+        assert "RuntimeWarning" not in (tag_result.stderr or "")
+
+        update_result = _run_cli(
+            "enrich",
+            "update",
+            operation_id,
+            "--fields",
+            json.dumps({"cli_enrich_field": True}),
+            "--flt",
+            json.dumps({"operation_ids": [operation_id]}),
+            "--wait",
+            config_dir=config_dir,
+        )
+        assert update_result.returncode == 0, (
+            update_result.stderr or update_result.stdout
+        )
+
+        remove_result = _run_cli(
+            "enrich",
+            "remove",
+            operation_id,
+            "--fields",
+            "cli_enrich_field",
+            "--flt",
+            json.dumps({"operation_ids": [operation_id]}),
+            "--wait",
+            config_dir=config_dir,
+        )
+        assert remove_result.returncode == 0, (
+            remove_result.stderr or remove_result.stdout
+        )
+
+        untag_result = _run_cli(
+            "enrich",
+            "untag",
+            operation_id,
+            "--tag",
+            "cli_enrich_tag",
+            "--flt",
+            json.dumps({"operation_ids": [operation_id]}),
+            "--wait",
+            config_dir=config_dir,
+        )
+        assert untag_result.returncode == 0, untag_result.stderr or untag_result.stdout
+
+        async def _verify() -> None:
+            async with GulpClient("http://localhost:8080") as client:
+                await client.auth.login("admin", "admin")
+                fetched = await client.queries.query_single_id(operation_id, doc_id)
+                assert isinstance(fetched, dict)
+                tags = fetched.get("gulp.tags") or fetched.get("gulp", {}).get(
+                    "tags", []
+                )
+                assert "cli_enrich_tag" not in tags
+                assert "cli_enrich_field" not in fetched
+
+        asyncio.run(_verify())
+    finally:
+        try:
+            asyncio.run(_cleanup_operation(operation_id))
+        except Exception:
+            pass
+        try:
+            shutil.rmtree(config_dir, ignore_errors=True)
+        except Exception:
+            pass
+
+
+@pytest.mark.integration
+def test_enrich_cli_documents_and_single_id(tmp_path: Path) -> None:
+    config_dir = tmp_path / "cli-config"
+    config_dir.mkdir(parents=True, exist_ok=True)
+
+    login = _run_cli(
+        "auth",
+        "login",
+        "--url",
+        "http://localhost:8080",
+        "--username",
+        "admin",
+        "--password",
+        "admin",
+        config_dir=config_dir,
+    )
+    assert login.returncode == 0, login.stderr or login.stdout
+
+    operation_id, doc_id = asyncio.run(_setup_sample_docs())
+
+    login = _run_cli(
+        "auth",
+        "login",
+        "--url",
+        "http://localhost:8080",
+        "--username",
+        "admin",
+        "--password",
+        "admin",
+        config_dir=config_dir,
+    )
+    assert login.returncode == 0, login.stderr or login.stdout
+
+    try:
+        documents_result = _run_cli(
+            "enrich",
+            "documents",
+            operation_id,
+            "--plugin",
+            "enrich_whois",
+            "--fields",
+            json.dumps({"source.ip": None}),
+            "--flt",
+            json.dumps({"operation_ids": [operation_id]}),
+            "--plugin-params",
+            json.dumps({"custom_parameters": {}}),
+            "--wait",
+            config_dir=config_dir,
+        )
+
+        if documents_result.returncode != 0:
+            pytest.skip(
+                f"enrich_whois unavailable for CLI documents test: {documents_result.stderr or documents_result.stdout}"
+            )
+
+        single_result = _run_cli(
+            "enrich",
+            "single-id",
+            operation_id,
+            doc_id,
+            "--plugin",
+            "enrich_whois",
+            "--fields",
+            json.dumps({"sdk_cli_whois": "8.8.8.8"}),
+            "--plugin-params",
+            json.dumps({"custom_parameters": {}}),
+            config_dir=config_dir,
+        )
+        if single_result.returncode != 0:
+            pytest.skip(
+                f"enrich_whois unavailable for CLI single-id test: {single_result.stderr or single_result.stdout}"
+            )
+
+        async def _verify() -> None:
+            async with GulpClient("http://localhost:8080") as client:
+                await client.auth.login("admin", "admin")
+                fetched = await client.queries.query_single_id(operation_id, doc_id)
+                assert isinstance(fetched, dict)
+                assert fetched.get("_id") == doc_id or fetched.get("id") == doc_id
+
+        asyncio.run(_verify())
+    finally:
+        try:
+            asyncio.run(_cleanup_operation(operation_id))
+        except Exception:
+            pass
+        try:
+            shutil.rmtree(config_dir, ignore_errors=True)
+        except Exception:
+            pass