gulp-cli 1.0.7__tar.gz → 1.0.9__tar.gz

{gulp_cli-1.0.7/src/gulp_cli.egg-info → gulp_cli-1.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gulp-cli
-Version: 1.0.7
+Version: 1.0.9
 Summary: Command-line client for gULP
 Author-email: Mentat <info@mentat.is>
 Requires-Python: >=3.12

{gulp_cli-1.0.7 → gulp_cli-1.0.9}/docs/command-reference.md

@@ -520,6 +520,8 @@ gulp-cli ingest file-to-source source123 '/new/*.evtx'
 
 Ingest from a ZIP archive.
 
+> TO BE DEPRECATED...
+
 ```bash
 gulp-cli ingest zip OPERATION_ID ZIP_FILE [OPTIONS]
 ```
@@ -527,7 +529,7 @@ gulp-cli ingest zip OPERATION_ID ZIP_FILE [OPTIONS]
 **Arguments:**
 
 - `OPERATION_ID` — Target operation
-- `ZIP_FILE` — Path to ZIP file
+- `ZIP_FILE` — Path to ZIP file containing a `metadata.json` in the root, describing the content as specified in gulp's `ingest_zip` docs.
 
 **Options:**
 
@@ -546,6 +548,47 @@ gulp-cli ingest zip my_op /data/evidence.zip --create-operation
 
 ---
 
+#### `zip-create`
+
+Create a ZIP archive from one or more source path expressions.
+
+Path expressions may be:
+
+- a single file path
+- a directory path
+- a glob mask (for example `/bla/somef*.txt`, `/bla/*`, `**/*.evtx`)
+
+Environment variables and home shortcuts are expanded in all input paths (for example `$EVIDENCE_DIR/*.evtx`, `~/cases/case-01/*`).
+
+```bash
+gulp-cli ingest zip-create OUTPUT_ZIP [PATH_PATTERN...] [OPTIONS]
+```
+
+**Arguments:**
+
+- `OUTPUT_ZIP` — Destination ZIP path (supports environment variables and `~`)
+- `PATH_PATTERN` — File, directory, or glob path expression (multiple allowed)
+
+**Options:**
+
+- `--paths-file TEXT` — Text file with one source path expression per line (supports environment variables and `~` per line)
+- `--overwrite` — Overwrite output ZIP if it already exists
+
+**Examples:**
+
+```bash
+# Create a ZIP from mixed path expressions
+gulp-cli ingest zip-create /tmp/evidence.zip /data/host1/*.evtx /data/host2 /data/readme.txt
+
+# Use environment variables and home shortcuts
+gulp-cli ingest zip-create '$CASE_DIR/evidence.zip' '$CASE_DIR/raw/*.json' '~/pcaps/*.pcap' --overwrite
+
+# Read source path expressions from a file
+gulp-cli ingest zip-create /tmp/evidence.zip --paths-file /tmp/evidence_paths.txt --overwrite
+```
+
+---
+
 #### `raw`
 
 Ingest raw payload chunks into an operation.

{gulp_cli-1.0.7 → gulp_cli-1.0.9}/docs/examples.md

@@ -9,6 +9,7 @@
     - [JSON Logs Ingestion](#json-logs-ingestion)
     - [Add More Evidence to Existing Source](#add-more-evidence-to-existing-source)
     - [ZIP Archive Ingestion](#zip-archive-ingestion)
+    - [Create ZIP Archive from Paths and Masks](#create-zip-archive-from-paths-and-masks)
     - [Raw Payload Ingestion](#raw-payload-ingestion)
   - [Request Stats Monitoring Workflows](#request-stats-monitoring-workflows)
     - [Monitor Ongoing Requests (Live)](#monitor-ongoing-requests-live)
@@ -122,7 +123,7 @@ gulp-cli auth logout
 
 ---
 
-  ## Ingestion Workflows
+## Ingestion Workflows
 
 ### Single File Ingestion
 
@@ -171,6 +172,14 @@ wait  # Wait for all background jobs
 # Ingest CSV with specific delimiter and encoding
 gulp-cli ingest file incident-001 csv /data/access_log.csv \
   --plugin-params '{"delimiter":";","encoding":"iso-8859-1","has_header":true}'
+
+# pass mapping directly without using a mapping file
+gulp-cli ingest file test_operation csv ./samples/mftecmd/sample_record.csv --plugin-params '{ "mapping_parameters": { "mappings": { "test
+_mapping": { "fields": { "Created0x10": { "ecs": [ "@timestamp" ] } } } } } }' --reset-operation --wait
+
+# pass mapping using a gulp mapping file with mapping_id to specify which mapping to use in the file
+gulp-cli ingest file test_operation csv ./samples/mftecmd/sample_record.csv --plugin-params '{ "mapping_parameters"
+: { "mapping_file": "mftecmd_csv.json", "mapping_id": "record" } }' --wait --reset-operation
 ```
 
 ### JSON Logs Ingestion
@@ -206,6 +215,29 @@ gulp-cli ingest zip incident-001 /evidence/evidence.zip --wait
 gulp-cli ingest zip incident-001 /evidence/evidence.zip --create-operation
 ```
 
+### Create ZIP Archive from Paths and Masks
+
+```bash
+# Build ZIP from mixed sources: file, directory, and wildcard mask
+gulp-cli ingest zip-create /evidence/evidence.zip /forensic/host1/*.evtx /forensic/host2 /forensic/notes.txt --overwrite
+
+# Use environment variables and ~ in source expressions and output path
+export CASE_ROOT=~/cases/incident-001
+gulp-cli ingest zip-create '$CASE_ROOT/evidence.zip' '$CASE_ROOT/windows/*.evtx' '$CASE_ROOT/network/*' --overwrite
+
+# Build ZIP from a text file (one path expression per line)
+cat > /tmp/evidence_paths.txt <<'EOF'
+$CASE_ROOT/windows/*.evtx
+$CASE_ROOT/linux/**/*.log
+~/captures/*.pcap
+EOF
+
+gulp-cli ingest zip-create '$CASE_ROOT/evidence.zip' --paths-file /tmp/evidence_paths.txt --overwrite
+
+# Then ingest the generated ZIP
+gulp-cli ingest zip incident-001 '$CASE_ROOT/evidence.zip' --wait
+```
+
 ### Raw Payload Ingestion
 
 ```bash
@@ -505,6 +537,7 @@ gulp-cli query external incident-001 \
   --q '{"query":{"match_all":{}}}' \
   --preview --limit 100 --offset 0
 ```
+
 ```
 
 ### Export Query Results
@@ -967,7 +1000,7 @@ gulp-cli acl make-private link-456 --obj-type link
 
 ---
 
-## Index Management Workflows 
+## Index Management Workflows
 
 ### Inspect Indexes
 
@@ -1093,7 +1126,7 @@ gulp-cli enhance-map delete <enhance_map_obj_id>
 ### Query Sigma Rules from ZIP (Extension)
 
 > needs non-free `query_sigma_zip` plugin to be installed on the server, this is provided just as an example.
- 
+
 ```bash
 # Execute all Sigma rules inside a zip archive
 gulp-cli query sigma-zip incident-001 \

{gulp_cli-1.0.7 → gulp_cli-1.0.9}/src/gulp_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.7'
-__version_tuple__ = version_tuple = (1, 0, 7)
+__version__ = version = '1.0.9'
+__version_tuple__ = version_tuple = (1, 0, 9)
 
-__commit_id__ = commit_id = 'g6e69ea4d4'
+__commit_id__ = commit_id = 'gb7a79dce7'

{gulp_cli-1.0.7 → gulp_cli-1.0.9}/src/gulp_cli/commands/ingest.py

@@ -2,7 +2,9 @@ from __future__ import annotations
 
 import asyncio
 import json
-from glob import glob
+import os
+import zipfile
+from glob import glob, has_magic
 from pathlib import Path
 from typing import Any
 
@@ -64,6 +66,140 @@ def _expand_file_patterns(file_patterns: list[str]) -> list[str]:
     return unique_files
 
 
+def _resolve_path(raw_path: str) -> Path:
+    expanded = _expand_path_expression(raw_path)
+    return Path(expanded).resolve()
+
+
+def _expand_path_expression(raw_path: str) -> str:
+    """Expand shell-style path expressions (env vars and '~')."""
+    return os.path.expandvars(os.path.expanduser(raw_path.strip()))
+
+
+def _expand_zip_source_patterns(
+    path_patterns: list[str],
+    paths_file: str | None,
+) -> list[tuple[Path, Path]]:
+    if path_patterns and paths_file is not None:
+        raise typer.BadParameter(
+            "Provide paths either as arguments or via --paths-file, not both"
+        )
+    if not path_patterns and paths_file is None:
+        raise typer.BadParameter(
+            "Provide at least one path argument or --paths-file"
+        )
+
+    raw_entries = list(path_patterns)
+    if paths_file is not None:
+        file_path = _resolve_path(paths_file)
+        if not file_path.is_file():
+            raise typer.BadParameter(f"Paths file not found: {file_path}")
+        raw_entries.extend(
+            line.strip()
+            for line in file_path.read_text(encoding="utf-8").splitlines()
+            if line.strip() and not line.lstrip().startswith("#")
+        )
+
+    expanded_paths: list[tuple[Path, Path]] = []
+    seen: set[tuple[Path, Path]] = set()
+    for raw_entry in raw_entries:
+        expanded_entry = _expand_path_expression(raw_entry)
+        if not expanded_entry:
+            continue
+
+        if has_magic(expanded_entry):
+            pattern_path = Path(expanded_entry)
+            base_dir = _resolve_glob_base(pattern_path)
+            matches = [Path(match).resolve() for match in sorted(glob(expanded_entry, recursive=True))]
+            if not matches:
+                raise typer.BadParameter(f"Path mask did not match any files: {expanded_entry}")
+
+            for match in matches:
+                item = (match, base_dir)
+                if item not in seen:
+                    seen.add(item)
+                    expanded_paths.append(item)
+            continue
+
+        resolved_path = _resolve_path(expanded_entry)
+        if not resolved_path.exists():
+            raise typer.BadParameter(f"Path not found: {resolved_path}")
+
+        base_dir = resolved_path.parent
+        item = (resolved_path, base_dir)
+        if item not in seen:
+            seen.add(item)
+            expanded_paths.append(item)
+
+    return expanded_paths
+
+
+def _resolve_glob_base(pattern_path: Path) -> Path:
+    parts = pattern_path.parts
+    if pattern_path.is_absolute():
+        base_dir = Path(pattern_path.anchor)
+        part_index = 1
+    else:
+        base_dir = Path.cwd()
+        part_index = 0
+
+    for part in parts[part_index:]:
+        if has_magic(part):
+            break
+        base_dir /= part
+
+    return base_dir.resolve()
+
+
+def _build_zip_from_sources(output_zip: Path, sources: list[tuple[Path, Path]]) -> tuple[int, list[str]]:
+    output_zip.parent.mkdir(parents=True, exist_ok=True)
+
+    archived_entries: list[str] = []
+    archived_count = 0
+    seen_archive_entries: set[str] = set()
+    seen_source_files: set[Path] = set()
+
+    def _record_directory(archive_name_str: str, archive: zipfile.ZipFile) -> None:
+        entry_name = f"{archive_name_str}/"
+        if entry_name in seen_archive_entries:
+            return
+        archive.writestr(entry_name, "")
+        seen_archive_entries.add(entry_name)
+        archived_entries.append(entry_name)
+
+    def _record_file(source_path: Path, archive_name_str: str, archive: zipfile.ZipFile) -> None:
+        nonlocal archived_count
+        if archive_name_str in seen_archive_entries or source_path in seen_source_files:
+            return
+        archive.write(source_path, arcname=archive_name_str)
+        seen_archive_entries.add(archive_name_str)
+        seen_source_files.add(source_path)
+        archived_entries.append(archive_name_str)
+        archived_count += 1
+
+    with zipfile.ZipFile(output_zip, mode="w", compression=zipfile.ZIP_DEFLATED) as archive:
+        for source_path, base_dir in sources:
+            archive_root = source_path.relative_to(base_dir)
+
+            if source_path.is_file():
+                _record_file(source_path, archive_root.as_posix(), archive)
+                continue
+
+            child_paths = sorted(source_path.rglob("*"))
+            if not child_paths:
+                _record_directory(archive_root.as_posix(), archive)
+                continue
+
+            for current_path in child_paths:
+                archive_name_str = current_path.relative_to(base_dir).as_posix()
+                if current_path.is_dir():
+                    _record_directory(archive_name_str, archive)
+                    continue
+                _record_file(current_path, archive_name_str, archive)
+
+    return archived_count, archived_entries
+
+
 async def _wait_for_stats_create(
     client: Any,
     req_ids: list[str],
@@ -600,7 +736,10 @@ def ingest_file_to_source(
 @app.command("zip")
 def ingest_zip(
     operation_id: str,
-    zip_file: str,
+    zip_file: str = typer.Argument(
+        ...,
+        help="Path to a ZIP file which must contain a `metadata.json` in the root, describing the content as specified in gulp's `ingest_zip` docs.",
+    ),
     context_name: str = typer.Option("sdk_context", "--context-name"),
     flt: str | None = typer.Option(None, "--flt", help="JSON object for GulpIngestionFilter"),
     reset_operation: bool = typer.Option(
@@ -691,6 +830,52 @@ def ingest_zip(
     asyncio.run(_run())
 
 
+@app.command("zip-create")
+def ingest_zip_create(
+    output_zip: str = typer.Argument(
+        ...,
+        help="Path to the ZIP file to create (supports $VARS and ~)",
+    ),
+    path_patterns: list[str] = typer.Argument(
+        None,
+        help="One or more files, directories, or glob patterns to archive (supports $VARS and ~)",
+    ),
+    paths_file: str | None = typer.Option(
+        None,
+        "--paths-file",
+        help="Text file with one file, directory, or glob pattern per line (supports $VARS and ~ per line)",
+    ),
+    overwrite: bool = typer.Option(
+        False,
+        "--overwrite",
+        help="Overwrite the ZIP file if it already exists",
+    ),
+) -> None:
+    """Create a ZIP archive from files, directories, or glob patterns."""
+
+    output_path = _resolve_path(output_zip)
+    source_paths = _expand_zip_source_patterns(path_patterns or [], paths_file)
+
+    if output_path.exists() and not overwrite:
+        raise typer.BadParameter(
+            f"Output ZIP already exists: {output_path}. Use --overwrite to replace it"
+        )
+
+    archived_count, archived_entries = _build_zip_from_sources(output_path, source_paths)
+    print_result(
+        {
+            "zip_file": str(output_path),
+            "sources": [str(path) for path, _ in source_paths],
+            "files_archived": archived_count,
+            "entries": archived_entries,
+        },
+        formatter=lambda data: console.print(
+            f"Created ZIP {data['zip_file']} from {len(data['sources'])} source"
+            f"{'s' if len(data['sources']) != 1 else ''} with {data['files_archived']} file(s)."
+        ),
+    )
+
+
 @app.command("raw")
 def ingest_raw(
     operation_id: str,

{gulp_cli-1.0.7 → gulp_cli-1.0.9/src/gulp_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gulp-cli
-Version: 1.0.7
+Version: 1.0.9
 Summary: Command-line client for gULP
 Author-email: Mentat <info@mentat.is>
 Requires-Python: >=3.12