loggen-lg 0.1.0__tar.gz

loggen_lg-0.1.0/MANIFEST.in

@@ -0,0 +1,4 @@
+global-exclude *.pyc
+global-exclude *.pyo
+global-exclude __pycache__
+prune .vscode

loggen_lg-0.1.0/PKG-INFO

@@ -0,0 +1,141 @@
+Metadata-Version: 2.4
+Name: loggen-lg
+Version: 0.1.0
+Summary: Synthetic log generator for SIEM and IR exercises
+License-Expression: MIT
+Keywords: siem,logs,security,testing,ir,blue-team
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Information Technology
+Classifier: Topic :: Security
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# loggen
+
+Synthetic log generator for SIEM and IR exercises. Reproducible, deterministic, supports arbitrary scales from MB to TB. Profiles for Windows, Linux, network, and EDR-like telemetry.
+
+## Install
+
+```bash
+pip install -e .
+```
+
+## Quick start
+
+```python
+from loggen import LogGen, Session
+
+gen = LogGen(seed=42)
+
+# Stream 1 000 mixed entries
+for entry in gen.stream(["windows", "linux", "edr"], count=1000):
+    print(entry.asdict())
+
+# Write 1 GB of mixed logs (gzip)
+gen.write("corpus.jsonl.gz", ["windows", "linux", "network", "edr"], size="1GB")
+
+# Run a built-in attack scenario
+gen.write("attack.jsonl", [], scenario="lateral_movement", count=50)
+
+# In-memory list
+entries = gen.to_list(["network"], count=200)
+```
+
+## Custom data pools
+
+By default loggen ships with a built-in set of hostnames, users, and IPs.  
+You can replace any of them with your own data.
+
+**Inline lists**
+```python
+from loggen import Session, LogGen
+
+session = Session(
+    seed=42,
+    hosts=["prod-web-01", "prod-db-01", "prod-cache-01"],
+    users=["alice", "bob", "charlie"],
+)
+gen = LogGen(session=session)
+```
+
+**From a file**
+```python
+from loggen import Session, LogGen, WordList
+
+session = Session(
+    seed=42,
+    hosts=WordList.from_file("wordlists/hosts.txt"),
+    users=WordList.from_file("wordlists/users.json"),
+    internal_ips=WordList.from_file("wordlists/ips.csv", column=0),
+)
+gen = LogGen(session=session)
+```
+
+**From a JSON config file**
+```python
+from loggen import Session, LogGen
+
+session = Session.from_config("config/loggen.json")
+gen = LogGen(session=session)
+```
+
+See [WIKI.md](WIKI.md) for the config file format and all supported options.
+
+**From an environment variable**
+```python
+from loggen import WordList
+
+hosts = WordList.from_env("LOGGEN_HOSTS")        # comma-separated
+users = WordList.from_env("LOGGEN_USERS", fallback=["admin"])
+```
+
+## CLI
+
+```bash
+# 10 000 mixed entries to stdout (JSON Lines)
+python -m loggen generate -p windows -p linux -n 10000
+
+# 500 MB of network logs, gzip, reproducible
+python -m loggen generate -p network --size 500MB --seed 42 -o network.jsonl.gz
+
+# Brute-force scenario in CEF format
+python -m loggen generate --scenario brute_force -f cef -o brute.cef
+
+# Custom profile weights
+python -m loggen generate -p windows -p edr --weight windows:4 --weight edr:1 -n 50000 -o out.jsonl
+
+# List available options
+python -m loggen list-profiles
+python -m loggen list-scenarios
+python -m loggen list-formats
+```
+
+## Profiles
+
+| Profile   | Covers |
+|-----------|--------|
+| `windows` | Security events 4624/4625/4688/4672/4698/7045/5140/4104 … |
+| `linux`   | sshd auth, sudo, cron, systemd, auditd, PAM, kernel |
+| `network` | Firewall allow/deny, DNS, HTTP proxy, DHCP |
+| `edr`     | Process/file/network/registry/module events with hashes |
+
+## Scenarios
+
+| Scenario           | Description |
+|--------------------|-------------|
+| `brute_force`      | Repeated logon failures → success |
+| `lateral_movement` | Recon → SMB share access → remote execution |
+| `priv_esc`         | Service install → SYSTEM shell → 4672 |
+| `data_exfil`       | Archive creation → large outbound transfers → DNS tunnelling |
+| `persistence`      | Registry run key + scheduled task + binary drop |
+
+## Output formats
+
+| Format   | Description |
+|----------|-------------|
+| `jsonl`  | JSON Lines / NDJSON (default) |
+| `cef`    | ArcSight CEF:0 |
+| `syslog` | RFC 5424 syslog |
+
+See [WIKI.md](WIKI.md) for full API reference.

loggen_lg-0.1.0/README.md

@@ -0,0 +1,128 @@
+# loggen
+
+Synthetic log generator for SIEM and IR exercises. Reproducible, deterministic, supports arbitrary scales from MB to TB. Profiles for Windows, Linux, network, and EDR-like telemetry.
+
+## Install
+
+```bash
+pip install -e .
+```
+
+## Quick start
+
+```python
+from loggen import LogGen, Session
+
+gen = LogGen(seed=42)
+
+# Stream 1 000 mixed entries
+for entry in gen.stream(["windows", "linux", "edr"], count=1000):
+    print(entry.asdict())
+
+# Write 1 GB of mixed logs (gzip)
+gen.write("corpus.jsonl.gz", ["windows", "linux", "network", "edr"], size="1GB")
+
+# Run a built-in attack scenario
+gen.write("attack.jsonl", [], scenario="lateral_movement", count=50)
+
+# In-memory list
+entries = gen.to_list(["network"], count=200)
+```
+
+## Custom data pools
+
+By default loggen ships with a built-in set of hostnames, users, and IPs.  
+You can replace any of them with your own data.
+
+**Inline lists**
+```python
+from loggen import Session, LogGen
+
+session = Session(
+    seed=42,
+    hosts=["prod-web-01", "prod-db-01", "prod-cache-01"],
+    users=["alice", "bob", "charlie"],
+)
+gen = LogGen(session=session)
+```
+
+**From a file**
+```python
+from loggen import Session, LogGen, WordList
+
+session = Session(
+    seed=42,
+    hosts=WordList.from_file("wordlists/hosts.txt"),
+    users=WordList.from_file("wordlists/users.json"),
+    internal_ips=WordList.from_file("wordlists/ips.csv", column=0),
+)
+gen = LogGen(session=session)
+```
+
+**From a JSON config file**
+```python
+from loggen import Session, LogGen
+
+session = Session.from_config("config/loggen.json")
+gen = LogGen(session=session)
+```
+
+See [WIKI.md](WIKI.md) for the config file format and all supported options.
+
+**From an environment variable**
+```python
+from loggen import WordList
+
+hosts = WordList.from_env("LOGGEN_HOSTS")        # comma-separated
+users = WordList.from_env("LOGGEN_USERS", fallback=["admin"])
+```
+
+## CLI
+
+```bash
+# 10 000 mixed entries to stdout (JSON Lines)
+python -m loggen generate -p windows -p linux -n 10000
+
+# 500 MB of network logs, gzip, reproducible
+python -m loggen generate -p network --size 500MB --seed 42 -o network.jsonl.gz
+
+# Brute-force scenario in CEF format
+python -m loggen generate --scenario brute_force -f cef -o brute.cef
+
+# Custom profile weights
+python -m loggen generate -p windows -p edr --weight windows:4 --weight edr:1 -n 50000 -o out.jsonl
+
+# List available options
+python -m loggen list-profiles
+python -m loggen list-scenarios
+python -m loggen list-formats
+```
+
+## Profiles
+
+| Profile   | Covers |
+|-----------|--------|
+| `windows` | Security events 4624/4625/4688/4672/4698/7045/5140/4104 … |
+| `linux`   | sshd auth, sudo, cron, systemd, auditd, PAM, kernel |
+| `network` | Firewall allow/deny, DNS, HTTP proxy, DHCP |
+| `edr`     | Process/file/network/registry/module events with hashes |
+
+## Scenarios
+
+| Scenario           | Description |
+|--------------------|-------------|
+| `brute_force`      | Repeated logon failures → success |
+| `lateral_movement` | Recon → SMB share access → remote execution |
+| `priv_esc`         | Service install → SYSTEM shell → 4672 |
+| `data_exfil`       | Archive creation → large outbound transfers → DNS tunnelling |
+| `persistence`      | Registry run key + scheduled task + binary drop |
+
+## Output formats
+
+| Format   | Description |
+|----------|-------------|
+| `jsonl`  | JSON Lines / NDJSON (default) |
+| `cef`    | ArcSight CEF:0 |
+| `syslog` | RFC 5424 syslog |
+
+See [WIKI.md](WIKI.md) for full API reference.

loggen_lg-0.1.0/loggen-lg/__init__.py

@@ -0,0 +1,32 @@
+from loggen._version import VERSION
+from loggen.core.session import Session
+from loggen.core.generator import LogGen
+from loggen.core.loaders import WordList
+from loggen.profiles.base import LogEntry
+from loggen.profiles import (
+    WindowsProfile,
+    LinuxProfile,
+    NetworkProfile,
+    EDRProfile,
+    PROFILE_REGISTRY,
+)
+from loggen.formats import Format, get_formatter
+from loggen.scenarios import SCENARIO_REGISTRY
+
+__version__ = VERSION
+
+__all__ = [
+    "LogGen",
+    "Session",
+    "LogEntry",
+    "WordList",
+    "Format",
+    "get_formatter",
+    "WindowsProfile",
+    "LinuxProfile",
+    "NetworkProfile",
+    "EDRProfile",
+    "PROFILE_REGISTRY",
+    "SCENARIO_REGISTRY",
+    "__version__",
+]

loggen_lg-0.1.0/loggen-lg/__main__.py

@@ -0,0 +1,3 @@
+from loggen.cli import main
+
+main()

loggen_lg-0.1.0/loggen-lg/_version.py

@@ -0,0 +1 @@
+VERSION = "0.1.0"

loggen_lg-0.1.0/loggen-lg/cli.py

@@ -0,0 +1,221 @@
+# loggen CLI 
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from loggen._version import VERSION
+from loggen.core.session import Session
+from loggen.core.generator import LogGen
+from loggen.profiles import PROFILE_REGISTRY
+from loggen.formats import Format
+from loggen.scenarios import SCENARIO_REGISTRY
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="loggen",
+        description="Synthetic log generator for SIEM and IR exercises.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=_EXAMPLES,
+    )
+    p.add_argument("--version", action="version", version=f"loggen {VERSION}")
+
+    sub = p.add_subparsers(dest="command", metavar="COMMAND")
+
+    # ---- generate -------------------------------------------------------
+    gen = sub.add_parser("generate", aliases=["gen"], help="Generate log entries.")
+    gen.add_argument(
+        "-p", "--profile",
+        dest="profiles", action="append", default=[],
+        metavar="NAME",
+        help=f"Profile to include (repeatable). Choices: {', '.join(sorted(PROFILE_REGISTRY))}",
+    )
+    gen.add_argument(
+        "-s", "--scenario",
+        metavar="NAME",
+        help=f"Run a built-in scenario instead of random profiles. "
+             f"Choices: {', '.join(sorted(SCENARIO_REGISTRY))}",
+    )
+    gen.add_argument(
+        "-n", "--count",
+        type=int, default=None,
+        metavar="N",
+        help="Number of log entries to generate.",
+    )
+    gen.add_argument(
+        "--size",
+        metavar="SIZE",
+        help="Target output size, e.g. 100MB, 1GB, 2TB. Overrides --count.",
+    )
+    gen.add_argument(
+        "-f", "--format",
+        default="jsonl", choices=[f.value for f in Format],
+        help="Output format (default: jsonl).",
+    )
+    gen.add_argument(
+        "-o", "--output",
+        default=None, metavar="PATH",
+        help="Output file path. Use .gz suffix for gzip. Defaults to stdout.",
+    )
+    gen.add_argument(
+        "--seed",
+        type=int, default=0,
+        help="Random seed for reproducibility (default: 0).",
+    )
+    gen.add_argument(
+        "--start",
+        metavar="DATETIME",
+        help="Log start timestamp ISO-8601 (default: 2024-01-01T00:00:00Z).",
+    )
+    gen.add_argument(
+        "--duration",
+        type=float, default=24.0, metavar="HOURS",
+        help="Time window in hours (default: 24).",
+    )
+    gen.add_argument(
+        "--weight",
+        dest="weights", action="append", default=[], metavar="PROFILE:WEIGHT",
+        help="Relative sampling weight, e.g. windows:3 (repeatable).",
+    )
+    gen.add_argument(
+        "--progress",
+        action="store_true",
+        help="Print progress to stderr.",
+    )
+
+    # ---- list-profiles --------------------------------------------------
+    sub.add_parser("list-profiles", aliases=["profiles"],
+                   help="List available profiles.")
+
+    # ---- list-scenarios -------------------------------------------------
+    sub.add_parser("list-scenarios", aliases=["scenarios"],
+                   help="List available scenarios.")
+
+    # ---- list-formats ---------------------------------------------------
+    sub.add_parser("list-formats", aliases=["formats"],
+                   help="List available output formats.")
+
+    return p
+
+
+_EXAMPLES = """
+examples:
+  # 1 000 Windows + Linux events to stdout
+  loggen generate -p windows -p linux -n 1000
+
+  # 500 MB of network logs (gzipped), deterministic seed
+  loggen generate -p network --size 500MB --seed 42 -o network.jsonl.gz
+
+  # Run a brute-force scenario, CEF format
+  loggen generate --scenario brute_force -f cef -o brute.cef
+
+  # Mix all profiles with custom weights
+  loggen generate -p windows -p linux -p network -p edr \\
+      --weight windows:4 --weight edr:2 -n 50000 -o full.jsonl
+
+  # List what's available
+  loggen list-profiles
+  loggen list-scenarios
+"""
+
+
+def _parse_weights(raw: list[str]) -> dict[str, float]:
+    result: dict[str, float] = {}
+    for item in raw:
+        if ":" not in item:
+            raise SystemExit(f"Invalid weight format {item!r} — expected PROFILE:WEIGHT")
+        k, v = item.rsplit(":", 1)
+        result[k] = float(v)
+    return result
+
+
+def _parse_start(s: str | None):
+    if not s:
+        return None
+    from datetime import datetime, timezone
+    for fmt in ("%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
+        try:
+            dt = datetime.strptime(s, fmt)
+            return dt.replace(tzinfo=timezone.utc)
+        except ValueError:
+            continue
+    raise SystemExit(f"Cannot parse datetime {s!r}. Use ISO-8601, e.g. 2024-06-01T00:00:00Z")
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    if args.command in (None,):
+        parser.print_help()
+        return
+
+    if args.command in ("list-profiles", "profiles"):
+        print("Available profiles:")
+        for name in sorted(PROFILE_REGISTRY):
+            print(f"  {name}")
+        return
+
+    if args.command in ("list-scenarios", "scenarios"):
+        print("Available scenarios:")
+        from loggen.scenarios import SCENARIO_REGISTRY as SR
+        for name, cls in sorted(SR.items()):
+            print(f"  {name:<20}  {cls.description}")
+        return
+
+    if args.command in ("list-formats", "formats"):
+        print("Available formats:")
+        for f in Format:
+            print(f"  {f.value}")
+        return
+
+    if args.command in ("generate", "gen"):
+        _run_generate(args)
+        return
+
+    parser.print_help()
+
+
+def _run_generate(args) -> None:
+    if not args.profiles and not args.scenario:
+        raise SystemExit(
+            "Specify at least one --profile or a --scenario.\n"
+            f"Profiles: {', '.join(sorted(PROFILE_REGISTRY))}\n"
+            f"Scenarios: {', '.join(sorted(SCENARIO_REGISTRY))}"
+        )
+
+    if not args.count and not args.size and not args.scenario:
+        raise SystemExit("Specify --count N or --size SIZE (e.g. --size 100MB).")
+
+    start = _parse_start(args.start)
+    session_kwargs: dict = {"seed": args.seed, "duration_hours": args.duration}
+    if start:
+        session_kwargs["start"] = start
+    session = Session(**session_kwargs)
+    gen = LogGen(session=session)
+
+    weights = _parse_weights(args.weights)
+    dest = args.output  # None → stdout
+
+    try:
+        written = gen.write(
+            dest=dest,
+            profiles=args.profiles,
+            count=args.count,
+            size=args.size,
+            format=args.format,
+            progress=args.progress,
+            weights=weights if weights else None,
+            scenario=args.scenario,
+        )
+    except (BrokenPipeError, KeyboardInterrupt):
+        pass
+    else:
+        if args.output and args.progress:
+            sys.stderr.write(f"Wrote {written / 1_048_576:.2f} MB → {args.output}\n")
+
+
+if __name__ == "__main__":
+    main()

loggen_lg-0.1.0/loggen-lg/core/__init__.py

@@ -0,0 +1,5 @@
+from loggen.core.session import Session
+from loggen.core.generator import LogGen
+from loggen.core.loaders import WordList
+
+__all__ = ["Session", "LogGen", "WordList"]

loggen_lg-0.1.0/loggen-lg/core/generator.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from typing import Iterator
+
+from loggen.core.session import Session
+from loggen.profiles.base import LogEntry
+from loggen.profiles import PROFILE_REGISTRY
+
+
+def _parse_size(s: str | int | None) -> int | None:
+    if s is None:
+        return None
+    if isinstance(s, int):
+        return s
+    s = s.strip().upper()
+    for unit, mult in [("TB", 1 << 40), ("GB", 1 << 30), ("MB", 1 << 20), ("KB", 1 << 10), ("B", 1)]:
+        if s.endswith(unit):
+            return int(float(s[: -len(unit)]) * mult)
+    return int(s)
+
+
+class LogGen:
+    def __init__(self, seed: int = 0, session: Session | None = None) -> None:
+        self.session = session or Session(seed=seed)
+
+    def stream(
+        self,
+        profiles: list[str],
+        count: int | None = None,
+        weights: dict[str, float] | None = None,
+        scenario: str | None = None,
+    ) -> Iterator[LogEntry]:
+        if scenario:
+            yield from self._stream_scenario(scenario, count)
+            return
+
+        if not profiles:
+            raise ValueError("At least one profile required.")
+
+        for name in profiles:
+            if name not in PROFILE_REGISTRY:
+                raise ValueError(f"Unknown profile {name!r}. Available: {sorted(PROFILE_REGISTRY)}")
+
+        profile_instances = [(name, PROFILE_REGISTRY[name]()) for name in profiles]
+        w = weights or {}
+        profile_weights = [w.get(name, 1.0) for name, _ in profile_instances]
+
+        rng = self.session.rng
+        n = 0
+        while count is None or n < count:
+            (name, profile), = rng.choices(profile_instances, weights=profile_weights, k=1)
+            yield profile.generate_one(self.session.fork(n))
+            n += 1
+
+    def to_list(self, profiles: list[str], count: int, **kwargs) -> list[LogEntry]:
+        return list(self.stream(profiles, count=count, **kwargs))
+
+    def write(
+        self,
+        dest,
+        profiles: list[str],
+        count: int | None = None,
+        size: str | int | None = None,
+        format: str = "jsonl",
+        progress: bool = False,
+        **kwargs,
+    ) -> int:
+        from loggen.core.writer import write_stream
+        from loggen.formats import get_formatter
+
+        formatter = get_formatter(format)
+        size_bytes = _parse_size(size)
+        entries = self.stream(profiles, count=count, **kwargs)
+        return write_stream(entries, dest, formatter, size_limit=size_bytes, progress=progress)
+
+    def _stream_scenario(self, scenario: str, count: int | None) -> Iterator[LogEntry]:
+        from loggen.scenarios import SCENARIO_REGISTRY
+
+        cls = SCENARIO_REGISTRY.get(scenario)
+        if cls is None:
+            raise ValueError(f"Unknown scenario {scenario!r}. Available: {sorted(SCENARIO_REGISTRY)}")
+        yield from cls(self.session).stream(count=count)

loggen_lg-0.1.0/loggen-lg/core/loaders.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import csv
+import json
+import os
+from pathlib import Path
+
+
+class WordList(list):
+    """A list that can be populated from files or env vars."""
+
+    @classmethod
+    def from_file(cls, path: str | Path, *, column: int = 0, comment: str = "#") -> "WordList":
+        """Load from .txt, .json, or .csv."""
+        path = str(path)
+        ext = os.path.splitext(path)[1].lower()
+
+        if ext == ".json":
+            with open(path) as f:
+                data = json.load(f)
+            if not isinstance(data, list):
+                raise ValueError(f"{path}: expected a JSON array")
+            return cls(str(x) for x in data)
+
+        if ext == ".csv":
+            with open(path, newline="") as f:
+                rows = csv.reader(f)
+                return cls(
+                    row[column]
+                    for row in rows
+                    if row and not row[0].startswith(comment)
+                )
+
+        # plain text — one item per line
+        with open(path) as f:
+            return cls(
+                line.strip()
+                for line in f
+                if line.strip() and not line.startswith(comment)
+            )
+
+    @classmethod
+    def from_env(cls, var: str, sep: str = ",", fallback: list[str] | None = None) -> "WordList":
+        """Load from a comma-separated environment variable."""
+        raw = os.environ.get(var, "")
+        if raw:
+            return cls(item.strip() for item in raw.split(sep) if item.strip())
+        return cls(fallback or [])