txt2stix 0.0.4__py3-none-any.whl

txt2stix/txt2stix.py

@@ -0,0 +1,336 @@
+import argparse, dotenv
+from datetime import datetime
+import glob
+import uuid
+import itertools
+import fnmatch
+import re
+from pathlib import Path
+import sys, os
+
+from pydantic import BaseModel
+
+from txt2stix.ai_extractor.utils import DescribesIncident
+from txt2stix.attack_flow import parse_flow
+
+
+from .utils import Txt2StixData, remove_links
+
+from .common import UUID_NAMESPACE, FatalException
+
+from .bundler import txt2stixBundler, parse_stix, TLP_LEVEL
+from .import extractions, lookups, pattern
+from types import SimpleNamespace
+import functools
+from fnmatch import filter
+from .ai_extractor import ALL_AI_EXTRACTORS, BaseAIExtractor, ModelError
+from stix2.serialization import serialize as stix2_serialize
+from stix2 import Bundle
+
+import json, logging
+
+
+def newLogger(name: str) -> logging.Logger:
+    # Configure logging
+    stream_handler = logging.StreamHandler()  # Log to stdout and stderr
+    stream_handler.setLevel(logging.INFO)
+    logging.basicConfig(
+        level=logging.DEBUG,  # Set the desired logging level
+        format=f"%(asctime)s [{name}] [%(levelname)s] %(message)s",
+        handlers=[stream_handler],
+        datefmt='%d-%b-%y %H:%M:%S'
+    )
+
+    return logging.root
+
+def setLogFile(logger, file: Path):
+    file.parent.mkdir(parents=True, exist_ok=True)
+    logger.info(f"Saving log to `{file.absolute()}`")
+    handler = logging.FileHandler(file, "w")
+    handler.formatter = logging.Formatter(fmt='%(levelname)s %(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')
+    handler.setLevel(logging.DEBUG)
+    logger.addHandler(handler)
+    logger.info("=====================txt2stix======================")
+
+
+MODULE_PATH = Path(__file__).parent.parent
+INCLUDES_PATH = MODULE_PATH/"includes"
+try:
+    from . import includes
+    INCLUDES_PATH = Path(includes.__file__).parent
+except:
+    pass
+
+def split_comma(s: str) -> list[str]:
+    return [ss for ss in s.split(",") if ss]
+
+def range_type(min, max):
+    def fn(astr):
+        value = int(astr)
+        if min<= value <= max:
+            return value
+        else:
+            raise argparse.ArgumentTypeError(f'value {value} not in range [{min}-{max}]')
+    return fn
+
+def parse_labels(labels: str) -> list[str]:
+    labels = labels.split(",")
+    for label in labels:
+        if not re.fullmatch(r"[a-zA-Z0-9]+", label):
+            raise argparse.ArgumentTypeError(f"invalid label: {label}")
+
+    return labels
+
+def parse_extractors_globbed(type, all_extractors, names):
+    globbed_names = set()
+    for name in names.split(","):
+        matches = fnmatch.filter(all_extractors.keys(), name)
+        if not matches:
+            raise argparse.ArgumentTypeError(f'`{name}` has 0 matches')
+        globbed_names.update(matches)
+    filtered_extractors  = {}
+    for extractor_name in globbed_names:
+        try:
+            extractor = all_extractors[extractor_name]
+            extraction_processor  = filtered_extractors.get(extractor.type, {})
+            if extractor.type in ["lookup"]:
+                lookups.load_lookup(extractor)
+            if extractor.type == "pattern":
+                pattern.load_extractor(extractor)
+            filtered_extractors[extractor.type] =  extraction_processor
+            extraction_processor[extractor_name] = extractor
+        except KeyError:
+            raise argparse.ArgumentTypeError(f"no such {type} slug `{extractor_name}`")
+        except BaseException as e:
+            raise argparse.ArgumentTypeError(f"{type} `{extractor_name}`: {e}")
+    return filtered_extractors
+
+def parse_ref(value):
+    m = re.compile(r'(.+?)=(.+)').match(value)
+    if not m:
+        raise argparse.ArgumentTypeError("must be in format key=value")
+    return dict(source_name=m.group(1), external_id=m.group(2))
+
+def parse_model(value: str):
+    splits = value.split(':', 1)
+    provider = splits[0]
+    if provider not in ALL_AI_EXTRACTORS:
+        raise argparse.ArgumentTypeError(f"invalid AI provider in `{value}`, must be one of {list(ALL_AI_EXTRACTORS)}")
+    provider = ALL_AI_EXTRACTORS[provider]
+
+    try:
+        if len(splits) == 2:
+            return provider(model=splits[1])
+        return provider()
+    except Exception as e:
+        raise ModelError(f"Unable to initialize model `{value}`") from e
+
+def parse_bool(value: str):
+    value = value.lower()
+    return value in ["yes", "y", "true", "1"]
+
+def parse_args():
+    EXTRACTORS_PATH = INCLUDES_PATH/"extractions"
+    all_extractors = extractions.parse_extraction_config(INCLUDES_PATH)
+    
+    parser = argparse.ArgumentParser(description="File Conversion Tool")
+
+    inf_arg  = parser.add_argument("--input_file", "--input-file", required=True, help="The file to be converted. Must be .txt", type=Path)
+    parser.add_argument("--ai_content_check_provider", required=False, type=parse_model, help="Use an AI model to check wether the content of the file contains threat intelligence. Paticularly useful to weed out vendor marketing.")
+    name_arg = parser.add_argument("--name", required=True, help="Name of the file, max 124 chars", default="stix-out")
+    parser.add_argument("--created", required=False, default=datetime.now(), help="Allow user to optionally pass --created time in input, which will hardcode the time used in created times")
+    parser.add_argument("--ai_settings_extractions", required=False, type=parse_model, help="(required if AI extraction enabled): passed in format provider:model e.g. openai:gpt4o. Can pass more than one value to get extractions from multiple providers.", metavar="provider[:model]", nargs='+')
+    parser.add_argument("--ai_settings_relationships", required=False, type=parse_model, help="(required if AI relationship enabled): passed in format `provider:model`. Can only pass one model at this time.", metavar="provider[:model]")
+    parser.add_argument("--labels", type=parse_labels)
+    parser.add_argument("--relationship_mode", choices=["ai", "standard"], required=True)
+    parser.add_argument("--report_id", type=uuid.UUID, required=False, help="id to use instead of automatically generated `{name}+{created}`", metavar="VALID_UUID")
+    parser.add_argument("--confidence", type=range_type(0,100), default=None, help="value between 0-100. Default if not passed is null.", metavar="[0-100]")
+    parser.add_argument("--tlp_level", "--tlp-level", choices=TLP_LEVEL.levels().keys(), default="clear", help="TLP level, default is clear")
+    parser.add_argument("--use_extractions", "--use-extractions", default={}, type=functools.partial(parse_extractors_globbed, "extractor", all_extractors),  help="Specify extraction types from the default/local extractions .yaml file", metavar="EXTRACTION1,EXTRACTION2")
+    parser.add_argument("--use_identity", "--use-identity", help="Specify an identity file id (e.g., {\"type\":\"identity\",\"name\":\"demo\",\"identity_class\":\"system\"})", metavar="[stix2 identity json]", type=parse_stix)
+    parser.add_argument("--external_refs", type=parse_ref, help="pass additional `external_references` entry (or entries) to the report object created. e.g --external_ref author=dogesec link=https://dkjjadhdaj.net", default=[], metavar="{source_name}={external_id}", action="extend", nargs='+')
+    parser.add_argument('--ignore_image_refs', default=True, type=parse_bool)
+    parser.add_argument('--ignore_link_refs', default=True, type=parse_bool)
+    parser.add_argument("--ignore_extraction_boundary", default=False, type=parse_bool, help="default if not passed is `false`, but if set to `true` will ignore boundary capture logic for extractions")
+    parser.add_argument('--ai_create_attack_flow', default=False, action='store_true', help="create attack flow for attack objects in report/bundle")
+
+    args = parser.parse_args()
+    if not args.input_file.exists():
+        raise argparse.ArgumentError(inf_arg, "cannot open file")
+    if len(args.name) > 124:
+        raise argparse.ArgumentError(name_arg, "max 124 characters")
+
+    if args.relationship_mode == 'ai' and not args.ai_settings_relationships:
+        parser.error("relationship_mode is set to AI, --ai_settings_relationships is required")
+
+    if args.ai_create_attack_flow and not args.ai_settings_relationships:
+        parser.error("--ai_create_attack_flow requires --ai_settings_relationships")
+    #### process --use-extractions 
+    if args.use_extractions.get('ai') and not args.ai_settings_extractions:
+        parser.error("ai based extractors are passed, --ai_settings_extractions is required")
+
+    args.all_extractors  = all_extractors
+    return args
+
+REQUIRED_ENV_VARIABLES = [
+    "INPUT_TOKEN_LIMIT",
+    "CTIBUTLER_BASE_URL",
+    "VULMATCH_BASE_URL",
+]
+def load_env():
+    for env in REQUIRED_ENV_VARIABLES:
+        if not os.getenv(env):
+            raise FatalException(f"env variable `{env}` required")
+
+
+def log_notes(content, type):
+    logging.debug(f" ========================= {type} ========================= ")
+    logging.debug(f" ========================= {'+'*len(type)} ========================= ")
+    logging.debug(json.dumps(content, sort_keys=True, indent=4))
+    logging.debug(f" ========================= {'-'*len(type)} ========================= ")
+
+def extract_all(bundler: txt2stixBundler, extractors_map, text_content, ai_extractors: list[BaseAIExtractor]=[], **kwargs):
+    assert ai_extractors or not extractors_map.get("ai"), "There should be at least one AI extractor in ai_extractors"
+
+    text_content = "\n"+text_content+"\n"
+    all_extracts = dict()
+    if extractors_map.get("lookup"):
+        try:
+            lookup_extracts = lookups.extract_all(extractors_map["lookup"].values(), text_content)
+            bundler.process_observables(lookup_extracts)
+            all_extracts["lookup"] = lookup_extracts
+        except BaseException as e:
+            logging.exception("lookup extraction failed", exc_info=True)
+
+    if extractors_map.get("pattern"):
+        try:
+            logging.info("using pattern extractors")
+            pattern_extracts = pattern.extract_all(extractors_map["pattern"].values(), text_content, ignore_extraction_boundary=kwargs.get('ignore_extraction_boundary', False))
+            bundler.process_observables(pattern_extracts)
+            all_extracts["pattern"] = pattern_extracts
+        except BaseException as e:
+            logging.exception("pattern extraction failed", exc_info=True)
+
+    if extractors_map.get("ai"):
+        logging.info("using ai extractors")
+
+        for extractor in ai_extractors:
+            logging.info("running extractor: %s", extractor.extractor_name)
+            try:
+                ai_extracts = extractor.extract_objects(text_content, extractors_map["ai"].values())
+                ai_extracts = ai_extracts.model_dump().get('extractions', [])
+                bundler.process_observables(ai_extracts)
+                all_extracts[f"ai-{extractor.extractor_name}"] = ai_extracts
+            except BaseException as e:
+                logging.exception("AI extraction failed for %s", extractor.extractor_name, exc_info=True)
+
+    log_notes(all_extracts, "Extractions")
+    return all_extracts
+
+def extract_relationships_with_ai(bundler: txt2stixBundler, text_content, all_extracts, ai_extractor_session: BaseAIExtractor):
+    relationships = None
+    try:
+        all_extracts = list(itertools.chain(*all_extracts.values()))
+        relationship_types = (INCLUDES_PATH/"helpers/stix_relationship_types.txt").read_text().splitlines()
+        relationships = ai_extractor_session.extract_relationships(text_content, all_extracts, relationship_types)
+        relationships = relationships.model_dump()
+        log_notes(relationships, "Relationships")
+        bundler.process_relationships(relationships['relationships'])
+    except BaseException as e:
+        logging.exception("Relationship processing failed: %s", e)
+    return relationships
+
+def validate_token_count(max_tokens, input, extractors: list[BaseAIExtractor]):
+    logging.info('INPUT_TOKEN_LIMIT = %d', max_tokens)
+    for extractor in extractors:
+        token_count = _count_token(extractor, input)
+        if  token_count > max_tokens:
+            raise FatalException(f"{extractor.extractor_name}: input_file token count ({token_count}) exceeds INPUT_TOKEN_LIMIT ({max_tokens})")
+    
+
+@functools.lru_cache
+def _count_token(extractor: BaseAIExtractor, input: str):
+    return extractor.count_tokens(input)
+        
+def run_txt2stix(bundler: txt2stixBundler, preprocessed_text: str, extractors_map: dict,
+                ai_content_check_provider=None,
+                ai_create_attack_flow=None,
+                input_token_limit=10,
+                ai_settings_extractions=None,
+                ai_settings_relationships=None,
+                relationship_mode="standard",
+                ignore_extraction_boundary=False,
+                always_extract=False, # continue even if ai_content_check fails
+
+                **kwargs
+        ) -> Txt2StixData:
+    should_extract = True
+    retval = Txt2StixData.model_construct()
+    retval.extractions = retval.attack_flow = retval.relationships = None
+    if ai_content_check_provider:
+        logging.info("checking content")
+        model : BaseAIExtractor = ai_content_check_provider
+        validate_token_count(input_token_limit, preprocessed_text, [model])
+        retval.content_check = model.check_content(preprocessed_text)
+        should_extract = retval.content_check.describes_incident
+        logging.info("=== ai-check-content output ====")
+        logging.info(retval.content_check.model_dump_json())
+        for classification in retval.content_check.incident_classification:
+            bundler.report.labels.append(f'txt2stix:{classification}'.lower())
+
+    if should_extract or always_extract:
+        if extractors_map.get("ai"):
+            validate_token_count(input_token_limit, preprocessed_text, ai_settings_extractions)
+        if relationship_mode == "ai":
+            validate_token_count(input_token_limit, preprocessed_text, [ai_settings_relationships])
+
+        retval.extractions = extract_all(bundler, extractors_map, preprocessed_text, ai_extractors=ai_settings_extractions, ignore_extraction_boundary=ignore_extraction_boundary)
+        if relationship_mode == "ai" and sum(map(lambda x: len(x), retval.extractions.values())):
+            retval.relationships = extract_relationships_with_ai(bundler, preprocessed_text, retval.extractions, ai_settings_relationships)
+            
+        if ai_create_attack_flow:
+            logging.info("creating attack-flow bundle")
+            ex: BaseAIExtractor = ai_settings_relationships
+            retval.attack_flow = ex.extract_attack_flow(preprocessed_text, retval.extractions, retval.relationships)
+            bundler.flow_objects = parse_flow(bundler.report, retval.attack_flow)
+
+    return retval
+
+def main():
+    dotenv.load_dotenv()
+    logger = newLogger("txt2stix")
+    try:
+        args = parse_args()
+        job_id = args.report_id or str(uuid.uuid4())
+        setLogFile(logger, Path(f"logs/logs-{job_id}.log"))
+        logger.info(f"Arguments: {json.dumps(sys.argv[1:])}")
+
+        
+        input_text = args.input_file.read_text()
+        preprocessed_text = remove_links(input_text, args.ignore_image_refs, args.ignore_link_refs)
+        load_env()
+
+
+        bundler = txt2stixBundler(args.name, args.use_identity, args.tlp_level, input_text, args.confidence, args.all_extractors, args.labels, created=args.created, report_id=args.report_id, external_references=args.external_refs)
+        log_notes(sys.argv, "Config")
+        convo_str = None
+
+        data = run_txt2stix(
+            bundler, preprocessed_text, args.use_extractions,
+            input_token_limit=int(os.environ['INPUT_TOKEN_LIMIT']),
+            **args.__dict__,
+        )
+
+        ## write outputs
+        out = bundler.to_json()
+        output_path = Path("./output")/f"{bundler.bundle.id}.json"
+        output_path.parent.mkdir(exist_ok=True)
+        output_path.write_text(out)
+        logger.info(f"Wrote bundle output to `{output_path}`")
+        data_path = Path(str(output_path).replace('bundle--', 'data--'))
+        data_path.write_text(data.model_dump_json(indent=4))
+        logger.info(f"Wrote data output to `{data_path}`")
+    except argparse.ArgumentError as e:
+        logger.exception(e, exc_info=True)
+    except:
+        raise

txt2stix/utils.py

@@ -0,0 +1,86 @@
+import os
+import pkgutil
+import re
+from pathlib import Path
+from typing import Dict
+from pydantic import BaseModel, Field
+import bs4
+import mistune
+from mistune.renderers.markdown import MarkdownRenderer
+from mistune.util import unescape
+
+from txt2stix.ai_extractor.utils import AttackFlowList, DescribesIncident
+class ImageLinkRemover(MarkdownRenderer):
+    def __init__(self, remove_links: bool=False, remove_images: bool=False):
+        self.remove_links = remove_links
+        self.remove_images = remove_images
+        super().__init__()
+
+    def image(self, token: dict[str, dict], state: mistune.BlockState) -> str:
+        if self.remove_images:
+            token['attrs']['url'] = ''
+        return super().image(token, state)
+
+    def link(self, token: dict[str, dict], state: mistune.BlockState) -> str:
+        if self.remove_links and token.get('type') != 'image':
+            token['attrs']['url'] = ''
+        return super().link(token, state)
+    
+    def codespan(self, token: dict[str, dict], state: mistune.BlockState) -> str:
+        token['raw'] = unescape(token['raw'])
+        return super().codespan(token, state)
+    
+
+    def block_html(self, token: Dict[str, dict], state: mistune.BlockState) -> str:
+        return self.inline_html(token, state) + '\n\n'
+    
+    def inline_html(self, token: Dict[str, dict], state: mistune.BlockState) -> str:
+        raw = token['raw']
+        soup = bs4.BeautifulSoup(raw, 'html.parser')
+        if self.remove_links:
+            for a in soup.find_all('a'):
+                del a['href']
+        if self.remove_images:
+            for img in soup.find_all('img'):
+                del img['src']
+        return soup.decode()
+
+import tldextract
+
+
+class Txt2StixData(BaseModel):
+    content_check: DescribesIncident = Field(default=None)
+    extractions: dict = Field(default=None)
+    relationships: list[dict] = Field(default_factory=list)
+    attack_flow: AttackFlowList = Field(default=None)
+
+
+def remove_links(input_text: str, remove_images: bool, remove_anchors: bool):
+    modify_links = mistune.create_markdown(escape=False, renderer=ImageLinkRemover(remove_links=remove_anchors, remove_images=remove_images))
+    return modify_links(input_text)
+
+def read_included_file(path):
+    try:
+        return pkgutil.get_data("txt2stix.includes", path).decode()
+    except:
+        return (Path("includes")/path).read_text()
+    
+def validate_tld(domain: str):
+    _, _, suffix = domain.lower().rpartition('.')
+    return suffix in TLDs
+
+def validate_reg_key(reg_key: str):
+    reg_key = reg_key.lower()
+    for prefix in REGISTRY_PREFIXES:
+        if reg_key.startswith(prefix):
+            return True
+    return False
+
+def validate_file_mimetype(file_name):
+    _, ext = os.path.splitext(file_name)
+    return FILE_EXTENSIONS.get(ext)
+
+
+TLDs = [tld.lower() for tld in read_included_file('helpers/tlds.txt').splitlines()]
+REGISTRY_PREFIXES = [key.lower() for key in read_included_file('helpers/windows_registry_key_prefix.txt').splitlines()]
+FILE_EXTENSIONS = dict(line.lower().split(',') for line in read_included_file('helpers/mimetype_filename_extension_list.csv').splitlines())

txt2stix-0.0.4.dist-info/METADATA

@@ -0,0 +1,190 @@
+Metadata-Version: 2.4
+Name: txt2stix
+Version: 0.0.4
+Summary: txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle.
+Project-URL: Homepage, https://github.com/muchdogesec/txt2stix
+Project-URL: Issues, https://github.com/muchdogesec/txt2stix/issues
+Author-email: DOGESEC <support@dogesec.com>
+License-File: LICENSE
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Requires-Dist: base58>=2.1.1
+Requires-Dist: beautifulsoup4>=4.12.3
+Requires-Dist: llama-index-core>=0.12.42
+Requires-Dist: llama-index-llms-anthropic>=0.7.2
+Requires-Dist: llama-index-llms-deepseek>=0.1.2
+Requires-Dist: llama-index-llms-gemini>=0.5.0
+Requires-Dist: llama-index-llms-openai-like>=0.4.0
+Requires-Dist: llama-index-llms-openai>=0.4.5
+Requires-Dist: llama-index-llms-openrouter>=0.3.2
+Requires-Dist: mistune>=3.0.2
+Requires-Dist: pathvalidate>=3.2.0
+Requires-Dist: phonenumbers>=8.13.39
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: requests>=2.32.4
+Requires-Dist: schwifty>=2024.6.1
+Requires-Dist: stix2extensions
+Requires-Dist: tld>=0.13
+Requires-Dist: tldextract>=5.1.2
+Requires-Dist: validators>=0.28.3
+Provides-Extra: tests
+Requires-Dist: pytest; extra == 'tests'
+Requires-Dist: pytest-cov; extra == 'tests'
+Requires-Dist: pytest-subtests; extra == 'tests'
+Requires-Dist: requests; extra == 'tests'
+Description-Content-Type: text/markdown
+
+# txt2stix
+
+[![codecov](https://codecov.io/gh/muchdogesec/txt2stix/graph/badge.svg?token=KTHAIYBH1I)](https://codecov.io/gh/muchdogesec/txt2stix)
+
+## Before you begin...
+
+We have build two products on-top of txt2stix that provide more user-friendly experience:
+
+* [Stixify: Extract machine readable cyber threat intelligence from unstructured data](https://github.com/muchdogesec/stixify)
+* [Obstracts: Turn any blog into structured threat intelligence](https://github.com/muchdogesec/obstracts)
+
+## Overview
+
+![txt2stix](docs/txt2stix.png)
+
+txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle.
+
+The general design goal of txt2stix was to keep it flexible, but simple, so that new extractions could be added or modified over time.
+
+In short txt2stix;
+
+1. takes a txt file input
+2. extracts observables for enabled extractions (ai, pattern, or lookup)
+3. converts extracted observables to STIX 2.1 objects
+4. generates the relationships between extracted observables (ai, standard)
+5. converts extracted relationships to STIX 2.1 SRO objects
+6. outputs a STIX 2.1 bundle
+
+## tl;dr
+
+[![txt2stix](https://img.youtube.com/vi/TWVGCou9oGk/0.jpg)](https://www.youtube.com/watch?v=TWVGCou9oGk)
+
+[Watch the demo](https://www.youtube.com/watch?v=TWVGCou9oGk).
+
+## Usage
+
+### Setup
+
+Install the required dependencies using:
+
+```shell
+# clone the latest code
+git clone https://github.com/muchdogesec/txt2stix
+cd txt2stix
+# create a venv
+python3 -m venv txt2stix-venv
+source txt2stix-venv/bin/activate
+# install requirements
+pip3 install .
+```
+
+### Set variables
+
+txt2stix has various settings that are defined in an `.env` file.
+
+To create a template for the file:
+
+```shell
+cp .env.example .env
+```
+
+To see more information about how to set the variables, and what they do, read the `.env.markdown` file.
+
+### Usage
+
+```shell
+python3 txt2stix.py \
+	--relationship_mode MODE \
+	--input_file FILE.txt \
+	...
+```
+
+The following arguments are available:
+
+#### Input settings
+
+* `--input_file` (REQUIRED): the file to be converted. Must be `.txt`
+
+#### STIX Report generation settings
+
+
+* `--name` (REQUIRED): name of file, max 72 chars. Will be used in the STIX Report Object created.
+* `--report_id` (OPTIONAL): Sometimes it is required to control the id of the `report` object generated. You can therefore pass a valid UUIDv4 in this field to be assigned to the report. e.g. passing `2611965-930e-43db-8b95-30a1e119d7e2` would create a STIX object id `report--2611965-930e-43db-8b95-30a1e119d7e2`. If this argument is not passed, the UUID will be randomly generated.
+* `--tlp_level` (OPTIONAL): Options are `clear`, `green`, `amber`, `amber_strict`, `red`. Default if not passed, is `clear`.
+* `--confidence` (OPTIONAL): value between 0-100. Default if not passed is null.
+* `--labels` (OPTIONAL): comma seperated list of labels. Case-insensitive (will all be converted to lower-case). Allowed `a-z`, `0-9`. e.g.`label1,label2` would create 2 labels.
+* `--created` (OPTIONAL): by default all object `created` times will take the time the script was run. If you want to explicitly set these times you can do so using this flag. Pass the value in the format `YYYY-MM-DDTHH:MM:SS.sssZ` e.g. `2020-01-01T00:00:00.000Z`
+* `--use_identity` (OPTIONAL): can pass a full STIX 2.1 identity object (make sure to properly escape). Will be validated by the STIX2 library.
+* `--external_refs` (OPTIONAL): txt2stix will automatically populate the `external_references` of the report object it creates for the input. You can use this value to add additional objects to `external_references`. Note, you can only add `source_name` and `external_id` values currently. Pass as `source_name=external_id`. e.g. `--external_refs txt2stix=demo1 source=id` would create the following objects under the `external_references` property: `{"source_name":"txt2stix","external_id":"demo1"},{"source_name":"source","external_id":"id"}`
+
+#### Output settings
+
+How the extractions are performed
+
+* `--use_extractions` (REQUIRED): if you only want to use certain extraction types, you can pass their slug found in either `includes/ai/config.yaml`, `includes/lookup/config.yaml` `includes/pattern/config.yaml` (e.g. `pattern_ipv4_address_only`). Default if not passed, no extractions applied. You can also pass a catch all wildcard `*` which will match all extraction paths (e.g. `'pattern_*'` would run all extractions starting with `pattern_` -- make sure to use quotes when using a wildcard)
+	* Important: if using any AI extractions (`ai_*`), you must set an AI API key in your `.env` file
+	* Important: if you are using any MITRE ATT&CK, CAPEC, CWE, ATLAS or Location extractions you must set `CTIBUTLER` or NVD CPE or CVE extractions you must set `VULMATCH` settings in your `.env` file
+* `--relationship_mode` (REQUIRED): either.
+	* `ai`: AI provider must be enabled. extractions performed by either regex or AI for extractions user selected. Rich relationships created from AI provider from extractions.
+	* `standard`: extractions performed by either regex or AI (AI provider must be enabled) for extractions user selected. Basic relationships created from extractions back to master Report object generated.
+* `--ignore_extraction_boundary` (OPTIONAL, default `false`, not compatible with AI extractions): in some cases the same string will create multiple extractions depending on extractions set (e.g. `https://www.google.com/file.txt` could create a url, url with file, domain, subdomain, and file). The default behaviour is for txt2stix to take the longest extraction and ignore everything else (e.g. only extract url with file, and ignore url, file, domain, subdomain, and file). If you want to override this behaviour and get all extractions in the output, set this flag to `true`.
+* `--ignore_image_refs` (default `true`): images references in documents don't usually need extracting. e.g. `<img src="https://example.com/image.png" alt="something">` you would not want domain or file extractions extracting `example.com` and `image.png`. Hence these are ignored by default (they are removed from text sent to extraction). Note, only the `img src` is ignored, all other values e.g. `alt` are considered. If you want extractions to consider this data, set it to `false`
+* `--ignore_link_refs` (default `true`): link references in documents don't usually need extracting e.g. `<a href="https://example.com/link.html" title="something">Bad Actor</a>` you would only want `Bad actor` to be considered for extraction. Hence these part of the link are ignored by default (they are removed from text sent to extraction). Note, only the `a href` is ignored, all other values e.g. `title` are considered. Setting this to `false` will also include everything inside the link tag (e.g. `example.com` would extract as a domain)
+
+#### AI settings
+
+If any AI extractions, or AI relationship mode is set, you must set the following accordingly
+
+* `--ai_settings_extractions`:
+	* defines the `provider:model` to be used for extractions. You can supply more than one provider. Seperate with a space (e.g. `openrouter:openai/gpt-4o` `openrouter:deepseek/deepseek-chat`) If more than one provider passed, txt2stix will take extractions from all models, de-dupelicate them, and them package them in the output. Currently supports:
+		* Provider (env var required `OPENROUTER_API_KEY`): `openrouter:`, providers/models `openai/gpt-4o`, `deepseek/deepseek-chat` ([More here](https://openrouter.ai/models))
+		* Provider (env var required `OPENAI_API_KEY`): `openai:`, models e.g.: `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo`, `gpt-4` ([More here](https://platform.openai.com/docs/models))
+		* Provider (env var required `ANTHROPIC_API_KEY`): `anthropic:`, models e.g.: `claude-3-5-sonnet-latest`, `claude-3-5-haiku-latest`, `claude-3-opus-latest` ([More here](https://docs.anthropic.com/en/docs/about-claude/models))
+		* Provider (env var required `GOOGLE_API_KEY`): `gemini:models/`, models: `gemini-1.5-pro-latest`, `gemini-1.5-flash-latest` ([More here](https://ai.google.dev/gemini-api/docs/models/gemini))
+		* Provider (env var required `DEEPSEEK_API_KEY`): `deepseek:`, models `deepseek-chat` ([More here](https://api-docs.deepseek.com/quick_start/pricing))
+	* See `tests/manual-tests/cases-ai-extraction-type.md` for some examples
+* `--ai_settings_relationships`:
+	* similar to `ai_settings_extractions` but defines the model used to generate relationships. Only one model can be provided. Passed in same format as `ai_settings_extractions`
+	* See `tests/manual-tests/cases-ai-relationships.md` for some examples
+* `--ai_content_check_provider`: Passing this flag will get the AI to try and classify the text in the input to 1) determine if it is talking about threat intelligence, and 2) what type of threat intelligence it is talking about. For context, we use this to filter out non-threat intel posts in Obstracts and Stixify. You pass `provider:model` with this flag to determine the AI model you wish to use to perform the check.
+* `--ai_create_attack_flow`: passing this flag will also prompt the AI model (the same entered for `--ai_settings_relationships`) to generate an [Attack Flow](https://center-for-threat-informed-defense.github.io/attack-flow/) for the MITRE ATT&CK extractions to define the logical order in which they are being described. You must pass `--ai_settings_relationships` for this to work.
+
+## Adding new extractions
+
+It is very likely you'll want to extend txt2stix to include new extractions to;
+
+* Add a new lookup extraction: add your lookup to `includes/lookups` as a `.txt` file. Lookups should be a list of items seperated by new lines to be searched for in documents. Once this is added, update `includes/extractions/lookup/config.yaml` with a new record pointing to your lookup. You can now use this lookup time at script run-time.
+* Add a new AI extraction: Edit `includes/extractions/ai/config.yaml` with a new record for your extraction. You can craft the prompt used in the config to control how the LLM performs the extraction.
+
+Currently it is not possible to easily add any other types of extractions (without modifying the logic at a code level).
+
+## Detailed documentation
+
+If you would like to understand how txt2stix works in more detail, please refer to the documentation in `/docs/README.md`.
+
+This documentation is paticularly helpful to read for those of you wanting to add your own custom extractions.
+
+## Useful supporting tools
+
+* [A Quick Start Guide to txt2stix](https://www.dogesec.com/blog/txt2stix_quickstart_guide/)
+* [An example of how to use txt2stix with Attack Flows](https://www.dogesec.com/blog/understading_structure_attack_flows/)
+* [STIX2 Python Library](https://pypi.org/project/stix2/): APIs for serializing and de-serializing STIX2 JSON content
+* [STIX 2 Pattern Validator](https://pypi.org/project/stix2-patterns/): a tool for checking the syntax of the Cyber Threat Intelligence (CTI) STIX Pattern expressions
+* [STIX Viewer](https://github.com/traut/stixview): Quickly load bundles produced from your report
+
+## Support
+
+[Minimal support provided via the DOGESEC community](https://community.dogesec.com/).
+
+## License
+
+[Apache 2.0](/LICENSE).