cryptic-cti 0.1.0__py3-none-any.whl

classification/utils.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+from pathlib import Path
+from setfit import SetFitModel
+
+def load_model(model_dir: Path) -> SetFitModel:
+    if not model_dir.exists():
+        raise FileNotFoundError(f"SetFit model directory not found in: {model_dir}")
+    return SetFitModel.from_pretrained(str(model_dir), tokenizer_kwargs={"fix_mistral_regex": True})

cryptic_cti-0.1.0.dist-info/METADATA

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: cryptic-cti
+Version: 0.1.0
+Summary: Multilingual CTI collections pipeline for normalizing and structuring cybercrime leads
+Author: Cosmic Octopus
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown

cryptic_cti-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+file_utils.py,sha256=vDL4w0lNth2nlgjlGkGWGBg4O3KbR71TJIaHdqwmAiM,1240
+classification/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+classification/utils.py,sha256=9eIK4DixydVfmxMlp2pR9mC-3x1Im5nDm9_97CBhofs,363
+extraction/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extraction/base.py,sha256=MyS5T0QWT9Lf7qbhBXNbXrLfBMuEtXlkTjs1DEtJAvA,151
+extraction/engine.py,sha256=7sdecf8YGkR7CLmmrgxsaloB-RrVlxev5cCkmYnFA_4,493
+extraction/gliner_utils.py,sha256=TNbq-g2CbyZOT5rLeV1iJhPngnDFGbk6Ereulobj3cE,1361
+extraction/spacy_utils.py,sha256=P_dWVKKMBVMW4gafjD3uWZ9m5zsmRTmKcew9SyHDtik,970
+models/gliner_model.py,sha256=LGo59BkjYstgdE_znB8Ce3_vQmGnM_-2fmkMoYwLFHA,229
+normalization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+normalization/utils.py,sha256=iOOkS2oBfHWzXczXlYRHh84tMxcO8gpQmc5mJU7OFGE,560
+output/output_objects.py,sha256=wBLa7EeDX_qrJ47RL3rytPMrMPGYa00GysLWvtz7vKQ,4507
+preprocessing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+preprocessing/chunking.py,sha256=IBK7zNwuAIV_6XCVuq7h3os0Ex8uGxNRNtZ3_hflP5I,2826
+cryptic_cti-0.1.0.dist-info/METADATA,sha256=GkuyLavom3copHxYDXpg0WkeDwicrrxzIwCy8aG5mJA,244
+cryptic_cti-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+cryptic_cti-0.1.0.dist-info/top_level.txt,sha256=RPUozcvsB6DIKdF7Q8gnsQqUrQTzmQGOQjnoee7hYlI,88
+cryptic_cti-0.1.0.dist-info/RECORD,,

cryptic_cti-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

cryptic_cti-0.1.0.dist-info/top_level.txt

@@ -0,0 +1,8 @@
+__init__
+classification
+extraction
+file_utils
+models
+normalization
+output
+preprocessing

extraction/base.py

@@ -0,0 +1,6 @@
+from abc import ABC, abstractmethod
+
+class ExtractionRunner(ABC):
+    @abstractmethod
+    def extract(self, text: str) -> list[dict]:
+        pass

extraction/engine.py

@@ -0,0 +1,12 @@
+from src.extraction.gliner_utils import GlinerRunner
+from src.extraction.spacy_utils import SpacyRunner
+
+class ExtractionEngine:
+    def __init__(self):
+        self.runners = {"spacy": SpacyRunner(), "gliner": GlinerRunner()} # future: RegexRunner(), etc.
+    def run(self, text: str) -> dict:
+        results = {}
+        for name, runner in self.runners:
+            name = runner.__class__.__name__.lower()
+            results[name] = runner.extract(text)
+        return results

extraction/gliner_utils.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+from src.models.gliner_model import get_gliner_model
+from src.extraction.base import ExtractionRunner
+from src.preprocessing.chunking import chunk_block_w_offsets, dedupe_entities
+
+model_name = "urchade/gliner_medium-v2.1"
+
+labels = [
+    "malware or tool name",
+    "credential theft activity",
+    "credential or data type",
+    "platform or application",
+    "actor or group name",
+]
+
+def extract_candidates(text: str) -> list[dict]:
+    model = get_gliner_model()
+    chunks = chunk_block_w_offsets(text)
+    print(f"[extract_candidates] {len(chunks)} chunks | text len={len(text)}")
+    all_entities = []
+    for chunk in chunks:
+        results = model.predict_entities(chunk["text"], labels)
+        for result in results:
+            all_entities.append({
+                "text": result["text"],
+                "label": result["label"],
+                "score": float(result["score"]),
+                "start": chunk["start"] + result["start"],
+                "end": chunk["start"] + result["end"]
+            })
+    return dedupe_entities(all_entities)
+
+
+class GlinerRunner(ExtractionRunner):
+    def __init__(self):
+        self.model = get_gliner_model()
+        self.labels = labels
+    def extract(self, text: str) -> list[dict]:
+        return extract_candidates(text)
+

extraction/spacy_utils.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+import spacy
+import re
+
+from collections_workflow.src.extraction.base import ExtractionRunner
+
+nlp = spacy.load("en_core_web_sm")
+
+def has_chinese(text: str) -> bool:
+    return bool(re.search(r"[\u4e00-\u9fff]", text))
+
+def has_latin(text: str) -> bool:
+    return bool(re.search(r"[A-Za-z]", text))
+
+def detect_lang(text: str) -> str:
+    zh = has_chinese(text)
+    en = has_latin(text)
+    if zh and not en:
+        return "zh"
+    elif en and not zh:
+        return "en"
+    else:
+        return "mixed"
+    return "unknown"
+
+def spacy_prepare(text: str) -> dict:
+    doc = nlp(text)
+    sentences = list(doc.sents)
+    return {"lang": detect_lang(text), "sentence_count": len(sentences), "token_count": len(doc), "sentences": [sent.text.strip() for sent in sentences if sentences]}
+
+
+class SpacyRunner(ExtractionRunner):
+    def extract(self, text: str) -> dict:
+        return spacy_prepare(text)

file_utils.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+import json
+
+def write_jsonl(path: Path, records: list[dict]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as f:
+        for record in records:
+            json_line = json.dumps(record, ensure_ascii=False)
+            f.write(json_line + "\n")
+
+def read_jsonl(path: Path) -> list[dict]:
+    rows: list[dict] = []
+    with path.open("r", encoding="utf-8") as f:
+        for line_no, line in enumerate(f, start=1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rows.append(json.loads(line))
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Invalid JSONL in {path} at line {line_no}: {e}") from e
+    return rows
+
+
+def latest_matching_file(directory: Path, pattern: str) -> Path:
+    matches = [p for p in directory.glob(pattern) if p.is_file()]
+    if not matches:
+        raise FileNotFoundError(f"No files found matching {pattern} in {directory}")
+    return max(matches, key=lambda p: p.stat().st_mtime)
+
+def load_json(path: Path) -> dict:
+    with path.open("r", encoding="utf-8") as f:
+        return json.load(f)
+    

models/gliner_model.py

@@ -0,0 +1,11 @@
+from gliner import GLiNER
+
+model_name = "urchade/gliner_medium-v2.1"
+
+_model = None
+
+def get_gliner_model():
+    global _model
+    if _model is None:
+        _model = GLiNER.from_pretrained(model_name)
+    return _model

normalization/utils.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+def normalize_key(value: str) -> str:
+    return " ".join(value.strip().lower().split())
+
+def normalize_value(value: str, mapping: dict[str, str]) -> tuple[str, bool]:
+    cleaned = value.strip()
+    key = normalize_key(cleaned)
+    if key in mapping:
+        return mapping[key], True
+    return cleaned, False
+
+
+def dedupe_preserve_order(values: list) -> list:
+    seen = set()
+    out = []
+    for v in values:
+        if v not in seen:
+            seen.add(v)
+            out.append(v)
+    return out

output/output_objects.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+from uuid import uuid4
+
+def norm_fieldname(field_name: str) -> str:
+    return field_name.strip().lower()
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+@dataclass(slots=True)
+class Relationship:
+    type: str
+    target_id: str
+    description: str = ""
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+@dataclass(slots=True)
+class Output:
+    id: str = field(default_factory=lambda: str(uuid4()))
+    type: str = ""
+    generated_at: str = field(default_factory=utc_now_iso)
+    producer: str = ""
+    source_ids: list[str] = field(default_factory=list)
+    confidence: int | None = None
+    tlp: str = "TLP:CLEAR"
+    tags: list[str] = field(default_factory=list)
+    relationships: list[Relationship] = field(default_factory=list)
+    summary: str = ""
+    allowed_tlp = {"TLP:CLEAR", "TLP:GREEN", "TLP:AMBER", "TLP:RED"}
+    payload: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if self.confidence is not None and not (0 <= self.confidence <= 100):
+            raise ValueError("confidence must be between 0 and 100")
+        self.source_ids = self.dedupe("source_ids")
+        self.tags = self.dedupe("tags")
+        if self.tlp not in self.allowed_tlp:
+            raise ValueError(
+                f"tlp must be one of {sorted(self.allowed_tlp)}, got {self.tlp!r}")
+
+    def dedupe(self, field_name: str) -> list[Any]:
+        field_name = norm_fieldname(field_name)
+        if field_name not in self.__dataclass_fields__:
+            raise ValueError(f"Invalid field name: {field_name}")
+        values = getattr(self, field_name)
+        if not isinstance(values, list):
+            raise ValueError(f"Field {field_name} is not a list, cannot dedupe")
+        out = []
+        for value in values:
+            if value not in out:
+                out.append(value)
+        return out
+
+    def set_field(self, field_name: str, value: Any) -> None:
+        field_name = norm_fieldname(field_name)
+        if field_name not in self.__dataclass_fields__:
+            raise ValueError(f"Invalid field name: {field_name}")
+        elif value is None:
+            raise ValueError(f"Value for {field_name} cannot be empty")
+        elif field_name in {"source_ids", "tags", "relationships"}:
+            raise TypeError(f"use .add_to() to add values to {field_name}")
+        elif field_name == "confidence" and not (0 <= value <= 100):
+            raise ValueError("confidence must be between 0 and 100")
+        elif field_name == "tlp" and value not in self.allowed_tlp:
+            raise ValueError(f"tlp must be one of {sorted(self.allowed_tlp)}, got {value!r}")
+        else:
+            setattr(self, field_name, value)
+
+    def add_to(self, field_name: str, value: Any) -> None:
+        field_name = norm_fieldname(field_name)
+        if field_name not in self.__dataclass_fields__:
+            raise ValueError(f"Invalid field name: {field_name}")
+        current_value = getattr(self, field_name)
+        if not isinstance(current_value, list):
+            raise ValueError(f"Field {field_name} is not a list, use .set_field() instead")
+        elif field_name == "relationships":
+            if isinstance(value, list):
+                if not all(isinstance(item, Relationship) for item in value):
+                    raise TypeError(f"All items in {field_name} must be of type Relationship")
+            elif not isinstance(value, Relationship):
+                raise TypeError(f"relationships must be of type Relationship, got {type(value)}")
+        if isinstance(value, list):
+            current_value.extend(value)
+        else:
+            current_value.append(value)
+        current_value[:] = self.dedupe(field_name)
+        
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "type": self.type,
+            "generated_at": self.generated_at,
+            "producer": self.producer,
+            "source_ids": self.source_ids,
+            "confidence": self.confidence,
+            "tlp": self.tlp,
+            "tags": self.tags,
+            "relationships": [rel.to_dict() for rel in self.relationships],
+            "summary": self.summary,
+            "payload": self.payload,
+        }

preprocessing/chunking.py

@@ -0,0 +1,98 @@
+print(f"starting spacy import...")
+import spacy
+
+
+_nlp = None
+def get_nlp():
+    global _nlp
+    if _nlp is None:
+        _nlp = spacy.load("en_core_web_sm")
+    return _nlp
+
+
+def sentence_chunks(text: str, target_chars: int = 900, overlap_sentences: int = 1) -> list[str]:
+    nlp = get_nlp()
+    doc = nlp(text)
+    sentences = [sent.text.strip() for sent in doc.sents if sent.text.strip()]
+    if not sentences:
+        stripped = text.strip()
+        return [stripped] if stripped else []
+    chunks: list[str] = []
+    current: list[str] = []
+    i = 0
+    while i < len(sentences):
+        current = []
+        current_len = 0
+        start_i = i
+        while i < len(sentences):
+            sentence = sentences[i]
+            projected = current_len + len(sentence) + (1 if current else 0)
+            if current and projected > target_chars:
+                break
+            current.append(sentence)
+            current_len = projected
+            i += 1
+        chunk = " ".join(current).strip()
+        if chunk:
+            chunks.append(chunk)
+        if i >= len(sentences):
+            break
+        i = max(start_i + 1, i - overlap_sentences)
+    return chunks
+
+
+def char_chunks(text: str, target_chars: int = 900, overlap_chars: int = 150) -> list[str]:
+    text = text.strip()
+    if not text:
+        return []
+    chunks = []
+    start = 0
+    n = len(text)
+    while start < n:
+        end = min(start + target_chars, n)
+        chunk = text[start:end].strip()
+        if chunk:
+            chunks.append(chunk)
+        if end >= n:
+            break
+        start = max(end - overlap_chars, start + 1)
+    return chunks
+
+def chunk_block(block: str) -> list[str]:
+    block = block.strip()
+    if not block:
+        return []
+    if len(block) <= 1200 and "\n" not in block:
+        return [block]
+    if block.count(".") > 2:
+        try:
+            return sentence_chunks(block, target_chars=700, overlap_sentences=2)
+        except Exception as e:
+            print(f"Error during sentence chunking: {e}")
+            pass
+    return char_chunks(block, target_chars=700, overlap_chars=200)
+
+def chunk_block_w_offsets(block: str):
+    chunks = []
+    offset = 0
+    raw_chunks = chunk_block(block) 
+    for chunk in raw_chunks:
+        start = block.find(chunk, offset)
+        end = start + len(chunk)
+        chunks.append({
+            "text": chunk,
+            "start": start,
+            "end": end
+        })
+        offset = end
+    return chunks
+
+def dedupe_entities(entities):
+    seen = set()
+    result = []
+    for e in entities:
+        key = (e["text"], e["start"], e["end"], e["label"])
+        if key not in seen:
+            seen.add(key)
+            result.append(e)
+    return result