taranis-models 1.3.3.dev2__tar.gz → 1.3.3.dev4__tar.gz

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taranis-models
-Version: 1.3.3.dev2
+Version: 1.3.3.dev4
 Summary: Taranis AI Models
 Maintainer-email: AIT <benjamin.akhras@ait.ac.at>
 License-Expression: EUPL-1.2
@@ -17,6 +17,9 @@ License-File: LICENSE.md
 Requires-Dist: pydantic
 Requires-Dist: pydantic-settings
 Requires-Dist: python-dotenv
+Requires-Dist: langcodes
+Requires-Dist: requests
+Requires-Dist: beautifulsoup4
 Provides-Extra: dev
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pytest; extra == "dev"

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/models/admin.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from functools import cached_property
 from typing import Any, Literal
 
 from pydantic import AnyUrl, Field, SecretStr, field_serializer
@@ -63,7 +62,6 @@ class Address(TaranisBaseModel):
 class Organization(TaranisBaseModel):
     _core_endpoint = "/config/organizations"
     _model_name = "organization"
-    _search_fields = ["name", "description"]
 
     id: int | None = None
     name: str = ""
@@ -82,7 +80,6 @@ class Permission(TaranisBaseModel):
 class ACL(TaranisBaseModel):
     _core_endpoint = "/config/acls"
     _model_name = "acl"
-    _search_fields = ["name", "description"]
     _pretty_name = "ACL"
 
     id: int | None = None
@@ -108,7 +105,6 @@ class ParameterValue(TaranisBaseModel):
 class Worker(TaranisBaseModel):
     _core_endpoint = "/config/worker-types"
     _model_name = "worker_type"
-    _search_fields = ["name", "description"]
     _pretty_name = "Worker Type"
 
     id: str | None = None
@@ -122,7 +118,6 @@ class Worker(TaranisBaseModel):
 class Role(TaranisBaseModel):
     _core_endpoint = "/config/roles"
     _model_name = "role"
-    _search_fields = ["name", "description"]
     _pretty_name = "Role"
 
     id: int | None = None
@@ -135,7 +130,6 @@ class Role(TaranisBaseModel):
 class User(TaranisBaseModel):
     _core_endpoint = "/config/users"
     _model_name = "user"
-    _search_fields = ["name", "username"]
 
     id: int | None = None
     name: str
@@ -180,7 +174,6 @@ class WordList(TaranisBaseModel):
     _core_endpoint = "/config/word-lists"
     _model_name = "word_list"
     _pretty_name = "Word List"
-    _search_fields = ["name", "description"]
 
     id: int | None = None
     name: str
@@ -205,19 +198,11 @@ class OSINTSource(TaranisBaseModel):
     enabled: bool | None = True
     status: TaskResult | None = None
 
-    @cached_property
-    def search_field(self) -> str:
-        search = f"{self.name} {self.description} "
-        if self.parameters:
-            search += " ".join(self.parameters.values())
-        return search
-
 
 class OSINTSourceGroup(TaranisBaseModel):
     _core_endpoint = "/config/osint-source-groups"
     _model_name = "osint_source_group"
     _pretty_name = "OSINT Source Group"
-    _search_fields = ["name", "description"]
 
     id: str | None = None
     name: str
@@ -236,7 +221,6 @@ class ProductType(TaranisBaseModel):
     _core_endpoint = "/config/product-types"
     _model_name = "product_type"
     _pretty_name = "Product Type"
-    _search_fields = ["title", "description"]
 
     id: int | None = None
     title: str
@@ -251,7 +235,6 @@ class PublisherPreset(TaranisBaseModel):
     _core_endpoint = "/config/publisher-presets"
     _model_name = "publisher_preset"
     _pretty_name = "Publisher Preset"
-    _search_fields = ["name", "description"]
 
     id: str | None = None
     name: str
@@ -285,7 +268,6 @@ class ReportItemType(TaranisBaseModel):
     _core_endpoint = "/config/report-item-types"
     _model_name = "report_item_type"
     _pretty_name = "Report Item Type"
-    _search_fields = ["title", "description"]
 
     id: int | None = None
     title: str
@@ -316,7 +298,6 @@ class Attribute(TaranisBaseModel):
     _core_endpoint = "/config/attributes"
     _model_name = "attribute"
     _pretty_name = "Attribute"
-    _search_fields = ["name", "description"]
 
     id: int | None = None
     name: str
@@ -330,7 +311,6 @@ class Bot(TaranisBaseModel):
     _core_endpoint = "/config/bots"
     _model_name = "bot"
     _pretty_name = "Bot"
-    _search_fields = ["name", "description"]
 
     id: str | None = None
     name: str
@@ -345,7 +325,6 @@ class Connector(TaranisBaseModel):
     _core_endpoint = "/config/connectors"
     _model_name = "connector"
     _pretty_name = "Connector"
-    _search_fields = ["name", "description"]
 
     id: str | None = None
     name: str

taranis_models-1.3.3.dev4/models/assess.py

@@ -0,0 +1,158 @@
+import contextlib
+import hashlib
+import re
+from datetime import datetime
+from typing import Any, Literal, Self
+from urllib.parse import quote
+
+from bs4 import BeautifulSoup
+from langcodes import Language
+from langcodes.tag_parser import LanguageTagError
+from pydantic import ValidationInfo, field_validator, model_validator
+
+from models.base import TaranisBaseModel
+
+
+class NewsItem(TaranisBaseModel):
+    _core_endpoint = "/assess/news-items"
+
+    osint_source_id: str
+    id: str | None = None
+    hash: str | None = None
+    title: str | None = None
+    author: str | None = None
+    review: str | None = None
+    content: str | None = None
+    link: str | None = None
+    source: str | None = None
+    published: datetime | None = None
+    collected: datetime | None = None
+    updated: datetime | None = None
+    attributes: list[str | dict[str, Any]] | None = None
+    story_id: str | None = None
+    language: str | None = None
+    last_change: str | None = None
+
+    @classmethod
+    def from_input(cls, data: Any) -> Self:
+        """Untrusted input: run validators + sanitizers."""
+        return cls.model_validate(data)
+
+    @classmethod
+    def from_db(cls, data: dict[str, Any]) -> Self:
+        """Trusted DB row: skip *all* validators/sanitizers."""
+        return cls.model_construct(**data)
+
+    @field_validator("language", mode="before")
+    @classmethod
+    def normalize_language_code(cls, v: str, info: ValidationInfo) -> str:
+        if v:
+            try:
+                return Language.get(v).language or ""
+            except (LanguageTagError, ValueError, TypeError):
+                return ""
+        return ""
+
+    @model_validator(mode="after")
+    def check_required_fields(self) -> Self:
+        if not self.title and not self.content:
+            raise ValueError("Either title or content must be provided.")
+        return self
+
+    @field_validator("title", "content", mode="before")
+    @classmethod
+    def sanitize_html(cls, value: str, info: ValidationInfo) -> str:
+        if not value:
+            return ""
+        html = re.sub(r"(?i)( |\xa0)", " ", value, re.DOTALL)
+        return BeautifulSoup(html, "lxml").text
+
+    @field_validator("link", mode="before")
+    @classmethod
+    def sanitize_url(cls, url: str, info: ValidationInfo) -> str:
+        return quote(url or "", safe="/:@?&=+$,;")
+
+    @model_validator(mode="after")
+    def ensure_hash(self) -> Self:
+        if not self.hash:
+            title = self.title or ""
+            link = self.link or ""
+            self.hash = hashlib.sha256((title + link).encode()).hexdigest()
+        return self
+
+    @field_validator("published", "collected", mode="before")
+    @classmethod
+    def sanitize_date(cls, date: str | None | datetime) -> datetime:
+        if isinstance(date, datetime):
+            return date
+        if isinstance(date, str):
+            with contextlib.suppress(ValueError):
+                return datetime.fromisoformat(date)
+        return datetime.now()
+
+
+class StoryTag(TaranisBaseModel):
+    name: str
+    size: int
+    type: str | None = None
+
+
+class Story(TaranisBaseModel):
+    _core_endpoint = "/assess/stories"
+    _model_name = "story"
+    _pretty_name = "Story"
+    _cache_timeout = 30
+
+    id: str | None = None
+    title: str | None = None
+    description: str | None = None
+    created: datetime | None = None
+    updated: datetime | None = None
+    last_change: str | None = None
+    news_items: list[NewsItem] | None = None
+    links: list[str] | None = None
+    important: bool | None = None
+    read: bool | None = None
+    likes: int | None = None
+    dislikes: int | None = None
+    user_vote: Literal["like", "dislike", "", None] = None
+    summary: str | None = None
+    relevance: int | None = None
+    comments: str | None = None
+    in_reports_count: int | None = None
+    tags: list[dict[str, Any]] | None = None
+    attributes: list[dict[str, Any]] | None = None
+
+
+class AssessSource(TaranisBaseModel):
+    id: str | None = None
+    icon: str | None = None
+    name: str
+    type: str | None = None
+
+
+class FilterLists(TaranisBaseModel):
+    _core_endpoint = "/assess/filterlists"
+    _model_name = "filter_lists"
+    _pretty_name = "Filter Lists"
+
+    tags: list[str] = []
+    sources: list[AssessSource] = []
+    groups: list[dict[str, Any]] = []
+
+
+class StoryUpdatePayload(TaranisBaseModel):
+    vote: Literal["like", "dislike", ""] | None = None
+    important: bool | None = None
+    read: bool | None = None
+    title: str | None = None
+    description: str | None = None
+    comments: str | None = None
+    summary: str | None = None
+    tags: list[dict[str, Any]] | None = None
+    attributes: list[dict[str, Any]] | None = None
+
+
+class BulkAction(TaranisBaseModel):
+    story_ids: list[str] = []
+    payload: StoryUpdatePayload | None = None

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/models/asset.py

@@ -1,11 +1,11 @@
 from pydantic import Field
+
 from models.base import TaranisBaseModel
 
 
 class Asset(TaranisBaseModel):
     _core_endpoint = "/assets"
     _model_name = "asset"
-    _search_fields = ["name"]
     _pretty_name = "Assets"
 
     id: int | None = None

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/models/product.py

@@ -1,5 +1,8 @@
 from datetime import datetime
+from typing import Self
+
 from pydantic import Field, ValidationInfo, field_validator
+from requests import Response
 
 from models.base import TaranisBaseModel
 
@@ -8,11 +11,17 @@ class WorkerProduct(TaranisBaseModel):
     data: bytes | None = None
     mime_type: str | None = None
 
+    @classmethod
+    def from_response(cls, response: Response) -> Self:
+        return cls(
+            data=response.content,
+            mime_type=response.headers.get("Content-Type", ""),
+        )
+
 
 class Product(TaranisBaseModel):
     _core_endpoint = "/publish/products"
     _model_name = "product"
-    _search_fields = ["title", "description"]
     _pretty_name = "Product"
 
     id: str | None = None
@@ -27,27 +36,9 @@ class Product(TaranisBaseModel):
     render_result: str | None = None
     mime_type: str | None = None
 
-    @field_validator("auto_publish", mode="before")
-    @classmethod
-    def normalize_auto_publish(cls, value):
-        if isinstance(value, str):
-            lowered = value.strip().lower()
-            if lowered in {"yes", "true", "1", "on"}:
-                return True
-            if lowered in {"no", "false", "0", "off", ""}:
-                return False
-        return value
-
-    @field_validator("default_publisher", mode="before")
-    @classmethod
-    def empty_default_to_none(cls, value):
-        if isinstance(value, str) and not value.strip():
-            return None
-        return value
-
     @field_validator("default_publisher", mode="after")
     @classmethod
-    def require_default_when_autopublish(cls, value, info: ValidationInfo):
+    def require_default_when_autopublish(cls, value: str, info: ValidationInfo):
         auto_publish = info.data.get("auto_publish")
         if auto_publish and not value:
             raise ValueError("default_publisher is required when auto_publish is enabled")

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/models/report.py

@@ -1,5 +1,6 @@
-from typing import ClassVar, Final
 from datetime import datetime
+from typing import ClassVar
+
 from pydantic import Field
 
 from models.admin import ReportItemAttribute
@@ -20,7 +21,6 @@ class ReportItemAttributeGroup(TaranisBaseModel):
 class ReportItem(TaranisBaseModel):
     _core_endpoint: ClassVar[str] = "/analyze/report-items"
     _model_name: ClassVar[str] = "report"
-    _search_fields: Final[list[str]] = ["title"]
     _pretty_name: ClassVar[str] = "Report"
 
     id: str | None = None

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/models/user.py

@@ -32,7 +32,6 @@ class ProfileSettings(TaranisBaseModel):
 class UserProfile(TaranisBaseModel):
     _core_endpoint = "/users"
     _model_name = "user_profile"
-    _search_fields = ["name", "username"]
 
     id: int | None = None
     username: str = ""

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/pyproject.toml

@@ -18,6 +18,9 @@ dependencies = [
     "pydantic",
     "pydantic-settings",
     "python-dotenv",
+    "langcodes",
+    "requests",
+    "beautifulsoup4",
 ]
 dynamic = ["version"]
 

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/taranis_models.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taranis-models
-Version: 1.3.3.dev2
+Version: 1.3.3.dev4
 Summary: Taranis AI Models
 Maintainer-email: AIT <benjamin.akhras@ait.ac.at>
 License-Expression: EUPL-1.2
@@ -17,6 +17,9 @@ License-File: LICENSE.md
 Requires-Dist: pydantic
 Requires-Dist: pydantic-settings
 Requires-Dist: python-dotenv
+Requires-Dist: langcodes
+Requires-Dist: requests
+Requires-Dist: beautifulsoup4
 Provides-Extra: dev
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pytest; extra == "dev"

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/taranis_models.egg-info/requires.txt

@@ -1,6 +1,9 @@
 pydantic
 pydantic-settings
 python-dotenv
+langcodes
+requests
+beautifulsoup4
 
 [dev]
 ruff

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/typings/models/__init__.pyi

@@ -3,10 +3,13 @@ This type stub file was generated by pyright.
 """
 
 from models.admin import ACL, Address, Job, Organization, Permission, ProductType, ReportItemType, Role, Template, User
-from models.types import TLPLevel, WORKER_CATEGORY, WORKER_TYPES
 from models.base import TaranisBaseModel
 from models.dashboard import Dashboard, TrendingCluster
+from models.types import WORKER_CATEGORY, WORKER_TYPES, TLPLevel
 
+"""
+This type stub file was generated by pyright.
+"""
 __all__ = [
     "ACL",
     "Job",

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/typings/models/admin.pyi

@@ -2,24 +2,30 @@
 This type stub file was generated by pyright.
 """
 
-from pydantic import AnyUrl
+from datetime import datetime
 from functools import cached_property
 from typing import Any, Literal
-from datetime import datetime
+
+from pydantic import AnyUrl
+
 from models.base import TaranisBaseModel
 from models.types import (
-    AttributeType,
     BOT_TYPES,
     COLLECTOR_TYPES,
     CONNECTOR_TYPES,
-    ItemType,
     PRESENTER_TYPES,
     PUBLISHER_TYPES,
-    TLPLevel,
     WORKER_CATEGORY,
     WORKER_TYPES,
+    AttributeType,
+    ItemType,
+    TLPLevel,
 )
 
+"""
+This type stub file was generated by pyright.
+"""
+
 class Job(TaranisBaseModel):
     _core_endpoint = ...
     _model_name = ...

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/typings/models/assess.pyi

@@ -3,11 +3,17 @@ This type stub file was generated by pyright.
 """
 
 from datetime import datetime
-from pydantic import field_validator
-from typing import Literal
 from functools import cached_property
+from typing import Literal
+
+from pydantic import field_validator
+
 from models.base import TaranisBaseModel
 
+"""
+This type stub file was generated by pyright.
+"""
+
 class NewsItem(TaranisBaseModel):
     _core_endpoint = ...
     osint_source_id: str
@@ -23,8 +29,7 @@ class NewsItem(TaranisBaseModel):
     attributes: list[str | dict] | None = ...
     language: str | None = ...
     @field_validator("language", mode="before")
-    def normalize_language_code(cls, v):  # -> str | None:
-        ...
+    def normalize_language_code(cls, v): ...
 
 class StoryTag(TaranisBaseModel):
     name: str

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/typings/models/asset.pyi

@@ -4,6 +4,10 @@ This type stub file was generated by pyright.
 
 from models.base import TaranisBaseModel
 
+"""
+This type stub file was generated by pyright.
+"""
+
 class Asset(TaranisBaseModel):
     _core_endpoint = ...
     _model_name = ...

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/typings/models/base.pyi

@@ -2,9 +2,13 @@
 This type stub file was generated by pyright.
 """
 
-from pydantic import BaseModel
 from typing import ClassVar, TypeVar
 
+from pydantic import BaseModel
+
+"""
+This type stub file was generated by pyright.
+"""
 T = TypeVar("T", bound="TaranisBaseModel")
 
 class TaranisBaseModel(BaseModel):
@@ -13,5 +17,4 @@ class TaranisBaseModel(BaseModel):
     _model_name: ClassVar[str] = ...
     _pretty_name: ClassVar[str] = ...
     model_config = ...
-    def model_dump(self, *args, **kwargs):  # -> dict[str, Any]:
-        ...
+    def model_dump(self, *args, **kwargs): ...

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/typings/models/dashboard.pyi

@@ -3,8 +3,13 @@ This type stub file was generated by pyright.
 """
 
 from datetime import datetime
-from models.base import TaranisBaseModel
+
 from models.assess import StoryTag
+from models.base import TaranisBaseModel
+
+"""
+This type stub file was generated by pyright.
+"""
 
 class Dashboard(TaranisBaseModel):
     _core_endpoint = ...

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/typings/models/product.pyi

@@ -3,8 +3,13 @@ This type stub file was generated by pyright.
 """
 
 from datetime import datetime
+
 from models.base import TaranisBaseModel
 
+"""
+This type stub file was generated by pyright.
+"""
+
 class WorkerProduct(TaranisBaseModel):
     data: bytes | None = ...
     mime_type: str | None = ...

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/typings/models/report.pyi

@@ -2,12 +2,17 @@
 This type stub file was generated by pyright.
 """
 
-from typing import ClassVar, Final
 from datetime import datetime
+from typing import ClassVar, Final
+
 from models.admin import ReportItemAttribute
 from models.assess import Story
 from models.base import TaranisBaseModel
 
+"""
+This type stub file was generated by pyright.
+"""
+
 class ReportItemCpe(TaranisBaseModel):
     id: int | None = ...
     value: str

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/typings/models/types.pyi

@@ -4,6 +4,10 @@ This type stub file was generated by pyright.
 
 from enum import IntEnum, StrEnum
 
+"""
+This type stub file was generated by pyright.
+"""
+
 class TLPLevel(StrEnum):
     CLEAR = ...
     GREEN = ...

{taranis_models-1.3.3.dev2 → taranis_models-1.3.3.dev4}/typings/models/user.pyi

@@ -3,8 +3,13 @@ This type stub file was generated by pyright.
 """
 
 from typing import Any
+
 from models.base import TaranisBaseModel
 
+"""
+This type stub file was generated by pyright.
+"""
+
 class ProfileSettingsDashboard(TaranisBaseModel):
     show_trending_clusters: bool = ...
     show_charts: bool = ...