lightman_ai 0.20.1__tar.gz → 0.21.0__tar.gz

{lightman_ai-0.20.1 → lightman_ai-0.21.0}/PKG-INFO

@@ -1,33 +1,19 @@
 Metadata-Version: 2.4
 Name: lightman_ai
-Version: 0.20.1
-Summary: Cybersecurity news aggregator.
+Version: 0.21.0
+Summary: LLM-powered cybersecurity news aggregator.
 Author-email: sdn4z <imsdn4z@gmail.com>
 License-File: LICENSE
 Requires-Python: <4,>=3.13
 Requires-Dist: click<9.0.0,>=8.1.7
 Requires-Dist: httpx<1.0.0,>=0.28.0
-Requires-Dist: logfire>=3.25.0
 Requires-Dist: pydantic-ai-slim[google,openai]>=0.4.4
-Requires-Dist: pydantic-settings[dotenv]<3.0.0,>=2.9.1
+Requires-Dist: pydantic-settings<3.0.0,>=2.9.1
 Requires-Dist: python-dotenv<2.0.0,>=1.1.1
-Requires-Dist: sentry-sdk<3.0.0,>=2.21.0
 Requires-Dist: stamina<26.0.0,>=25.1.0
-Requires-Dist: tiktoken<1.0.0,>=0.9.0
 Requires-Dist: tomlkit<1.0.0,>=0.13.3
-Provides-Extra: lint
-Requires-Dist: mypy<2.0.0,>=1.1.1; extra == 'lint'
-Requires-Dist: ruff<1.0.0,>=0.11.0; extra == 'lint'
-Provides-Extra: local
-Requires-Dist: codespell<3.0.0,>=2.2.4; extra == 'local'
-Requires-Dist: commitizen<5.0.0,>=4.8.3; extra == 'local'
-Requires-Dist: ipdb<1.0.0,>=0.13.13; extra == 'local'
-Requires-Dist: pdbpp<1.0.0,>=0.11.6; extra == 'local'
-Requires-Dist: pre-commit<4.0.0,>=3.2.2; extra == 'local'
-Provides-Extra: test
-Requires-Dist: pytest-asyncio<1.0.0,>=0.26.0; extra == 'test'
-Requires-Dist: pytest-cov<6.0.0,>=5.0.0; extra == 'test'
-Requires-Dist: pytest<9.0.0,>=8.0.0; extra == 'test'
+Provides-Extra: sentry
+Requires-Dist: sentry-sdk<3.0.0,>=2.21.0; extra == 'sentry'
 Description-Content-Type: text/markdown
 
 # 🔍 Lightman AI

lightman_ai-0.21.0/VERSION

@@ -0,0 +1 @@
+v0.21.0

{lightman_ai-0.20.1 → lightman_ai-0.21.0}/pyproject.toml

@@ -9,41 +9,23 @@ dependencies = [
     "python-dotenv<2.0.0,>=1.1.1",
     "click<9.0.0,>=8.1.7",
     "stamina<26.0.0,>=25.1.0",
-    "pydantic-settings[dotenv]<3.0.0,>=2.9.1",
-    "tiktoken<1.0.0,>=0.9.0",
+    "pydantic-settings<3.0.0,>=2.9.1",
     "tomlkit<1.0.0,>=0.13.3",
     "pydantic-ai-slim[google,openai]>=0.4.4",
-    "logfire>=3.25.0",
-    "sentry-sdk>=2.21.0,<3.0.0",
 ]
 name = "lightman_ai"
-description = "Cybersecurity news aggregator."
+description = "LLM-powered cybersecurity news aggregator."
 readme = "README.md"
 dynamic = ["version"]
 
 [project.scripts]
 lightman-ai = "lightman_ai.cli:entry_point"
 
-
 [project.optional-dependencies]
-test = [
-    "pytest<9.0.0,>=8.0.0",
-    "pytest-cov<6.0.0,>=5.0.0",
-    "pytest-asyncio<1.0.0,>=0.26.0",
-]
-lint = [
-    "mypy<2.0.0,>=1.1.1",
-    "ruff<1.0.0,>=0.11.0",
-]
-local = [
-    "ipdb<1.0.0,>=0.13.13",
-    "pdbpp<1.0.0,>=0.11.6",
-    "pre-commit<4.0.0,>=3.2.2",
-    "commitizen<5.0.0,>=4.8.3",
-    "codespell<3.0.0,>=2.2.4",
+sentry = [
+    "sentry-sdk>=2.21.0,<3.0.0",
 ]
 
-
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
@@ -75,8 +57,6 @@ addopts = """
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
 
-[tool.logfire]
-ignore_no_config=true
 
 [tool.ruff]
 target-version = "py312"
@@ -159,3 +139,22 @@ exclude_lines = [
     "pragma: no cover",
     "raise NotImplementedError",
 ]
+
+[dependency-groups]
+test = [
+    "pytest<9.0.0,>=8.0.0",
+    "pytest-cov<6.0.0,>=5.0.0",
+    "pytest-asyncio<1.0.0,>=0.26.0",
+    "freezegun>=1.5.3",
+]
+lint = [
+    "mypy<2.0.0,>=1.1.1",
+    "ruff<1.0.0,>=0.11.0",
+]
+local = [
+    "ipdb<1.0.0,>=0.13.13",
+    "pdbpp<1.0.0,>=0.11.6",
+    "pre-commit<4.0.0,>=3.2.2",
+    "commitizen<5.0.0,>=4.8.3",
+    "codespell<3.0.0,>=2.2.4",
+]

lightman_ai-0.21.0/src/lightman_ai/article/exceptions.py

@@ -0,0 +1,6 @@
+class ArticleBaseError(Exception):
+    """Base class for article-related models errors."""
+
+
+class NoTimeZoneError(ArticleBaseError):
+    """Exception class for when there is no timezone associated to a date."""

{lightman_ai-0.20.1 → lightman_ai-0.21.0}/src/lightman_ai/article/models.py

@@ -1,14 +1,24 @@
 from abc import ABC
-from typing import override
+from datetime import datetime
+from typing import Self, override
 
-from pydantic import BaseModel
+from lightman_ai.article.exceptions import NoTimeZoneError
+from pydantic import BaseModel, Field, field_validator
 
 
 class BaseArticle(BaseModel, ABC):
     """Base abstract class for all Articles."""
 
-    title: str
-    link: str
+    title: str = Field(..., min_length=1)
+    link: str = Field(..., min_length=1)
+    published_at: datetime = Field(..., description="Must be timezone aware")
+
+    @field_validator("published_at", mode="after")
+    @classmethod
+    def validate_timezone_aware(cls, v: datetime) -> datetime:
+        if v.tzinfo is None:
+            raise NoTimeZoneError("published_at must be timezone aware")
+        return v
 
     @override
     def __eq__(self, value: object) -> bool:
@@ -28,8 +38,7 @@ class SelectedArticle(BaseArticle):
 
 
 class Article(BaseArticle):
-    title: str
-    description: str
+    description: str = Field(..., min_length=1)
 
 
 class BaseArticlesList[TArticle: BaseArticle](BaseModel):
@@ -40,11 +49,18 @@ class BaseArticlesList[TArticle: BaseArticle](BaseModel):
 
     @property
     def titles(self) -> list[str]:
-        return [new.title for new in self.articles]
+        return [article.title for article in self.articles]
 
     @property
     def links(self) -> list[str]:
-        return [new.link for new in self.articles]
+        return [article.link for article in self.articles]
+
+    @classmethod
+    def get_articles_from_date_onwards(cls, articles: list[TArticle], start_date: datetime) -> Self:
+        if not start_date.tzinfo:
+            raise NoTimeZoneError("A timezone is needed for filtering articles")
+        articles = [article for article in articles if article.published_at >= start_date]
+        return cls(articles=articles)
 
 
 class SelectedArticlesList(BaseArticlesList[SelectedArticle]):

{lightman_ai-0.20.1 → lightman_ai-0.21.0}/src/lightman_ai/cli.py

@@ -1,5 +1,7 @@
 import logging
+from datetime import date, datetime, time
 from importlib import metadata
+from zoneinfo import ZoneInfo
 
 import click
 from dotenv import load_dotenv
@@ -8,6 +10,7 @@ from lightman_ai.constants import DEFAULT_CONFIG_FILE, DEFAULT_CONFIG_SECTION, D
 from lightman_ai.core.config import FileConfig, FinalConfig, PromptConfig
 from lightman_ai.core.exceptions import ConfigNotFoundError, InvalidConfigError, PromptNotFoundError
 from lightman_ai.core.sentry import configure_sentry
+from lightman_ai.core.settings import settings
 from lightman_ai.main import lightman
 
 logger = logging.getLogger("lightman")
@@ -65,6 +68,8 @@ def entry_point() -> None:
         "When set, runs the script without publishing the results to the integrated services, just shows them in stdout."
     ),
 )
+@click.option("--start-date", type=click.DateTime(formats=["%Y-%m-%d"]), help="Start date to retrieve articles")
+@click.option("--today", is_flag=True, help="Retrieve articles from today.")
 def run(
     agent: str,
     prompt: str,
@@ -75,6 +80,8 @@ def run(
     config: str,
     env_file: str,
     dry_run: bool,
+    start_date: date | None,
+    today: bool,
 ) -> int:
     """
     Entrypoint of the application.
@@ -83,6 +90,16 @@ def run(
     """
     load_dotenv(env_file)
     configure_sentry()
+
+    if start_date and today:
+        raise click.UsageError("--today and --start-date cannot be set at the same time.")
+    elif today:
+        start_datetime = datetime.now(ZoneInfo(settings.TIME_ZONE))
+    elif isinstance(start_date, date):
+        start_datetime = datetime.combine(start_date, time(0, 0), tzinfo=ZoneInfo(settings.TIME_ZONE))
+    else:
+        start_datetime = None
+
     try:
         prompt_config = PromptConfig.get_config_from_file(path=prompt_file)
         config_from_file = FileConfig.get_config_from_file(config_section=config, path=config_file)
@@ -107,6 +124,7 @@ def run(
         project_key=config_from_file.service_desk_project_key,
         request_id_type=config_from_file.service_desk_request_id_type,
         model=final_config.model,
+        start_date=start_datetime,
     )
     relevant_articles_metadata = [f"{article.title} ({article.link})" for article in relevant_articles]
     logger.warning("Found these articles: \n- %s", "\n- ".join(relevant_articles_metadata))

lightman_ai-0.21.0/src/lightman_ai/core/sentry.py

@@ -0,0 +1,39 @@
+import logging
+import os
+from importlib import metadata
+
+logger = logging.getLogger("lightman")
+
+
+def configure_sentry() -> None:
+    """Configure Sentry for error tracking."""
+    try:
+        import sentry_sdk
+        from sentry_sdk.integrations.logging import LoggingIntegration
+    except ImportError:
+        logger.warning(
+            "Could not initialize sentry, it is not installed! Add it by installing the project with `lightman-ai[sentry]`."
+        )
+        return
+
+    try:
+        if not os.getenv("SENTRY_DSN"):
+            logger.info("SENTRY_DSN not configured, skipping Sentry initialization")
+            return
+
+        logging_level_str = os.getenv("LOGGING_LEVEL", "ERROR").upper()
+        try:
+            logging_level = getattr(logging, logging_level_str, logging.ERROR)
+        except AttributeError:
+            logger.warning("The specified logging level `%s` does not exist. Defaulting to ERROR.", logging_level_str)
+            logging_level = logging.ERROR
+
+        # Set up logging integration
+        sentry_logging = LoggingIntegration(level=logging.INFO, event_level=logging_level)
+
+        sentry_sdk.init(
+            release=metadata.version("lightman-ai"),
+            integrations=[sentry_logging],
+        )
+    except Exception as e:
+        logger.warning("Could not instantiate Sentry! %s.\nContinuing with the execution.", e)

{lightman_ai-0.20.1 → lightman_ai-0.21.0}/src/lightman_ai/core/settings.py

@@ -7,8 +7,8 @@ class Settings(BaseSettings):
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
 
-    OPENAI_ENCODING: str = "cl100k_base"
     PARALLEL_WORKERS: int = 5
+    TIME_ZONE: str = "UTC"
 
 
 settings = Settings()

{lightman_ai-0.20.1 → lightman_ai-0.21.0}/src/lightman_ai/main.py

@@ -1,5 +1,6 @@
 import asyncio
 import logging
+from datetime import datetime
 
 from lightman_ai.ai.base.agent import BaseAgent
 from lightman_ai.ai.utils import get_agent_class_from_agent_name
@@ -12,8 +13,8 @@ from lightman_ai.sources.the_hacker_news import TheHackerNewsSource
 logger = logging.getLogger("lightman")
 
 
-def _get_articles() -> ArticlesList:
-    return TheHackerNewsSource().get_articles()
+def _get_articles_from_source(start_date: datetime | None = None) -> ArticlesList:
+    return TheHackerNewsSource().get_articles(start_date)
 
 
 def _classify_articles(articles: ArticlesList, agent: BaseAgent) -> SelectedArticlesList:
@@ -60,8 +61,9 @@ def lightman(
     request_id_type: str | None = None,
     dry_run: bool = False,
     model: str | None = None,
+    start_date: datetime | None = None,
 ) -> list[SelectedArticle]:
-    articles: ArticlesList = _get_articles()
+    articles: ArticlesList = _get_articles_from_source(start_date)
 
     agent_class = get_agent_class_from_agent_name(agent)
     agent_instance = agent_class(prompt, model, logger=logger)

{lightman_ai-0.20.1 → lightman_ai-0.21.0}/src/lightman_ai/sources/base.py

@@ -1,8 +1,9 @@
 from abc import ABC, abstractmethod
+from datetime import datetime
 
 from lightman_ai.article.models import ArticlesList
 
 
 class BaseSource(ABC):
     @abstractmethod
-    def get_articles(self) -> ArticlesList: ...
+    def get_articles(self, date: datetime | None = None) -> ArticlesList: ...

lightman_ai-0.21.0/src/lightman_ai/sources/exceptions.py

@@ -0,0 +1,10 @@
+class BaseSourceError(Exception):
+    """Base Exception class for errors."""
+
+
+class MalformedSourceResponseError(BaseSourceError):
+    """Exception for when the respose format does not match the expected one."""
+
+
+class IncompleteArticleFromSourceError(MalformedSourceResponseError):
+    """Exception for when all the mandatory fields could not be retrieved from an article."""

lightman_ai-0.21.0/src/lightman_ai/sources/the_hacker_news.py

@@ -0,0 +1,81 @@
+import logging
+from datetime import datetime
+from typing import override
+from xml.etree import ElementTree
+
+import httpx
+import stamina
+from httpx import Client
+from lightman_ai.article.models import Article, ArticlesList
+from lightman_ai.sources.base import BaseSource
+from lightman_ai.sources.exceptions import IncompleteArticleFromSourceError, MalformedSourceResponseError
+from pydantic import ValidationError
+
+logger = logging.getLogger("lightman")
+
+_RETRY_ON = httpx.TransportError
+_ATTEMPTS = 5
+_TIMEOUT = 5
+
+
+THN_URL = "https://feeds.feedburner.com/TheHackersNews"
+
+
+class TheHackerNewsSource(BaseSource):
+    @override
+    def get_articles(self, date: datetime | None = None) -> ArticlesList:
+        """Return the articles that are present in THN feed."""
+        feed = self.get_feed()
+        articles = self._xml_to_list_of_articles(feed)
+        if date:
+            return ArticlesList.get_articles_from_date_onwards(articles=articles, start_date=date)
+        else:
+            return ArticlesList(articles=articles)
+
+    def get_feed(self) -> str:
+        """Retrieve the TheHackerNews' RSS Feed."""
+        for attempt in stamina.retry_context(
+            on=_RETRY_ON,
+            attempts=_ATTEMPTS,
+            timeout=_TIMEOUT,
+        ):
+            with Client() as http_client, attempt:
+                hacker_news_feed = http_client.get(THN_URL)
+                hacker_news_feed.raise_for_status()
+        return hacker_news_feed.text
+
+    def _xml_to_list_of_articles(self, xml: str) -> list[Article]:
+        try:
+            root = ElementTree.fromstring(xml)
+        except ElementTree.ParseError as e:
+            raise MalformedSourceResponseError(f"Invalid XML format: {e}") from e
+        channel = root.find("channel")
+
+        if channel is None:
+            raise MalformedSourceResponseError("No channel element found in RSS feed")
+        items = channel.findall("item")
+
+        parsed = []
+
+        for item in items:
+            try:
+                title = item.findtext("title", default="").strip()
+                description = self._clean(item.findtext("description", default="").strip())
+                link = item.findtext("link", default="").strip()
+                published_at_str = item.findtext("pubDate", default="").strip()
+
+                if not published_at_str:
+                    logger.exception("Missing publication date. link: `%s`", link)
+                    raise IncompleteArticleFromSourceError()
+                published_at = datetime.strptime(published_at_str, "%a, %d %b %Y %H:%M:%S %z")
+
+                parsed.append(Article(title=title, description=description, link=link, published_at=published_at))
+            except (ValidationError, ValueError) as e:
+                raise IncompleteArticleFromSourceError from e
+
+        return parsed
+
+    @staticmethod
+    def _clean(text: str) -> str:
+        """Remove non-useful characters. Helps cleaning the fields that will be sent to the Agent."""
+        return text.replace("\\n", "").replace("       ", "")

lightman_ai-0.20.1/VERSION

@@ -1 +0,0 @@
-v0.20.1

lightman_ai-0.20.1/src/lightman_ai/core/sentry.py

@@ -1,28 +0,0 @@
-import logging
-import os
-from importlib import metadata
-
-import sentry_sdk
-from sentry_sdk.integrations.logging import LoggingIntegration
-
-
-def configure_sentry() -> None:
-    """Configure Sentry for error tracking and performance monitoring using env vars with fallbacks."""
-    try:
-        if not os.getenv("SENTRY_DSN"):
-            logging.getLogger("lightman").info("SENTRY_DSN not configured, skipping Sentry initialization")
-            return
-
-        # Logging level from ENV
-        logging_level_str = os.getenv("LOGGING_LEVEL", "ERROR").upper()
-        logging_level = getattr(logging, logging_level_str, logging.ERROR)
-
-        # Set up logging integration
-        sentry_logging = LoggingIntegration(level=logging.INFO, event_level=logging_level)
-
-        sentry_sdk.init(
-            release=metadata.version("lightman-ai"),
-            integrations=[sentry_logging],
-        )
-    except Exception as e:
-        logging.getLogger("lightman").warning("Could not instantiate Sentry! %s.\nContinuing with the execution.", e)

lightman_ai-0.20.1/src/lightman_ai/sources/the_hacker_news.py

@@ -1,57 +0,0 @@
-from typing import override
-from xml.etree import ElementTree
-
-import httpx
-import stamina
-from httpx import Client
-from lightman_ai.article.models import Article, ArticlesList
-from lightman_ai.sources.base import BaseSource
-
-_RETRY_ON = httpx.TransportError
-_ATTEMPTS = 5
-_TIMEOUT = 5
-
-
-THN_URL = "https://feeds.feedburner.com/TheHackersNews"
-
-
-class TheHackerNewsSource(BaseSource):
-    @override
-    def get_articles(self) -> ArticlesList:
-        """Return the articles that are present in THN feed."""
-        feed = self.get_feed()
-        articles = self._xml_to_list_of_articles(feed)
-        return ArticlesList(articles=articles)
-
-    def get_feed(self) -> str:
-        """Retrieve the TheHackerNews' RSS Feed."""
-        for attempt in stamina.retry_context(
-            on=_RETRY_ON,
-            attempts=_ATTEMPTS,
-            timeout=_TIMEOUT,
-        ):
-            with Client() as http_client, attempt:
-                hacker_news_feed = http_client.get(THN_URL)
-                hacker_news_feed.raise_for_status()
-        return hacker_news_feed.text
-
-    def _xml_to_list_of_articles(self, xml: str) -> list[Article]:
-        root = ElementTree.fromstring(xml)
-        channel = root.find("channel")
-        assert channel
-        items = channel.findall("item")
-
-        parsed = []
-
-        for item in items:
-            title = item.findtext("title", default="").strip()
-            description = self._clean(item.findtext("description", default="").strip())
-            link = item.findtext("link", default="").strip()
-
-            parsed.append(Article(title=title, description=description, link=link))
-        return parsed
-
-    @staticmethod
-    def _clean(text: str) -> str:
-        """Remove non-useful characters."""
-        return text.replace("\\n", "").replace("       ", "")