lightman-ai 0.16.0__tar.gz

lightman_ai-0.16.0/LICENSE

@@ -0,0 +1,6 @@
+Copyright (c) 2025, Elements Interactive B.V.
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

lightman_ai-0.16.0/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.3
+Name: lightman_ai
+Version: 0.16.0
+Summary: Cybersecurity news aggregator.
+Author: sdn4z
+Author-email: imsdn4z@gmail.com
+Requires-Python: >=3.13,<4
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: httpx (>=0.28.0,<0.29.0)
+Requires-Dist: pydantic-ai[logfire] (>=0.2.0,<0.3.0)
+Requires-Dist: pydantic-settings[dotenv] (>=2.9.1,<3.0.0)
+Requires-Dist: python-dotenv (>=1.1.1,<2.0.0)
+Requires-Dist: stamina (>=25.1.0,<26.0.0)
+Requires-Dist: tiktoken (>=0.9.0,<0.10.0)
+Requires-Dist: tomlkit (>=0.13.3,<0.14.0)

lightman_ai-0.16.0/pyproject.toml

@@ -0,0 +1,243 @@
+[tool.poetry]
+name = "lightman_ai"
+version = "0.16.0"
+description = "Cybersecurity news aggregator."
+authors = ["sdn4z <imsdn4z@gmail.com>"]
+
+[tool.poetry.scripts]
+lightman = "lightman_ai.cli:entry_point"
+
+[tool.poetry.dependencies]
+python = ">=3.13,<4"
+httpx = "^0.28.0"
+python-dotenv = "^1.1.1"
+pydantic-ai = {version = "^0.2.0", extras = ["logfire"]}
+click = "^8.1.7"
+stamina = "^25.1.0"
+pydantic-settings = {extras = ["dotenv"], version = "^2.9.1"}
+tiktoken = "^0.9.0"
+tomlkit = "^0.13.3"
+
+[tool.poetry.group.test.dependencies]
+pytest = "^8.0.0"
+pytest-cov = "^5.0.0"
+pytest-asyncio = "^0.26.0"
+
+[tool.poetry.group.lint.dependencies]
+mypy = "^1.1.1"
+ruff = "^0.11.0"
+
+[tool.poetry.group.local.dependencies]
+ipdb = "^0.13.13"
+pdbpp = "^0.11.6"
+pre-commit = "^3.2.2"
+commitizen = "^4.8.3"
+codespell = "^2.2.4"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+
+[tool.pytest.ini_options]
+addopts = """
+    -s
+    --cov
+    --cov-report term-missing
+    --cov-report=xml:./tests/coverage.xml
+    --junitxml=./tests/junit.xml
+"""
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "session"
+
+[tool.logfire]
+ignore_no_config=true
+
+[tool.ruff]
+target-version = "py312"
+line-length = 120
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = [
+  "B",   # flake8-bugbear
+  "C",   # mccabe
+  "E",   # pycodestyle
+  "W",   # pycodestyle warnings
+  "F",   # pyflakes
+  "I",   # isort
+  "B",   # bugbear
+  "D",   # pydocstyle
+  "I",   # isort
+  "C4",  # comprehensions
+  "SIM", # simplify
+  "N",   # pep8-naming
+  "TRY", # tryceratops
+  "UP",  # pyupgrade
+  "PT",  # pytest-style
+  "TCH", # type-checking
+  "PL",  # pylint
+  "G",   # logging format
+  "TID", # tidy-imports
+]
+ignore = [
+  "E501",    # line-length (enforced by black instead)
+  'D1',      # pydocstyle enforces docstrings everywhere
+  'TRY003',  # tryceratops rule not useful
+  "PLR0913", # pylint rule not useful
+  "B008",    # checks for function calls in default function arguments.
+  'D206',    # conflicts with formatter
+  'W191',    # conflicts with formatter
+  'D203',    # conflicts with formatter
+  'D211',    # conflicts with formatter
+  'D212',    # conflicts with formatter
+  'UP031',   # allow strings formated with %s
+  'PLR2004'  # magic value
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 10
+
+[tool.ruff.lint.pydocstyle]
+convention = "pep257"
+
+[tool.ruff.lint.isort]
+known-third-party = ["lightman_ai", "tests"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = [
+    "ANN", # no enforcement of type annotations in tests
+    "S",   # bandit is not useful in tests
+]
+
+[tool.ruff.lint.flake8-tidy-imports.banned-api]
+"dateutil.tz".msg = "Use `zoneinfo` instead."
+"pytz".msg = "Use `zoneinfo` instead."
+
+
+[tool.mypy]
+python_version = "3.13"
+mypy_path = "src"
+strict = true
+pretty = true
+warn_unused_ignores = true
+enable_error_code = ["explicit-override"]
+
+
+[tool.coverage.run]
+omit = ["tests/*", "eval/*"]
+
+[tool.coverage.report]
+fail_under = 95
+exclude_lines = [
+    "if TYPE_CHECKING:",
+    "pragma: no cover",
+    "raise NotImplementedError",
+]
+
+
+[tool.commitizen]
+version_files = ["pyproject.toml:^version"]
+name = "cz_customize"
+version = "0.16.0"
+tag_format = "v$version"
+
+[tool.commitizen.customize]
+message_template = "{{prefix}}{% if scope %}({{scope}}){% endif %}: {{subject}}{% if body %}\n\n{{body}}{% endif %}{% if is_breaking_change %}\n\nBREAKING CHANGE: {{footer}}{% else %}\n\n{{footer}}{% endif %}"
+
+bump_pattern = "^(BREAKING[\\-\\ ]CHANGE|feat|fix|refactor|perf|ci|docs|style|test|chore|revert|build)(\\(.+\\))?(!)?"
+change_type_order = [
+    "BREAKING CHANGE",
+    "Feat",
+    "Fix",
+    "Refactor",
+    "Perf",
+    "CI",
+    "Docs",
+    "Technical",
+    "Tests",
+]
+bump_message = "bump: version $current_version → $new_version"
+schema = "<type>(<scope>): <subject>\n<BLANK LINE>\n<body>\n<BLANK LINE>\n(BREAKING CHANGE: )<footer>"
+schema_pattern = "(?s)(build|ci|docs|feat|fix|perf|refactor|style|test|chore|revert|bump)(\\(\\S+\\))?!?:( [^\\n\\r]+)((\\n\\n.*)|(\\s*))?$"
+commit_parser = "^(?P<change_type>feat|fix|refactor|perf|BREAKING CHANGE|ci|docs|style|test|chore|revert|build)(?:\\((?P<scope>[^()\\r\\n]*)\\)|\\()?(?P<breaking>!)?:\\s(?P<message>.*)?"
+version_parser = "(?P<version>([0-9]+)\\.([0-9]+)\\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*))?(?:\\+[0-9A-Za-z-]+)?(\\w+)?)"
+changelog_pattern = "^(BREAKING[\\-\\ ]CHANGE|feat|fix|refactor|perf|ci|docs|style|test|chore|revert|build)(\\(.+\\))?(!)?" # same as bump_pattern
+[tool.commitizen.customize.bump_map]
+"^.+!$" = "MAJOR"
+"^BREAKING[\\-\\ ]CHANGE" = "MAJOR"
+"^feat" = "MINOR"
+"^fix" = "PATCH"
+"^refactor" = "PATCH"
+"^perf" = "PATCH"
+"^ci" = "PATCH"
+"^docs" = "PATCH"
+"^style" = "PATCH"
+"^test" = "PATCH"
+"^chore" = "PATCH"
+"^revert" = "PATCH"
+"^build" = "PATCH"
+
+[tool.commitizen.customize.change_type_map]
+"feat" = "Feat"
+"fix" = "Fix"
+"refactor" = "Refactor"
+"perf" = "Perf"
+"ci" = "CI"
+"docs" = "Docs"
+"style" = "Technical"
+"test" = "Tests"
+"chore" = "Technical"
+"revert" = "Fix"
+"build" = "Technical"
+
+[[tool.commitizen.customize.questions]]
+type = "list"
+name = "prefix"
+message = "Select the type of change you are committing"
+choices = [
+    { value = "fix", name = "fix: A bug fix. Correlates with PATCH in SemVer", key = "x" },
+    { value = "feat", name = "feat: A new feature. Correlates with MINOR in SemVer", key = "f" },
+    { value = "docs", name = "docs: Documentation only changes", key = "d" },
+    { value = "style", name = "style: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)", key = "s" },
+    { value = "refactor", name = "refactor: A code change that neither fixes a bug nor adds a feature", key = "r" },
+    { value = "perf", name = "perf: A code change that improves performance", key = "p" },
+    { value = "test", name = "test: Adding missing or correcting existing tests", key = "t" },
+    { value = "build", name = "build: Changes that affect the build system or external dependencies (example scopes: pip, docker, npm)", key = "b" },
+    { value = "ci", name = "ci: Changes to our CI configuration files and scripts (example scopes: GitLabCI)", key = "c" },
+]
+
+[[tool.commitizen.customize.questions]]
+type = "input"
+name = "scope"
+message = "What is the scope of this change? ((story number, class or file name): (press [enter] to skip)\n"
+
+[[tool.commitizen.customize.questions]]
+type = "input"
+name = "subject"
+message = "Write a short and imperative summary of the code changes: (lower case and no period)\n"
+
+[[tool.commitizen.customize.questions]]
+type = "input"
+name = "body"
+message = "Provide additional contextual information about the code changes: (press [enter] to skip)\n"
+
+[[tool.commitizen.customize.questions]]
+type = "confirm"
+message = "Is this a BREAKING CHANGE? Correlates with MAJOR in SemVer"
+name = "is_breaking_change"
+default = false
+
+[[tool.commitizen.customize.questions]]
+type = "input"
+name = "footer"
+message = "Footer. Information about Breaking Changes and reference issues that this commit closes: (press [enter] to skip)\n"
+
+# TODO: Find a way to include pyproject.toml without the version changes
+[tool.cruft]
+skip = ["pyproject.toml"]
+
+
+[[IgnoredVulns]]
+# False positive for mkdocs, more info: https://github.com/mkdocs/mkdocs/issues/2601
+id = "PYSEC-2021-878"

lightman_ai-0.16.0/src/lightman_ai/ai/base/agent.py

@@ -0,0 +1,26 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import Never
+
+from lightman_ai.article.models import SelectedArticlesList
+from pydantic_ai import Agent
+from pydantic_ai.models.google import GoogleModel
+from pydantic_ai.models.openai import OpenAIModel
+
+
+class BaseAgent(ABC):
+    _model: type[OpenAIModel] | type[GoogleModel]
+    _model_name: str
+
+    def __init__(self, system_prompt: str, logger: logging.Logger | None = None) -> None:
+        model = self._model(self._model_name)
+        self.agent: Agent[Never, SelectedArticlesList] = Agent(
+            model=model, output_type=SelectedArticlesList, system_prompt=system_prompt
+        )
+        self.logger = logger or logging.getLogger("lightman")
+
+    def get_prompt_result(self, prompt: str) -> SelectedArticlesList:
+        return self._run_prompt(prompt)
+
+    @abstractmethod
+    def _run_prompt(self, prompt: str) -> SelectedArticlesList: ...

lightman_ai-0.16.0/src/lightman_ai/ai/base/exceptions.py

@@ -0,0 +1,4 @@
+from lightman_ai.core.exceptions import BaseHackermanError
+
+
+class BaseAgentError(BaseHackermanError): ...

lightman_ai-0.16.0/src/lightman_ai/ai/gemini/agent.py

@@ -0,0 +1,19 @@
+from typing import override
+
+from lightman_ai.ai.base.agent import BaseAgent
+from lightman_ai.ai.gemini.exceptions import map_gemini_exceptions
+from lightman_ai.article.models import SelectedArticlesList
+from pydantic_ai.models.google import GoogleModel
+
+
+class GeminiAgent(BaseAgent):
+    """Class that provides an interface to operate with the Gemini model."""
+
+    _model = GoogleModel
+    _model_name = "gemini-2.5-pro-preview-05-06"
+
+    @override
+    def _run_prompt(self, prompt: str) -> SelectedArticlesList:
+        with map_gemini_exceptions():
+            result = self.agent.run_sync(prompt)
+        return result.output

lightman_ai-0.16.0/src/lightman_ai/ai/gemini/exceptions.py

@@ -0,0 +1,19 @@
+from collections.abc import Generator
+from contextlib import contextmanager
+from typing import Any
+
+from lightman_ai.core.exceptions import BaseHackermanError
+
+
+class BaseGeminiError(BaseHackermanError): ...
+
+
+class GeminiError(BaseGeminiError): ...
+
+
+@contextmanager
+def map_gemini_exceptions() -> Generator[Any, Any]:
+    try:
+        yield
+    except Exception as err:
+        raise GeminiError from err

lightman_ai-0.16.0/src/lightman_ai/ai/openai/agent.py

@@ -0,0 +1,29 @@
+import time
+from typing import override
+
+from lightman_ai.ai.base.agent import BaseAgent
+from lightman_ai.ai.openai.exceptions import LimitTokensExceededError, map_openai_exceptions
+from lightman_ai.article.models import SelectedArticlesList
+from pydantic_ai.agent import AgentRunResult
+from pydantic_ai.models.openai import OpenAIModel
+
+
+class OpenAIAgent(BaseAgent):
+    """Class that provides an interface to operate with the OpenAI model."""
+
+    _model = OpenAIModel
+    _model_name = "gpt-4.1"
+
+    def _execute_agent(self, prompt: str) -> AgentRunResult[SelectedArticlesList]:
+        with map_openai_exceptions():
+            return self.agent.run_sync(prompt)
+
+    @override
+    def _run_prompt(self, prompt: str) -> SelectedArticlesList:
+        try:
+            result = self._execute_agent(prompt)
+        except LimitTokensExceededError as err:
+            self.logger.warning("waiting %s", err.wait_time)
+            time.sleep(err.wait_time)
+            result = self._execute_agent(prompt)
+        return result.output

lightman_ai-0.16.0/src/lightman_ai/ai/openai/exceptions.py

@@ -0,0 +1,85 @@
+import math
+import re
+from collections.abc import Generator
+from contextlib import contextmanager
+from typing import Any, override
+
+from lightman_ai.core.exceptions import BaseHackermanError
+from pydantic_ai.exceptions import ModelHTTPError
+
+from openai import RateLimitError
+
+
+class BaseOpenAIError(BaseHackermanError): ...
+
+
+class UnknownOpenAIError(BaseOpenAIError): ...
+
+
+class OpenAIRateLimitError(BaseOpenAIError):
+    regex: str
+
+    @classmethod
+    def get_matches(cls, message: str) -> tuple[str, ...]:
+        match = re.search(cls.regex, message)
+        return match.groups() if match else ()
+
+    @classmethod
+    def is_match(cls, message: str) -> bool:
+        return bool(re.search(cls.regex, message))
+
+
+class InputTooLargeError(OpenAIRateLimitError):
+    limit: int
+    requested: int
+
+    regex: str = (
+        r"Limit (\d+), Requested (\d+)\. The input or output tokens must be reduced in order to run successfully\."
+    )
+
+    def __init__(self, values: tuple[str, ...]) -> None:
+        self.limit = int(values[0])
+        self.requested = int(values[1])
+
+
+class LimitTokensExceededError(OpenAIRateLimitError):
+    limit: int
+    used: int
+    requested: int
+    wait_time: int
+
+    regex = r"Limit (\d+), Used (\d+), Requested (\d+)\. Please try again in (\d+\.?(\d+)?)s\."
+
+    def __init__(self, values: tuple[str, ...]) -> None:
+        self.limit = int(values[0])
+        self.used = int(values[1])
+        self.requested = int(values[2])
+        self.wait_time = math.ceil(float(values[3]))
+
+
+class QuotaExceededError(OpenAIRateLimitError):
+    regex = r"You exceeded your current quota"
+
+    @override
+    @classmethod
+    def get_matches(cls, message: str) -> tuple[str, ...]:
+        match = re.search(cls.regex, message)
+        return match.groups() if match else ()
+
+
+type TRateLimitErr = type[InputTooLargeError | LimitTokensExceededError]
+RATE_LIMIT_ERRORS: list[TRateLimitErr] = [LimitTokensExceededError, InputTooLargeError]
+
+
+@contextmanager
+def map_openai_exceptions() -> Generator[Any, Any]:
+    try:
+        yield
+    except RateLimitError as err:
+        for error in RATE_LIMIT_ERRORS:
+            if matches := error.get_matches(err.message):
+                raise error(matches) from err
+        raise UnknownOpenAIError from err
+    except ModelHTTPError as err:
+        if QuotaExceededError.is_match(err.message):
+            raise QuotaExceededError() from err

lightman_ai-0.16.0/src/lightman_ai/ai/utils.py

@@ -0,0 +1,14 @@
+from lightman_ai.ai.base.agent import BaseAgent
+from lightman_ai.ai.gemini.agent import GeminiAgent
+from lightman_ai.ai.openai.agent import OpenAIAgent
+
+AGENT_MAPPING = {"openai": OpenAIAgent, "gemini": GeminiAgent}
+
+
+def get_agent_instance_from_agent_name(agent: str) -> type[BaseAgent]:
+    if agent not in AGENT_MAPPING:
+        raise ValueError(f"Agent '{agent}' is not recognized. Available agents: {list(AGENT_MAPPING.keys())}")
+    return AGENT_MAPPING[agent]
+
+
+AGENT_CHOICES = list(AGENT_MAPPING.keys())

lightman_ai-0.16.0/src/lightman_ai/article/models.py

@@ -0,0 +1,107 @@
+from abc import ABC, abstractmethod
+from typing import override
+
+import tiktoken
+from lightman_ai.core.settings import settings
+from pydantic import BaseModel
+
+
+class BaseArticle(BaseModel, ABC):
+    """Base abstract class for all Articles."""
+
+    title: str
+    link: str
+    _encoding: tiktoken.Encoding = tiktoken.get_encoding(settings.OPENAI_ENCODING)
+
+    @override
+    def __eq__(self, value: object) -> bool:
+        if not isinstance(value, BaseArticle):
+            return False
+
+        return self.link == value.link
+
+    @override
+    def __hash__(self) -> int:
+        return hash(self.link.encode())
+
+    @property
+    @abstractmethod
+    def number_of_tokens(self) -> int: ...
+
+
+class SelectedArticle(BaseArticle):
+    why_is_relevant: str
+    relevance_score: int
+
+    @override
+    @property
+    def number_of_tokens(self) -> int:
+        """
+        Number of tokens that this Article has.
+
+        Worth taking into account that this is not the final number of tokens to be sent,
+        as pydantic-ai adds some extra characters.
+
+        It is a rough estimation of the total tokens to be sent for this Article.
+        """
+        tokens = self._encoding.encode(
+            f'"link": "{self.link}", "title": "{self.title}", "why_is_relevant": "{self.why_is_relevant}", "score_threshold": "{self.relevance_score}"'
+        )
+        return len(tokens)
+
+
+class Article(BaseArticle):
+    title: str
+    description: str
+
+    @override
+    @property
+    def number_of_tokens(self) -> int:
+        """
+        Number of tokens that this Article has.
+
+        Worth taking into account that this is not the final number of tokens to be sent,
+        as pydantic-ai adds some extra characters.
+
+        It is a rough estimation of the total tokens to be sent for this Article.
+        """
+        tokens = self._encoding.encode(
+            f'"title": "{self.title}", "description": "{self.description}", "link": "{self.link}"'
+        )
+        return len(tokens)
+
+
+class BaseArticlesList[TArticle: BaseArticle](BaseModel):
+    articles: list[TArticle]
+
+    def __len__(self) -> int:
+        return len(self.articles)
+
+    @property
+    def titles(self) -> list[str]:
+        return [new.title for new in self.articles]
+
+    @property
+    def links(self) -> list[str]:
+        return [new.link for new in self.articles]
+
+    @property
+    def total_number_of_tokens(self) -> int:
+        return sum(article.number_of_tokens for article in self.articles)
+
+
+class SelectedArticlesList(BaseArticlesList[SelectedArticle]):
+    """
+    Model that holds all the articles that were selected by the AI model.
+
+    It saves the minimum information so that they are identifiable.
+    """
+
+    def get_articles_with_score_gte_threshold(self, score_threshold: int) -> list[SelectedArticle]:
+        if not score_threshold > 0:
+            raise ValueError("score threshold must be > 0.")
+        return [article for article in self.articles if article.relevance_score >= score_threshold]
+
+
+class ArticlesList(BaseArticlesList[Article]):
+    """Model that saves articles with all their information."""

lightman_ai-0.16.0/src/lightman_ai/cli.py

@@ -0,0 +1,100 @@
+import logging
+
+import click
+from dotenv import load_dotenv
+from lightman_ai.ai.utils import AGENT_CHOICES
+from lightman_ai.constants import DEFAULT_CONFIG_FILE, DEFAULT_CONFIG_SECTION, DEFAULT_ENV_FILE
+from lightman_ai.core.config import FileConfig, FinalConfig, PromptConfig
+from lightman_ai.core.exceptions import ConfigNotFoundError, InvalidConfigError, PromptNotFoundError
+from lightman_ai.main import lightman
+
+logger = logging.getLogger("lightman")
+
+
+@click.group()
+def entry_point() -> None:
+    pass
+
+
+@entry_point.command()
+@click.option("--agent", type=click.Choice(AGENT_CHOICES), help=("Which agent to use"))
+@click.option(
+    "--prompt-file",
+    type=str,
+    default=DEFAULT_CONFIG_FILE,
+    help=(f"Location of the config file containing the prompts. Defaults to `{DEFAULT_CONFIG_FILE}`."),
+)
+@click.option("--prompt", type=str, help=("Which prompt to use"))
+@click.option(
+    "--score",
+    type=int,
+    help=("The minimum score relevance that an article needs to have to be considered relevant"),
+    default=None,
+)
+@click.option(
+    "--config-file",
+    type=str,
+    default=DEFAULT_CONFIG_FILE,
+    help=(f"The config file path. Defaults to `{DEFAULT_CONFIG_FILE}`."),
+)
+@click.option(
+    "--config",
+    type=str,
+    default=DEFAULT_CONFIG_SECTION,
+    help=(f"The config settings to use. Defaults to `{DEFAULT_CONFIG_SECTION}`."),
+)
+@click.option(
+    "--env-file",
+    type=str,
+    default=DEFAULT_ENV_FILE,
+    help=(f"Path to the environment file. Defaults to `{DEFAULT_ENV_FILE}`."),
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help=(
+        "When set, runs the script without publishing the results to the integrated services, just shows them in stdout."
+    ),
+)
+def run(
+    agent: str,
+    prompt: str,
+    prompt_file: str,
+    score: int | None,
+    config_file: str,
+    config: str,
+    env_file: str,
+    dry_run: bool,
+) -> int:
+    """
+    Entrypoint of the application.
+
+    Holds no logic. It calls the main method and returns 0 when succesful .
+    """
+    load_dotenv(env_file)
+    try:
+        prompt_config = PromptConfig.get_config_from_file(path=prompt_file)
+        config_from_file = FileConfig.get_config_from_file(config_section=config, path=config_file)
+        final_config = FinalConfig.init_from_dict(
+            data={
+                "agent": agent or config_from_file.agent,
+                "prompt": prompt or config_from_file.prompt,
+                "score_threshold": score or config_from_file.score_threshold,
+            }
+        )
+
+        prompt_text = prompt_config.get_prompt(final_config.prompt)
+    except (InvalidConfigError, PromptNotFoundError, ConfigNotFoundError) as err:
+        raise click.BadParameter(err.args[0]) from None
+
+    relevant_articles = lightman(
+        agent=final_config.agent,
+        prompt=prompt_text,
+        score_threshold=final_config.score_threshold,
+        dry_run=dry_run,
+        project_key=config_from_file.service_desk_project_key,
+        request_id_type=config_from_file.service_desk_request_id_type,
+    )
+    relevant_articles_titles = [article.title for article in relevant_articles]
+    logger.warning("Found these articles: %s", "\n".join(relevant_articles_titles))
+    return 0

lightman_ai-0.16.0/src/lightman_ai/constants.py

@@ -0,0 +1,5 @@
+DEFAULT_CONFIG_SECTION = "default"
+
+DEFAULT_CONFIG_FILE = "lightman.toml"
+
+DEFAULT_ENV_FILE = ".env"

lightman_ai-0.16.0/src/lightman_ai/core/config.py

@@ -0,0 +1,90 @@
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Self, cast
+
+import tomlkit
+from lightman_ai.core.exceptions import ConfigNotFoundError, InvalidConfigError, PromptNotFoundError
+from pydantic import BaseModel, ConfigDict, PositiveInt, ValidationError, field_validator
+from pydantic_core.core_schema import FieldValidationInfo
+
+PROMPTS_SECTION = "prompts"
+logger = logging.getLogger("lightman")
+
+
+def read_config_from_file(*, config_section: str, path: str) -> dict[str, Any]:
+    fpath = Path(path)
+    if not fpath.exists():
+        raise ConfigNotFoundError(f"`{path}` not found!")
+
+    content = fpath.read_text()
+    parsed_content = tomlkit.parse(content)
+
+    return cast("dict[str, Any]", parsed_content.get(config_section, {}))
+
+
+@dataclass
+class PromptConfig:
+    prompts: dict[str, str]
+
+    @classmethod
+    def get_config_from_file(cls, path: str) -> Self:
+        config = read_config_from_file(config_section=PROMPTS_SECTION, path=path)
+        return cls(prompts=config)
+
+    def get_prompt(self, prompt: str) -> str:
+        if prompt not in self.prompts:
+            raise PromptNotFoundError(f"prompt `{prompt}` not found in config file")
+        return self.prompts[prompt]
+
+
+class FileConfig(BaseModel):
+    prompt: str | None = None
+    agent: str | None = None
+    score_threshold: int | None = None
+    service_desk_project_key: str | None = None
+    service_desk_request_id_type: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+    @field_validator("service_desk_project_key", "service_desk_request_id_type", mode="before")
+    @classmethod
+    def _cast_service_fields(cls, v: Any, info: FieldValidationInfo) -> Any:
+        if v is None:
+            return v
+        if isinstance(v, int):
+            return str(v)
+        if isinstance(v, str):
+            if v == "":
+                return v
+            try:
+                int(v)
+            except ValueError as err:
+                raise ValueError(f"{info.field_name} must be a number!") from err
+        return v
+
+    @classmethod
+    def get_config_from_file(cls, config_section: str, path: str) -> Self:
+        try:
+            config = read_config_from_file(config_section=config_section, path=path)
+        except ConfigNotFoundError:
+            logger.warning("Config file `%s` not found! Proceeding with empty config.", path)
+            config = {}
+        return cls(**config)
+
+
+class FinalConfig(BaseModel):
+    prompt: str
+    agent: str
+    score_threshold: PositiveInt
+
+    @classmethod
+    def init_from_dict(cls, data: dict[str, Any]) -> Self:
+        try:
+            return cls(**data)
+        except ValidationError as error:
+            error_list = []
+            for err in error.errors():
+                error_list.append(f"`{err['loc'][0]}`: {err['msg']}")
+            err_msg = f"Invalid configuration provided: [{','.join(error_list)}]"
+            raise InvalidConfigError(err_msg) from error

lightman_ai-0.16.0/src/lightman_ai/core/exceptions.py

@@ -0,0 +1,13 @@
+class BaseHackermanError(Exception): ...
+
+
+class BaseConfigError(BaseHackermanError): ...
+
+
+class InvalidConfigError(BaseConfigError): ...
+
+
+class ConfigNotFoundError(BaseConfigError): ...
+
+
+class PromptNotFoundError(BaseConfigError): ...

lightman_ai-0.16.0/src/lightman_ai/core/settings.py

@@ -0,0 +1,14 @@
+from typing import Any
+
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+
+    OPENAI_ENCODING: str = "cl100k_base"
+    PARALLEL_WORKERS: int = 5
+
+
+settings = Settings()

lightman_ai-0.16.0/src/lightman_ai/integrations/service_desk/__init__.py

@@ -0,0 +1 @@
+# This file marks the service_desk directory as a Python package.

lightman_ai-0.16.0/src/lightman_ai/integrations/service_desk/constants.py

@@ -0,0 +1,7 @@
+import httpx
+from lightman_ai.integrations.service_desk.exceptions import ServiceDeskServerError
+
+# Retry configuration for ServiceDeskIntegration
+SERVICE_DESK_RETRY_ON = (httpx.TransportError, httpx.TimeoutException, ServiceDeskServerError)
+SERVICE_DESK_RETRY_ATTEMPTS = 3
+SERVICE_DESK_RETRY_TIMEOUT = 10

lightman_ai-0.16.0/src/lightman_ai/integrations/service_desk/exceptions.py

@@ -0,0 +1,76 @@
+import logging
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from typing import Any
+
+import httpx
+from lightman_ai.core.exceptions import BaseHackermanError
+
+logger = logging.getLogger("lightman")
+
+
+class BaseServiceDeskError(BaseHackermanError):
+    """Base exception for all SERVICE_DESK integration errors."""
+
+
+class ServiceDeskConnectionError(BaseServiceDeskError):
+    """Raised when there are network or connection issues with SERVICE_DESK."""
+
+
+class ServiceDeskAuthenticationError(BaseServiceDeskError):
+    """Raised when authentication with SERVICE_DESK fails."""
+
+
+class ServiceDeskPermissionError(BaseServiceDeskError):
+    """Raised when user lacks permissions to perform the requested operation."""
+
+
+class MissingIssueIDError(BaseServiceDeskError):
+    """Error for when we don't get the issue ID in the response."""
+
+    pass
+
+
+class ServiceDeskHTTPStatusError(BaseServiceDeskError):
+    def __init__(self, status_code: int, message: str) -> None:
+        self.status_code = status_code
+        self.message = message
+        super().__init__(f"SERVICE_DESK API error {status_code}: {message}")
+
+
+class ServiceDeskClientError(ServiceDeskHTTPStatusError):
+    """Exception for unmapped HTTP 4xx errors."""
+
+    pass
+
+
+class ServiceDeskServerError(ServiceDeskHTTPStatusError):
+    """Exception for HTTP 5xx errors."""
+
+    pass
+
+
+class ServiceDeskApiResponseParsingError(BaseServiceDeskError):
+    """Raised when SERVICE_DESK API returns an error response that cannot be parsed."""
+
+    def __init__(self, status_code: int, raw_response: str) -> None:
+        self.status_code = status_code
+        self.raw_response = raw_response
+        super().__init__(f"SERVICE_DESK API error {status_code}: Unable to parse error response")
+
+
+@asynccontextmanager
+async def handle_service_desk_exceptions() -> AsyncGenerator[Any, Any]:
+    try:
+        yield
+    except httpx.HTTPStatusError as e:
+        if e.response.status_code == httpx.codes.UNAUTHORIZED:
+            raise ServiceDeskAuthenticationError from e
+        elif e.response.status_code == httpx.codes.FORBIDDEN:
+            raise ServiceDeskPermissionError from e
+        elif e.response.is_client_error:
+            raise ServiceDeskClientError(e.response.status_code, e.response.text) from e
+        else:
+            raise ServiceDeskServerError(e.response.status_code, e.response.text) from e
+    except httpx.TransportError as e:
+        raise ServiceDeskConnectionError(f"Network error: {e}") from e

lightman_ai-0.16.0/src/lightman_ai/integrations/service_desk/integration.py

@@ -0,0 +1,89 @@
+import logging
+import os
+from typing import Any, Self
+
+import httpx
+import stamina
+from lightman_ai.integrations.service_desk.constants import (
+    SERVICE_DESK_RETRY_ATTEMPTS,
+    SERVICE_DESK_RETRY_ON,
+    SERVICE_DESK_RETRY_TIMEOUT,
+)
+from lightman_ai.integrations.service_desk.exceptions import (
+    MissingIssueIDError,
+    handle_service_desk_exceptions,
+)
+
+logger = logging.getLogger("lightman")
+
+
+class ServiceDeskIntegration:
+    """
+    Service Desk integration using httpx to create tickets.
+
+    Provides a simple interface to create Service Desk issues with proper
+    authentication and error handling.
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        username: str,
+        api_token: str,
+        verify_ssl: bool = True,
+        timeout: float = 30.0,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.username = username
+        self.api_token = api_token
+        self.client = httpx.AsyncClient(
+            base_url=self.base_url,
+            auth=(username, api_token),
+            headers={
+                "Content-Type": "application/json",
+                "Accept": "application/json",
+            },
+            verify=verify_ssl,
+            timeout=timeout,
+        )
+
+    @classmethod
+    def from_env(cls) -> Self:
+        required_vars = ["SERVICE_DESK_URL", "SERVICE_DESK_USER", "SERVICE_DESK_TOKEN"]
+        missing = [var for var in required_vars if not os.environ.get(var)]
+        if missing:
+            raise ValueError(f"Missing required environment variable(s): {', '.join(missing)}")
+        return cls(
+            base_url=os.environ["SERVICE_DESK_URL"],
+            username=os.environ["SERVICE_DESK_USER"],
+            api_token=os.environ["SERVICE_DESK_TOKEN"],
+        )
+
+    async def create_request_of_type(
+        self,
+        *,
+        project_key: str,
+        summary: str,
+        description: str,
+        request_id_type: str,
+    ) -> str:
+        payload: dict[str, Any] = {
+            "serviceDeskId": project_key,
+            "requestTypeId": request_id_type,
+            "requestFieldValues": {"summary": summary, "description": description},
+        }
+        for attempt in stamina.retry_context(
+            on=SERVICE_DESK_RETRY_ON,
+            attempts=SERVICE_DESK_RETRY_ATTEMPTS,
+            timeout=SERVICE_DESK_RETRY_TIMEOUT,
+        ):
+            with attempt:
+                async with handle_service_desk_exceptions():
+                    response = await self.client.post("/rest/servicedeskapi/request", json=payload)
+                    response.raise_for_status()
+                    data = response.json()
+                    issue_id = data.get("issueId")
+                    if not issue_id:
+                        raise MissingIssueIDError()
+                    logger.info("Successfully created Service Desk issue: %s", issue_id)
+        return str(issue_id)

lightman_ai-0.16.0/src/lightman_ai/main.py

@@ -0,0 +1,92 @@
+import asyncio
+import logging
+
+from lightman_ai.ai.base.agent import BaseAgent
+from lightman_ai.ai.utils import get_agent_instance_from_agent_name
+from lightman_ai.article.models import ArticlesList, SelectedArticle, SelectedArticlesList
+from lightman_ai.integrations.service_desk.integration import (
+    ServiceDeskIntegration,
+)
+from lightman_ai.sources.the_hacker_news import TheHackerNewsSource
+
+logger = logging.getLogger("lightman")
+
+
+def _get_articles() -> ArticlesList:
+    return TheHackerNewsSource().get_articles()
+
+
+def _classify_articles(articles: ArticlesList, agent: BaseAgent) -> SelectedArticlesList:
+    return agent.get_prompt_result(prompt=str(articles))
+
+
+def _create_service_desk_issues(
+    selected_articles: list[SelectedArticle],
+    service_desk_client: ServiceDeskIntegration,
+    project_key: str,
+    request_id_type: str,
+) -> None:
+    async def schedule_task(article: SelectedArticle) -> None:
+        try:
+            await service_desk_client.create_request_of_type(
+                project_key=project_key,
+                summary=article.title,
+                description=article.link,
+                request_id_type=request_id_type,
+            )
+        except Exception:
+            logger.exception("Could not create ServiceDesk issue: %s, %s", article.title, article.link)
+            raise
+
+    async def create_all() -> None:
+        tasks = []
+        for article in selected_articles:
+            tasks.append(schedule_task(article))
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        errors = [result for result in results if isinstance(result, Exception)]
+        if errors:
+            raise ExceptionGroup("Could not create all ServiceDesk issues", errors)
+
+    asyncio.run(create_all())
+
+
+def lightman(
+    agent: str,
+    prompt: str,
+    score_threshold: int,
+    project_key: str | None = None,
+    request_id_type: str | None = None,
+    dry_run: bool = False,
+) -> list[SelectedArticle]:
+    articles: ArticlesList = _get_articles()
+
+    agent_class = get_agent_instance_from_agent_name(agent)
+    agent_instance = agent_class(prompt)
+    logger.info("Selected %s.", agent_instance)
+
+    classified_articles = _classify_articles(
+        articles=articles,
+        agent=agent_instance,
+    )
+
+    selected_articles: list[SelectedArticle] = classified_articles.get_articles_with_score_gte_threshold(
+        score_threshold
+    )
+    if selected_articles:
+        logger.warning("Found these articles: %s", selected_articles)
+    else:
+        logger.warning("No articles found to be relevant. Total returned articles by AI %s", len(classified_articles))
+
+    if not dry_run:
+        if not project_key or not request_id_type:
+            raise ValueError("Missing Service Desk's project key or issue id type")
+
+        service_desk_client = ServiceDeskIntegration.from_env()
+        _create_service_desk_issues(
+            selected_articles=selected_articles,
+            service_desk_client=service_desk_client,
+            project_key=project_key,
+            request_id_type=request_id_type,
+        )
+
+    return selected_articles

lightman_ai-0.16.0/src/lightman_ai/sources/base.py

@@ -0,0 +1,8 @@
+from abc import ABC, abstractmethod
+
+from lightman_ai.article.models import ArticlesList
+
+
+class BaseSource(ABC):
+    @abstractmethod
+    def get_articles(self) -> ArticlesList: ...

lightman_ai-0.16.0/src/lightman_ai/sources/the_hacker_news.py

@@ -0,0 +1,57 @@
+from typing import override
+from xml.etree import ElementTree
+
+import httpx
+import stamina
+from httpx import Client
+from lightman_ai.article.models import Article, ArticlesList
+from lightman_ai.sources.base import BaseSource
+
+_RETRY_ON = httpx.TransportError
+_ATTEMPTS = 5
+_TIMEOUT = 5
+
+
+THN_URL = "https://feeds.feedburner.com/TheHackersNews"
+
+
+class TheHackerNewsSource(BaseSource):
+    @override
+    def get_articles(self) -> ArticlesList:
+        """Return the articles that are present in THN feed."""
+        feed = self.get_feed()
+        articles = self._xml_to_list_of_articles(feed)
+        return ArticlesList(articles=articles)
+
+    def get_feed(self) -> str:
+        """Retrieve the TheHackerNews' RSS Feed."""
+        for attempt in stamina.retry_context(
+            on=_RETRY_ON,
+            attempts=_ATTEMPTS,
+            timeout=_TIMEOUT,
+        ):
+            with Client() as http_client, attempt:
+                hacker_news_feed = http_client.get(THN_URL)
+                hacker_news_feed.raise_for_status()
+        return hacker_news_feed.text
+
+    def _xml_to_list_of_articles(self, xml: str) -> list[Article]:
+        root = ElementTree.fromstring(xml)
+        channel = root.find("channel")
+        assert channel
+        items = channel.findall("item")
+
+        parsed = []
+
+        for item in items:
+            title = item.findtext("title", default="").strip()
+            description = self._clean(item.findtext("description", default="").strip())
+            link = item.findtext("link", default="").strip()
+
+            parsed.append(Article(title=title, description=description, link=link))
+        return parsed
+
+    @staticmethod
+    def _clean(text: str) -> str:
+        """Remove non-useful characters."""
+        return text.replace("\\n", "").replace("       ", "")