novastack-core 1.0.0__tar.gz

novastack_core-1.0.0/.gitignore

@@ -0,0 +1,85 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+#IDE
+.DS_Store
+.idea
+.vscode
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Mkdocs documentation
+docs/_build/
+docs/api_reference/site/
+
+# Ruff
+.ruff_cache/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+.python-version
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/

novastack_core-1.0.0/PKG-INFO

@@ -0,0 +1,30 @@
+Metadata-Version: 2.4
+Name: novastack-core
+Version: 1.0.0
+Summary: Forge data pipelines, unleash smart AI applications.
+Project-URL: Repository, https://github.com/novastack-project/novastack
+Author-email: Leonardo Furnielis <leonardofurnielis@outlook.com>
+License: Apache-2.0
+Keywords: AI,LLM,QA,RAG,data,observability,retrieval,semantic-search
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: <3.14,>=3.11
+Requires-Dist: deprecated<2.0.0,>=1.3.1
+Requires-Dist: nltk<4.0.0,>=3.9.2
+Requires-Dist: numpy<2.0.0,>=1.26.4
+Requires-Dist: pydantic<3.0.0,>=2.12.5
+Requires-Dist: tiktoken<0.13.0,>=0.12.0
+Provides-Extra: dev
+Requires-Dist: ruff>=0.15.8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# NovaStack core
+
+Core Python package providing essential abstractions and interfaces for building LLM applications, with a focus on Retrieval-Augmented Generation (RAG).
+
+## Installation 
+
+```bash
+pip install novastack-core
+```

novastack_core-1.0.0/README.md

@@ -0,0 +1,9 @@
+# NovaStack core
+
+Core Python package providing essential abstractions and interfaces for building LLM applications, with a focus on Retrieval-Augmented Generation (RAG).
+
+## Installation 
+
+```bash
+pip install novastack-core
+```

novastack_core-1.0.0/novastack/core/bridge/pydantic.py

@@ -0,0 +1,69 @@
+import pydantic
+from pydantic import (
+    AnyUrl,
+    BaseModel,
+    BeforeValidator,
+    ConfigDict,
+    Field,
+    FilePath,
+    GetCoreSchemaHandler,
+    GetJsonSchemaHandler,
+    PlainSerializer,
+    PrivateAttr,
+    Secret,
+    SecretStr,
+    SerializationInfo,
+    SerializeAsAny,
+    SerializerFunctionWrapHandler,
+    StrictFloat,
+    StrictInt,
+    StrictStr,
+    TypeAdapter,
+    ValidationError,
+    ValidationInfo,
+    WithJsonSchema,
+    WrapSerializer,
+    computed_field,
+    create_model,
+    field_serializer,
+    field_validator,
+    model_serializer,
+    model_validator,
+)
+from pydantic.fields import FieldInfo
+from pydantic.json_schema import JsonSchemaValue
+
+__all__ = [
+    "pydantic",
+    "BaseModel",
+    "ConfigDict",
+    "GetJsonSchemaHandler",
+    "GetCoreSchemaHandler",
+    "Field",
+    "PlainSerializer",
+    "PrivateAttr",
+    "model_validator",
+    "field_validator",
+    "computed_field",
+    "create_model",
+    "StrictFloat",
+    "StrictInt",
+    "StrictStr",
+    "FieldInfo",
+    "ValidationInfo",
+    "TypeAdapter",
+    "ValidationError",
+    "WithJsonSchema",
+    "BeforeValidator",
+    "JsonSchemaValue",
+    "SerializeAsAny",
+    "WrapSerializer",
+    "field_serializer",
+    "Secret",
+    "SecretStr",
+    "model_serializer",
+    "AnyUrl",
+    "FilePath",
+    "SerializationInfo",
+    "SerializerFunctionWrapHandler",
+]

novastack_core-1.0.0/novastack/core/document/__init__.py

@@ -0,0 +1,3 @@
+from novastack.core.document.base import BaseDocument, Document, DocumentWithScore
+
+__all__ = ["BaseDocument", "Document", "DocumentWithScore"]

novastack_core-1.0.0/novastack/core/document/base.py

@@ -0,0 +1,117 @@
+import uuid
+from abc import ABC, abstractmethod
+from hashlib import sha256
+from typing import Any
+
+import numpy as np
+from novastack.core.bridge.pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    computed_field,
+    field_validator,
+)
+
+
+class BaseDocument(BaseModel, ABC):
+    """Abstract base class defining the interface for retrievable documents."""
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        validate_assignment=True,
+    )
+
+    id_: str = Field(
+        default_factory=lambda: str(uuid.uuid4()),
+        description="Unique Id of the document.",
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="A flat dictionary of metadata fields.",
+    )
+    embedding: list[float] | np.ndarray | None = Field(
+        default=None,
+        description="Embedding of the document.",
+    )
+
+    @field_validator("metadata", mode="before")
+    @classmethod
+    def _validate_metadata(cls, v) -> dict:
+        """Ensure metadata is always a dict."""
+        if v is None:
+            return {}
+        return v
+
+    @abstractmethod
+    def get_content(self) -> str:
+        """Get document content."""
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement the get_content() method"
+        )
+
+    @property
+    @abstractmethod
+    def hash(self) -> str:
+        """Get document hash."""
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement the get_content() method"
+        )
+
+
+class Document(BaseDocument):
+    """Generic interface for data document."""
+
+    text: str = Field(default="", description="Text content of the document.")
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "Document"
+
+    def get_content(self) -> str:
+        """Get the text content."""
+        return self.text
+
+    @computed_field
+    @property
+    def hash(self) -> str:
+        """Get document hash based on text content."""
+        return str(sha256(str(self.text).encode("utf-8", "surrogatepass")).hexdigest())
+
+
+class DocumentWithScore(BaseModel):
+    """Document with associated relevance score."""
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        validate_assignment=True,
+    )
+
+    document: BaseDocument
+    score: float | None = Field(
+        default=None,
+        description="Relevance score for the document.",
+    )
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "DocumentWithScore"
+
+    @property
+    def normalized_score(self) -> float:
+        """Get normalized score (0.0 if None)."""
+        return self.score if self.score is not None else 0.0
+
+    # #### pass through properties to BaseDocument ####
+    @property
+    def id_(self) -> str:
+        """Get document Id."""
+        return self.document.id_
+
+    @property
+    def metadata(self) -> dict:
+        """Get document metadata."""
+        return self.document.metadata
+
+    def get_content(self) -> str:
+        """Get document content."""
+        return self.document.get_content()

novastack_core-1.0.0/novastack/core/embeddings/__init__.py

@@ -0,0 +1,8 @@
+from novastack.core.embeddings.base import BaseEmbedding, Embedding
+from novastack.core.embeddings.enums import SimilarityMode
+
+__all__ = [
+    "BaseEmbedding",
+    "Embedding",
+    "SimilarityMode",
+]

novastack_core-1.0.0/novastack/core/embeddings/base.py

@@ -0,0 +1,101 @@
+from abc import ABC, abstractmethod
+
+import numpy as np
+from novastack.core.bridge.pydantic import BaseModel, ConfigDict, Field
+from novastack.core.document import Document
+from novastack.core.embeddings.enums import SimilarityMode
+from novastack.core.schema import TransformerComponent
+
+Embedding = list[float]
+
+
+def similarity(
+    embedding1: Embedding,
+    embedding2: Embedding,
+    mode: SimilarityMode = SimilarityMode.COSINE,
+) -> float:
+    """
+    Calculate similarity between two embeddings.
+
+    Args:
+        embedding1: First embedding vector
+        embedding2: Second embedding vector
+        mode: Similarity calculation mode (cosine, dot_product, or euclidean)
+    """
+    # Validate embeddings are not empty
+    if len(embedding1) == 0 or len(embedding2) == 0:
+        raise ValueError("Embeddings cannot be empty")
+
+    # Validate embeddings have same dimension
+    if len(embedding1) != len(embedding2):
+        raise ValueError(
+            f"Embeddings must have same dimension. "
+            f"Got {len(embedding1)} and {len(embedding2)}"
+        )
+
+    if mode == SimilarityMode.EUCLIDEAN:
+        return -float(np.linalg.norm(np.array(embedding1) - np.array(embedding2)))
+
+    elif mode == SimilarityMode.DOT_PRODUCT:
+        return float(np.dot(embedding1, embedding2))
+
+    else:
+        # Cosine similarity calculation
+        X = np.array(embedding1)
+        Y = np.array(embedding2)
+        product = np.dot(X, Y)
+        norm = np.linalg.norm(X) * np.linalg.norm(Y)
+        return float(product / norm)
+
+
+class BaseEmbedding(BaseModel, TransformerComponent, ABC):
+    """
+    Abstract base class defining the interface for embedding models.
+    """
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        validate_assignment=True,
+    )
+
+    model_name: str = Field(..., description="Name of the embedding model")
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "BaseEmbedding"
+
+    @staticmethod
+    def similarity(
+        embedding1: Embedding,
+        embedding2: Embedding,
+        mode: SimilarityMode = SimilarityMode.COSINE,
+    ):
+        """Get embedding similarity."""
+        return similarity(embedding1, embedding2, mode)
+
+    @abstractmethod
+    def embed_text(self, input: str | list[str]) -> list[Embedding]:
+        """
+        Embed one or more text strings.
+
+        Args:
+            input: Single text string or list of text strings to embed
+        """
+
+    def embed_documents(self, documents: list[Document]) -> list[Document]:
+        """
+        Embed a list of documents and assign the computed embeddings to the 'embedding' attribute.
+
+        Args:
+            documents (list[Document]): List of documents to compute embeddings.
+        """
+        texts = [document.get_content() for document in documents]
+        embeddings = self.embed_text(texts)
+
+        for document, embedding in zip(documents, embeddings):
+            document.embedding = embedding
+
+        return documents
+
+    def __call__(self, documents: list[Document]) -> list[Document]:
+        return self.embed_documents(documents)

novastack_core-1.0.0/novastack/core/embeddings/enums.py

@@ -0,0 +1,9 @@
+from enum import Enum
+
+
+class SimilarityMode(str, Enum):
+    """Modes for similarity."""
+
+    COSINE = "cosine"
+    DOT_PRODUCT = "dot_product"
+    EUCLIDEAN = "euclidean"

novastack_core-1.0.0/novastack/core/evaluation/__init__.py

@@ -0,0 +1,9 @@
+from novastack.core.evaluation.base import BaseEvaluator
+from novastack.core.evaluation.context_similarity import (
+    ContextSimilarityEvaluator,
+)
+
+__all__ = [
+    "BaseEvaluator",
+    "ContextSimilarityEvaluator",
+]

novastack_core-1.0.0/novastack/core/evaluation/base.py

@@ -0,0 +1,63 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from novastack.core.bridge.pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class BaseEvaluator(BaseModel, ABC):
+    """
+    Abstract base class defining the interface for evaluation metrics.
+
+    All evaluators should inherit from this class and implement the evaluate method.
+    """
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        validate_assignment=True,
+    )
+
+    score_threshold: float = Field(
+        default=0.8,
+        ge=0.0,
+        le=1.0,
+        description="Minimum required score for evaluation approval",
+    )
+
+    @field_validator("score_threshold")
+    @classmethod
+    def _validate_threshold(cls, v: float) -> float:
+        """Validate that threshold is within valid range."""
+        if not 0.0 <= v <= 1.0:
+            raise ValueError(f"score_threshold must be between 0.0 and 1.0, got: {v}")
+        return v
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "BaseEvaluator"
+
+    @abstractmethod
+    def evaluate(
+        self,
+        query: str | None = None,
+        generated_text: str | None = None,
+        contexts: list[str] | None = None,
+        **kwargs: Any,
+    ) -> dict:
+        """
+        Evaluate the given inputs and return evaluation results.
+
+        This method should be implemented by all concrete evaluator classes.
+        The specific parameters will vary depending on the evaluation type.
+
+        Returns:
+            dict: Dictionary containing evaluation results. Should include at minimum:
+                - score (float): The evaluation score
+                - passing (bool): Whether the evaluation passed a threshold
+                Additional keys can be included for specific evaluation details.
+
+        Raises:
+            NotImplementedError: If the method is not implemented by a subclass.
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement the evaluate() method"
+        )