novastack-core 0.1.0__tar.gz

novastack_core-0.1.0/.gitignore

@@ -0,0 +1,85 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+#IDE
+.DS_Store
+.idea
+.vscode
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Mkdocs documentation
+docs/_build/
+docs/api_reference/site/
+
+# Ruff
+.ruff_cache/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+.python-version
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/

novastack_core-0.1.0/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.4
+Name: novastack-core
+Version: 0.1.0
+Summary: Forge data pipelines, unleash smart AI applications.
+Project-URL: Repository, https://github.com/novastack-project/novastack/tree/main/novastack-core
+Author-email: Leonardo Furnielis <leonardofurnielis@outlook.com>
+License: Apache-2.0
+Keywords: AI,LLM,QA,RAG,data,observability,retrieval,semantic-search
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: <3.14,>=3.11
+Requires-Dist: deprecated<2.0.0,>=1.3.1
+Requires-Dist: httpx<1.0.0,>=0.28.1
+Requires-Dist: nltk<4.0.0,>=3.9.2
+Requires-Dist: novastack-instrumentation<0.2.0,>=0.1.0
+Requires-Dist: novastack-utils<0.2.0,>=0.1.0
+Requires-Dist: novastack-workflows<0.2.0,>=0.1.0
+Requires-Dist: numpy<2.0.0,>=1.26.4
+Requires-Dist: pydantic<3.0.0,>=2.12.5
+Requires-Dist: tiktoken<0.13.0,>=0.12.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio<2.0.0,>=1.4.0; extra == 'dev'
+Requires-Dist: pytest-mock<4.0.0,>=3.15.1; extra == 'dev'
+Requires-Dist: pytest<10.0.0,>=9.1.1; extra == 'dev'
+Requires-Dist: ruff<0.16.0,>=0.15.19; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Novastack Core
+
+Core Python package providing essential abstractions and interfaces for building LLM applications, with a focus on Retrieval-Augmented Generation (RAG).
+
+## Installation 
+
+```bash
+pip install novastack-core
+```

novastack_core-0.1.0/README.md

@@ -0,0 +1,9 @@
+# Novastack Core
+
+Core Python package providing essential abstractions and interfaces for building LLM applications, with a focus on Retrieval-Augmented Generation (RAG).
+
+## Installation 
+
+```bash
+pip install novastack-core
+```

novastack_core-0.1.0/novastack/core/__init__.py

@@ -0,0 +1,7 @@
+from novastack.core.instrumentation import (
+    set_global_handler,
+)
+
+__all__ = [
+    "set_global_handler",
+]

novastack_core-0.1.0/novastack/core/bridge/pydantic.py

@@ -0,0 +1,69 @@
+import pydantic
+from pydantic import (
+    AnyUrl,
+    BaseModel,
+    BeforeValidator,
+    Field,
+    FilePath,
+    GetCoreSchemaHandler,
+    GetJsonSchemaHandler,
+    PlainSerializer,
+    PrivateAttr,
+    Secret,
+    SecretStr,
+    SerializationInfo,
+    SerializeAsAny,
+    SerializerFunctionWrapHandler,
+    StrictFloat,
+    StrictInt,
+    StrictStr,
+    TypeAdapter,
+    ValidationError,
+    ValidationInfo,
+    WithJsonSchema,
+    WrapSerializer,
+    computed_field,
+    create_model,
+    field_serializer,
+    field_validator,
+    model_serializer,
+    model_validator,
+    validate_call,
+)
+from pydantic.fields import FieldInfo
+from pydantic.json_schema import JsonSchemaValue
+
+__all__ = [
+    "pydantic",
+    "BaseModel",
+    "GetJsonSchemaHandler",
+    "GetCoreSchemaHandler",
+    "Field",
+    "PlainSerializer",
+    "PrivateAttr",
+    "model_serializer",
+    "model_validator",
+    "validate_call",
+    "field_validator",
+    "computed_field",
+    "create_model",
+    "StrictFloat",
+    "StrictInt",
+    "StrictStr",
+    "FieldInfo",
+    "ValidationInfo",
+    "TypeAdapter",
+    "ValidationError",
+    "WithJsonSchema",
+    "BeforeValidator",
+    "JsonSchemaValue",
+    "SerializeAsAny",
+    "WrapSerializer",
+    "field_serializer",
+    "Secret",
+    "SecretStr",
+    "AnyUrl",
+    "FilePath",
+    "SerializationInfo",
+    "SerializerFunctionWrapHandler",
+]

novastack_core-0.1.0/novastack/core/components.py

@@ -0,0 +1,34 @@
+import json
+from abc import abstractmethod
+from typing import Any
+
+from novastack.core.bridge.pydantic import BaseModel
+from novastack.core.document import Document
+
+
+class BaseComponent(BaseModel):
+    """Base component object."""
+
+    @classmethod
+    def class_name(cls) -> str:
+        """Return the class name for identification purposes."""
+        return "BaseComponent"
+
+    def to_dict(self, **kwargs: Any) -> dict[str, Any]:
+        data = self.model_dump(**kwargs)
+        data["class_name"] = self.class_name()
+        return data
+
+    def to_json(self, **kwargs: Any) -> str:
+        data = self.to_dict(**kwargs)
+        return json.dumps(data)
+
+
+class TransformerComponent(BaseComponent):
+    """Abstract base class for document transformer components."""
+
+    @abstractmethod
+    def __call__(self, documents: list[Document]) -> list[Document]:
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement the __call__() method."
+        )

novastack_core-0.1.0/novastack/core/document/__init__.py

@@ -0,0 +1,3 @@
+from novastack.core.document.base import BaseDocument, Document, DocumentWithScore
+
+__all__ = ["BaseDocument", "Document", "DocumentWithScore"]

novastack_core-0.1.0/novastack/core/document/base.py

@@ -0,0 +1,106 @@
+import uuid
+from abc import ABC, abstractmethod
+from hashlib import sha256
+from typing import Any
+
+import numpy as np
+from novastack.core.bridge.pydantic import Field, computed_field, field_validator
+from novastack.core.components import BaseComponent
+
+
+class BaseDocument(BaseComponent, ABC):
+    """Abstract base class defining the interface for retrievable documents."""
+
+    model_config = {"arbitrary_types_allowed": True, "validate_assignment": True}
+
+    id_: str = Field(
+        default_factory=lambda: str(uuid.uuid4()),
+        description="Unique Id of the document.",
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="A flat dictionary of metadata fields.",
+    )
+    embedding: list[float] | np.ndarray | None = Field(
+        default=None,
+        description="Embedding of the document.",
+    )
+
+    @field_validator("metadata", mode="before")
+    @classmethod
+    def _validate_metadata(cls, v) -> dict:
+        """Ensure metadata is always a dict."""
+        if v is None:
+            return {}
+        return v
+
+    @abstractmethod
+    def get_content(self) -> str:
+        """Get document content."""
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement the get_content() method"
+        )
+
+    @property
+    @abstractmethod
+    def hash(self) -> str:
+        """Get document hash."""
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement the get_content() method"
+        )
+
+
+class Document(BaseDocument):
+    """Generic interface for data document."""
+
+    text: str = Field(default="", description="Text content of the document.")
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "Document"
+
+    def get_content(self) -> str:
+        """Get the text content."""
+        return self.text
+
+    @computed_field
+    @property
+    def hash(self) -> str:
+        """Get document hash based on text content."""
+        return str(sha256(str(self.text).encode("utf-8", "surrogatepass")).hexdigest())
+
+
+class DocumentWithScore(BaseComponent):
+    """Document with associated relevance score."""
+
+    model_config = {"arbitrary_types_allowed": True, "validate_assignment": True}
+
+    document: BaseDocument
+    score: float | None = Field(
+        default=None,
+        description="Relevance score for the document.",
+    )
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "DocumentWithScore"
+
+    @property
+    def normalized_score(self) -> float:
+        """Get normalized score (0.0 if None)."""
+        return self.score if self.score is not None else 0.0
+
+    # #### pass through properties to BaseDocument ####
+    @property
+    def id_(self) -> str:
+        """Get document Id."""
+        return self.document.id_
+
+    @property
+    def metadata(self) -> dict:
+        """Get document metadata."""
+        return self.document.metadata
+
+    def get_content(self) -> str:
+        """Get document content."""
+        return self.document.get_content()

novastack_core-0.1.0/novastack/core/embeddings/__init__.py

@@ -0,0 +1,6 @@
+from novastack.core.embeddings.base import BaseEmbedding, Embedding
+
+__all__ = [
+    "BaseEmbedding",
+    "Embedding",
+]

novastack_core-0.1.0/novastack/core/embeddings/base.py

@@ -0,0 +1,144 @@
+from abc import abstractmethod
+
+import numpy as np
+from novastack.core.bridge.pydantic import Field
+from novastack.core.components import TransformerComponent
+from novastack.core.document import Document
+from novastack.core.enums import SimilarityMode
+from novastack.core.instrumentation import DispatcherSpanMixin, get_dispatcher
+from novastack.core.instrumentation.events.embedding import (
+    EmbeddingEndEvent,
+    EmbeddingStartEvent,
+)
+from novastack.core.utils import validate_enum
+
+dispatcher = get_dispatcher(__name__)
+Embedding = list[float]
+
+
+def similarity(
+    embedding1: Embedding,
+    embedding2: Embedding,
+    mode: str = SimilarityMode.COSINE,
+) -> float:
+    """
+    Calculate similarity between two embeddings.
+
+    Args:
+        embedding1: First embedding vector
+        embedding2: Second embedding vector
+        mode: Similarity calculation mode (cosine, dot_product, or euclidean)
+    """
+    validate_enum(el=mode, el_name="mode", expected_enum=SimilarityMode)
+    # Validate embeddings are not empty
+    if len(embedding1) == 0 or len(embedding2) == 0:
+        raise ValueError("Embeddings cannot be empty")
+
+    # Validate embeddings have same dimension
+    if len(embedding1) != len(embedding2):
+        raise ValueError(
+            f"Embeddings must have same dimension. "
+            f"Got {len(embedding1)} and {len(embedding2)}"
+        )
+
+    if mode == SimilarityMode.EUCLIDEAN:
+        return -float(np.linalg.norm(np.array(embedding1) - np.array(embedding2)))
+
+    elif mode == SimilarityMode.DOT_PRODUCT:
+        return float(np.dot(embedding1, embedding2))
+
+    else:
+        # Cosine similarity calculation
+        X = np.array(embedding1)
+        Y = np.array(embedding2)
+        product = np.dot(X, Y)
+        norm = np.linalg.norm(X) * np.linalg.norm(Y)
+        return float(product / norm)
+
+
+class BaseEmbedding(TransformerComponent, DispatcherSpanMixin):
+    """
+    Abstract base class defining the interface for embedding models.
+    """
+
+    model_config = {
+        "arbitrary_types_allowed": True,
+        "validate_assignment": True,
+        "validate_default": True,
+    }
+
+    model_name: str = Field(..., description="Name of the embedding model")
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "BaseEmbedding"
+
+    @staticmethod
+    def similarity(
+        embedding1: Embedding,
+        embedding2: Embedding,
+        mode: str = SimilarityMode.COSINE,
+    ):
+        """Get embedding similarity."""
+        return similarity(embedding1, embedding2, mode)
+
+    @abstractmethod
+    def _get_text_embeddings(self, input: str | list[str]) -> list[Embedding]:
+        """Embed one or more text strings."""
+
+    @dispatcher.span
+    def get_text_embeddings(self, input: str | list[str]) -> list[Embedding]:
+        """
+        Embed one or more text strings.
+
+        Args:
+            input: Single text string or list of text strings to embed
+        """
+        config_dict = self.to_dict(exclude={"api_key"})
+        dispatcher.event(
+            EmbeddingStartEvent(
+                config_dict=config_dict,
+            )
+        )
+
+        embeddings = self._get_text_embeddings(input)
+
+        dispatcher.event(
+            EmbeddingEndEvent(
+                embeddings=embeddings,
+            )
+        )
+        return embeddings
+
+    @dispatcher.span
+    def get_document_embeddings(self, documents: list[Document]) -> list[Document]:
+        """
+        Embed a list of documents and assign the computed embeddings to the 'embedding' attribute.
+
+        Args:
+            documents (list[Document]): List of documents to compute embeddings.
+        """
+        config_dict = self.to_dict(exclude={"api_key"})
+        dispatcher.event(
+            EmbeddingStartEvent(
+                config_dict=config_dict,
+            )
+        )
+
+        texts = [document.get_content() for document in documents]
+        embeddings = self._get_text_embeddings(texts)
+
+        for document, embedding in zip(documents, embeddings):
+            document.embedding = embedding
+
+        config_dict = self.to_dict(exclude={"api_key"})
+
+        dispatcher.event(
+            EmbeddingEndEvent(
+                embeddings=embeddings,
+            )
+        )
+        return documents
+
+    def __call__(self, documents: list[Document]) -> list[Document]:
+        return self.get_document_embeddings(documents)

novastack_core-0.1.0/novastack/core/enums.py

@@ -0,0 +1,13 @@
+class SimilarityMode:
+    """
+    Describes possible similarity types.
+
+    Attributes:
+        COSINE (str): "cosine".
+        DOT_PRODUCT (str): "dot_product".
+        EUCLIDEAN (str): "euclidean".
+    """
+
+    COSINE = "cosine"
+    DOT_PRODUCT = "dot_product"
+    EUCLIDEAN = "euclidean"

novastack_core-0.1.0/novastack/core/evaluation/__init__.py

@@ -0,0 +1,9 @@
+from novastack.core.evaluation.base import BaseEvaluator
+from novastack.core.evaluation.context_similarity import (
+    ContextSimilarityEvaluator,
+)
+
+__all__ = [
+    "BaseEvaluator",
+    "ContextSimilarityEvaluator",
+]

novastack_core-0.1.0/novastack/core/evaluation/base.py

@@ -0,0 +1,56 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from novastack.core.bridge.pydantic import Field, field_validator
+from novastack.core.components import BaseComponent
+
+
+class BaseEvaluator(BaseComponent, ABC):
+    """
+    Abstract base class defining the interface for evaluation metrics.
+
+    All evaluators should inherit from this class and implement the evaluate method.
+    """
+
+    model_config = {
+        "arbitrary_types_allowed": True,
+        "validate_assignment": True,
+        "validate_default": True,
+    }
+
+    score_threshold: float = Field(
+        default=0.8,
+        ge=0.0,
+        le=1.0,
+        description="Minimum required score for evaluation approval",
+    )
+
+    @field_validator("score_threshold")
+    @classmethod
+    def _validate_threshold(cls, v: float) -> float:
+        """Validate that threshold is within valid range."""
+        if not 0.0 <= v <= 1.0:
+            raise ValueError(f"score_threshold must be between 0.0 and 1.0, got: {v}")
+        return v
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "BaseEvaluator"
+
+    @abstractmethod
+    def evaluate(
+        self,
+        query: str | None = None,
+        generated_text: str | None = None,
+        contexts: list[str] | None = None,
+        **kwargs: Any,
+    ) -> dict:
+        """
+        Evaluate the given inputs and return evaluation results.
+
+        This method should be implemented by all concrete evaluator classes.
+        The specific parameters will vary depending on the evaluation type.
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement the evaluate() method"
+        )