nlptoolkit-computationalgraph 1.0.1__tar.gz

nlptoolkit_computationalgraph-1.0.1/ComputationalGraph/ComputationalGraph.py

@@ -0,0 +1,373 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections import deque
+from typing import Optional
+import pickle
+
+from ComputationalGraph.Node.ComputationalNode import ComputationalNode
+from ComputationalGraph.Node.MultiplicationNode import MultiplicationNode
+from ComputationalGraph.Node.FunctionNode import FunctionNode
+from ComputationalGraph.Node.ConcatenatedNode import ConcatenatedNode
+from ComputationalGraph.Function.Function import Function
+from ComputationalGraph.Function.Dropout import Dropout
+from ComputationalGraph.NeuralNetworkParameter import NeuralNetworkParameter
+from Math.Tensor import Tensor
+
+
+class ComputationalGraph(ABC):
+
+    def __init__(self, parameters: NeuralNetworkParameter):
+        self.input_nodes: list[ComputationalNode] = []
+        self.output_node: Optional[ComputationalNode] = None
+        self.parameters = parameters
+        self._leaf_nodes: Optional[list[ComputationalNode]] = None
+
+    # ------------------------------------------------------------------
+    # Abstract interface
+    # ------------------------------------------------------------------
+
+    @abstractmethod
+    def train(self, train_set: list[Tensor]):
+        pass
+
+    @abstractmethod
+    def test(self, test_set: list[Tensor]):
+        pass
+
+    @abstractmethod
+    def getOutputValue(self, output_node: ComputationalNode) -> list[float]:
+        pass
+
+    # ------------------------------------------------------------------
+    # Edge construction helpers
+    # ------------------------------------------------------------------
+
+    def addEdge(self, first: ComputationalNode, second, is_biased: bool = False) -> ComputationalNode:
+        if isinstance(second, Function):
+            return second.addEdge([first], is_biased)
+        elif isinstance(second, MultiplicationNode):
+            new_node = MultiplicationNode(
+                learnable=False,
+                is_biased=is_biased,
+                is_hadamard=second.isHadamard(),
+                priority_node=first
+            )
+            first.addChild(new_node)
+            new_node.addParent(first)
+            second.addChild(new_node)
+            new_node.addParent(second)
+            return new_node
+        else:
+            raise ValueError("Illegal type for argument 'second'")
+
+    def addFunctionEdge(self, input_nodes: list[ComputationalNode], second: Function, is_biased: bool = False) -> ComputationalNode:
+        return second.addEdge(input_nodes, is_biased)
+
+    def addEdgeHadamard(self, first: ComputationalNode, second: ComputationalNode,
+                        is_biased: bool, is_hadamard: bool) -> ComputationalNode:
+        new_node = MultiplicationNode(learnable=False, is_biased=is_biased, is_hadamard=is_hadamard, priority_node=first)
+        first.addChild(new_node)
+        new_node.addParent(first)
+        second.addChild(new_node)
+        new_node.addParent(second)
+        return new_node
+
+    def addAdditionEdge(self, first: ComputationalNode, second: ComputationalNode,
+                        is_biased: bool = False) -> ComputationalNode:
+        new_node = ComputationalNode(learnable=False, is_biased=is_biased)
+        first.addChild(new_node)
+        new_node.addParent(first)
+        second.addChild(new_node)
+        new_node.addParent(second)
+        return new_node
+
+    def concatEdges(self, nodes: list[ComputationalNode], dimension: int) -> ComputationalNode:
+        new_node = ConcatenatedNode(dimension)
+        for node in nodes:
+            node.addChild(new_node)
+            new_node.addParent(node)
+            new_node.addNode(node)
+        return new_node
+
+    # ------------------------------------------------------------------
+    # Topological sort
+    # ------------------------------------------------------------------
+
+    def _sortRecursive(self, node: ComputationalNode, visited: set) -> deque:
+        queue = deque()
+        visited.add(node)
+        for i in range(node.childrenSize()):
+            child = node.getChild(i)
+            if child not in visited:
+                queue.extend(self._sortRecursive(child, visited))
+        queue.append(node)
+        return queue
+
+    def _topologicalSort(self) -> deque:
+        sorted_list = deque()
+        visited = set()
+        for node in self._leaf_nodes:
+            if node not in visited:
+                q = self._sortRecursive(node, visited)
+                sorted_list.extend(q)
+        return sorted_list
+
+    # ------------------------------------------------------------------
+    # Clear
+    # ------------------------------------------------------------------
+
+    def _clearRecursive(self, visited: set, node: ComputationalNode):
+        visited.add(node)
+        if not node.isLearnable():
+            node.setValue(None)
+        node.setBackward(None)
+        for i in range(node.childrenSize()):
+            child = node.getChild(i)
+            if child not in visited:
+                self._clearRecursive(visited, child)
+
+    def _clear(self):
+        visited = set()
+        for node in self._leaf_nodes:
+            if node not in visited:
+                self._clearRecursive(visited, node)
+
+    # ------------------------------------------------------------------
+    # Bias helpers
+    # ------------------------------------------------------------------
+
+    def _transposeAxes(self, length: int) -> tuple[int, ...]:
+        axes = list(range(length))
+        axes[-1], axes[-2] = axes[-2], axes[-1]
+        return tuple(axes)
+
+    def _getBiasedPartial(self, tensor: Tensor) -> Tensor:
+        shape = tensor.getShape()
+        end_indices = list(shape)
+        end_indices[-1] -= 1
+
+        start_indices = (0,) * len(shape)
+
+        return tensor.partial(start_indices, tuple(end_indices))
+
+    def _getBiased(self, node: ComputationalNode):
+        shape = node.getValue().getShape()
+        last_dim = shape[-1]
+        old_values = list(node.getValue().getData())
+        values = []
+        for i, v in enumerate(old_values):
+            values.append(v)
+            if (i + 1) % last_dim == 0:
+                values.append(1.0)
+        new_shape = list(shape)
+        new_shape[-1] += 1
+        node.setValue(Tensor(values, tuple(new_shape)))
+
+    # ------------------------------------------------------------------
+    # Derivative calculation
+    # ------------------------------------------------------------------
+
+    def _calculateDerivative(self, node: ComputationalNode, child: ComputationalNode) -> Optional[Tensor]:
+        if child.parentsSize() == 0:
+            return None
+
+        backward = self._getBiasedPartial(child.getBackward()) if child.isBiased() else child.getBackward()
+
+        if isinstance(child, FunctionNode):
+            function = child.getFunction()
+            child_value = self._getBiasedPartial(child.getValue()) if child.isBiased() else child.getValue()
+            return function.derivative(child_value, backward)
+
+        if isinstance(child, ConcatenatedNode):
+            index = child.getIndex(node)
+            block_size = backward.getShape()[child.getDimension()] // child.parentsSize()
+            dimensions = block_size
+            shape = list(backward.getShape())
+            for i in range(len(shape)):
+                dim = child.getDimension()
+                if dim > i:
+                    pass
+                elif dim < i:
+                    dimensions *= shape[i]
+                else:
+                    shape[i] = block_size
+            child_values = list(backward.getData())
+            new_values = []
+            i = index * dimensions
+            while i < len(child_values):
+                for k in range(dimensions):
+                    new_values.append(child_values[i + k])
+                i += child.parentsSize() * dimensions
+            return Tensor(new_values, tuple(shape))
+
+        if isinstance(child, MultiplicationNode):
+            left = child.getParent(0)
+            right = child.getParent(1)
+            if left is node:
+                right_value = right.getValue()
+                if child.isHadamard():
+                    return right_value.hadamardProduct(backward)
+                return backward.multiply(right_value.transpose(self._transposeAxes(len(right_value.getShape()))))
+            left_value = left.getValue()
+            if child.isHadamard():
+                return left_value.hadamardProduct(backward)
+            if left_value is not None and backward is not None:
+                return left_value.transpose(self._transposeAxes(len(left_value.getShape()))).multiply(backward)
+            raise ValueError("Backward and/or left child values are None")
+
+        # Plain addition node
+        return backward
+
+    # ------------------------------------------------------------------
+    # Backpropagation
+    # ------------------------------------------------------------------
+
+    def backpropagation(self):
+        sorted_nodes = self._topologicalSort()
+        if not sorted_nodes:
+            return
+
+        output_node = sorted_nodes.popleft()
+        batch_size = self.parameters.getBatchSize()
+        backward = [1.0 / batch_size] * len(list(output_node.getValue().getData()))
+        output_node.setBackward(Tensor(backward, output_node.getValue().getShape()))
+
+        while sorted_nodes:
+            node = sorted_nodes.popleft()
+            if node.childrenSize() > 0:
+                for i in range(node.childrenSize()):
+                    child = node.getChild(i)
+                    derivative = self._calculateDerivative(node, child)
+                    if derivative is not None:
+                        if node.getBackward() is None:
+                            node.setBackward(derivative)
+                        else:
+                            node.setBackward(node.getBackward().add(derivative))
+
+        self.parameters.getOptimizer().updateValues(self._leaf_nodes)
+        self._clear()
+
+    # ------------------------------------------------------------------
+    # Forward pass
+    # ------------------------------------------------------------------
+
+    def _findOutputNode(self, node: ComputationalNode) -> ComputationalNode:
+        if node.childrenSize() == 0:
+            return node
+        return self._findOutputNode(node.getChild(0))
+
+    def _findLeafNodes(self) -> list[ComputationalNode]:
+        leaf_nodes = []
+        output_node = self._findOutputNode(self.input_nodes[0])
+        queue = [output_node]
+        visited = set()
+        while queue:
+            current = queue.pop(0)
+            if current.parentsSize() == 0:
+                leaf_nodes.append(current)
+            for i in range(current.parentsSize()):
+                parent = current.getParent(i)
+                if parent not in visited:
+                    visited.add(parent)
+                    queue.append(parent)
+        return leaf_nodes
+
+    def predict(self) -> list[float]:
+        class_labels = self._forwardCalculation(enable_dropout=False)
+        self._clear()
+        return class_labels
+
+    def forwardCalculation(self) -> list[float]:
+        if self._leaf_nodes is None:
+            self._leaf_nodes = self._findLeafNodes()
+        return self._forwardCalculation(enable_dropout=True)
+
+    def _forwardCalculation(self, enable_dropout: bool) -> list[float]:
+        sorted_nodes = self._topologicalSort()
+        if not sorted_nodes:
+            return []
+
+        concatenated_node_map: dict[ConcatenatedNode, list[Optional[ComputationalNode]]] = {}
+        counter_map: dict[ComputationalNode, int] = {}
+
+        while len(sorted_nodes) > 1:
+            current_node = sorted_nodes.pop()  # removeLast
+
+            if current_node.isBiased():
+                self._getBiased(current_node)
+
+            if current_node.getValue() is None:
+                raise ValueError("Current node's value is None")
+
+            if current_node.childrenSize() > 0:
+                if current_node is self.output_node and not enable_dropout:
+                    break
+
+                for t in range(current_node.childrenSize()):
+                    child = current_node.getChild(t)
+
+                    if child.getValue() is None:
+                        if isinstance(child, FunctionNode):
+                            function = child.getFunction()
+                            current_value = current_node.getValue()
+                            if isinstance(function, Dropout):
+                                if enable_dropout:
+                                    child.setValue(function.calculate(current_value))
+                                else:
+                                    child.setValue(Tensor(current_value.getData(), current_value.getShape()))
+                            else:
+                                child.setValue(function.calculate(current_value))
+
+
+                        elif isinstance(child, ConcatenatedNode):
+                            if child not in concatenated_node_map:
+                                concatenated_node_map[child] = [None] * child.parentsSize()
+                            concatenated_node_map[child][child.getIndex(current_node)] = current_node
+                            counter_map[child] = counter_map.get(child, 0) + 1
+                            if child.parentsSize() == counter_map[child]:
+                                nodes_arr = concatenated_node_map[child]
+                                if any(n is None for n in nodes_arr):
+                                    raise ValueError("Concatenation nodes missing parents")
+                                nodes_arr = [n for n in nodes_arr if n is not None]
+                                child.setValue(nodes_arr[0].getValue())
+                                for i in range(1, len(nodes_arr)):
+                                    child.setValue(
+                                        child.getValue().concat(nodes_arr[i].getValue(), child.getDimension())
+                                    )
+                        else:
+                            child.setValue(current_node.getValue())
+
+                    else:
+                        if isinstance(child, MultiplicationNode):
+                            child_value = child.getValue()
+                            current_value = current_node.getValue()
+                            if child.isHadamard():
+                                child.setValue(child_value.hadamardProduct(current_value))
+                            elif child.getPriorityNode() is not current_node:
+                                child.setValue(child_value.multiply(current_value))
+                            else:
+                                child.setValue(current_value.multiply(child_value))
+                        else:
+                            child.setValue(child.getValue().add(current_node.getValue()))
+
+        return self.getOutputValue(self.output_node)
+
+    # ------------------------------------------------------------------
+    # Serialization
+    # ------------------------------------------------------------------
+
+    def save(self, file_name: str):
+        try:
+            with open(file_name, 'wb') as f:
+                pickle.dump(self, f)
+        except IOError:
+            print("Object could not be saved.")
+
+    @staticmethod
+    def loadModel(file_name: str) -> Optional[ComputationalGraph]:
+        try:
+            with open(file_name, 'rb') as f:
+                return pickle.load(f)
+        except (IOError, pickle.UnpicklingError):
+            return None

nlptoolkit_computationalgraph-1.0.1/ComputationalGraph/Function/CrossEntropyLoss.py

@@ -0,0 +1,30 @@
+from typing import List
+
+from ComputationalGraph.Node.ComputationalNode import ComputationalNode
+from ComputationalGraph.Node.FunctionNode import FunctionNode
+from ComputationalGraph.Node.MultiplicationNode import MultiplicationNode
+from ComputationalGraph.Function.Logarithm import Logarithm
+
+
+class CrossEntropyLoss(Logarithm):
+    """
+    Cross Entropy Loss function representation for the computational graph.
+    Inherits the calculate and derivative math from Logarithm, but overrides the graph wiring.
+    """
+
+    def addEdge(self, input_nodes: List[ComputationalNode], is_biased: bool) -> ComputationalNode:
+        """
+        Constructs the subgraph for the cross-entropy loss operation.
+        """
+        # Create a function node that uses the Logarithm math (since 'self' is a Logarithm)
+        logy = FunctionNode(function=self, is_biased=False)
+        input_nodes[0].add(logy)
+
+        # Create a Hadamard multiplication node for y * log(y_hat)
+        ylogy = MultiplicationNode(learnable=False, is_biased=is_biased, is_hadamard=True)
+        input_nodes[1].add(ylogy)
+
+        # Connect logy to the multiplication node
+        logy.add(ylogy)
+
+        return ylogy

nlptoolkit_computationalgraph-1.0.1/ComputationalGraph/Function/DELU.py

@@ -0,0 +1,71 @@
+import math
+from typing import List
+
+from ComputationalGraph.Function.Function import Function
+from ComputationalGraph.Node.ComputationalNode import ComputationalNode
+from ComputationalGraph.Node.FunctionNode import FunctionNode
+from Math.Tensor import Tensor
+
+
+class DELU(Function):
+    """
+    DELU activation function representation for the computational graph.
+    """
+
+    def __init__(self, a: float = 1.0, b: float = 2.0, xc: float = 1.25643):
+        """
+        Initializes the DELU activation function.
+        Consolidates both Java constructors using default keyword arguments.
+
+        :param a: Parameter 'a' for the DELU function.
+        :param b: Parameter 'b' for the DELU function.
+        :param xc: Threshold parameter 'xc' for the DELU function.
+        """
+        self.a: float = a
+        self.b: float = b
+        self.xc: float = xc
+
+    def calculate(self, value: Tensor) -> Tensor:
+        """
+        Computes the DELU activation for the given value tensor.
+
+        :param value: The tensor whose values are to be computed.
+        :return: A new Tensor containing DELU(x).
+        """
+        new_data = []
+
+        for val in value.getData():
+            if val > self.xc:
+                new_data.append(val)
+            else:
+                new_data.append((math.exp(self.a * val) - 1) / self.b)
+
+        return Tensor(new_data, value.getShape())
+
+    def derivative(self, value: Tensor, backward: Tensor) -> Tensor:
+        """
+        Computes the derivative of the DELU activation function.
+
+        :param value: The output tensor of the DELU(x) operation.
+        :param backward: The backward gradient tensor.
+        :return: A new Tensor containing the gradient values.
+        """
+        new_data = []
+
+        for val, back_val in zip(value.getData(), backward.getData()):
+            if val > self.xc:
+                new_data.append(back_val)
+            else:
+                derivative_val = (val * self.b + 1) * (self.a / self.b)
+                new_data.append(back_val * derivative_val)
+
+        return Tensor(new_data, value.getShape())
+
+
+    def addEdge(self, input_nodes: List[ComputationalNode], is_biased: bool) -> ComputationalNode:
+        """
+        Adds a DELU node to the computational graph.
+        """
+        new_node = FunctionNode(function=self, is_biased=is_biased)
+        input_nodes[0].add(new_node)
+        return new_node

nlptoolkit_computationalgraph-1.0.1/ComputationalGraph/Function/Dropout.py

@@ -0,0 +1,60 @@
+import random
+from typing import List, Optional
+
+from ComputationalGraph.Function.Function import Function
+from ComputationalGraph.Node.ComputationalNode import ComputationalNode
+from ComputationalGraph.Node.FunctionNode import FunctionNode
+from Math.Tensor import Tensor
+
+
+class Dropout(Function):
+    """
+    Implements the Dropout function for regularization in the computational graph.
+    """
+
+    def __init__(self, p: float, rng: Optional[random.Random] = None):
+        """
+        Initializes the Dropout function.
+
+        :param p: The probability of dropping an element.
+        :param rng: A random.Random instance (equivalent to Java's Random).
+        """
+        self.p: float = p
+        # Use the provided RNG or the global random module
+        self.random: random.Random = rng if rng is not None else random.Random()
+        self.mask: List[float] = []
+
+    def calculate(self, value: Tensor) -> Tensor:
+        """
+        Computes the dropout values and stores the mask for the backward pass.
+        """
+        self.mask.clear()
+        multiplier = 1.0 / (1.0 - self.p)
+        new_data = []
+
+        old_values = value.getData()
+        for old_value in old_values:
+            r = self.random.random()
+            if r > self.p:
+                self.mask.append(multiplier)
+                new_data.append(old_value * multiplier)
+            else:
+                self.mask.append(0.0)
+                new_data.append(0.0)
+
+        return Tensor(new_data, value.getShape())
+
+    def derivative(self, value: Tensor, backward: Tensor) -> Tensor:
+        """
+        Calculates the derivative of the dropout using the stored mask.
+        """
+        mask_tensor = Tensor(self.mask, value.getShape())
+        return backward.hadamardProduct(mask_tensor)
+
+    def addEdge(self, input_nodes: List[ComputationalNode], is_biased: bool) -> ComputationalNode:
+        """
+        Adds a Dropout node to the computational graph.
+        """
+        new_node = FunctionNode(function=self, is_biased=is_biased)
+        input_nodes[0].add(new_node)
+        return new_node

nlptoolkit_computationalgraph-1.0.1/ComputationalGraph/Function/ELU.py

@@ -0,0 +1,65 @@
+import math
+from typing import List
+
+from ComputationalGraph.Function.Function import Function
+from ComputationalGraph.Node.ComputationalNode import ComputationalNode
+from ComputationalGraph.Node.FunctionNode import FunctionNode
+from Math.Tensor import Tensor
+
+class ELU(Function):
+    """
+    Exponential Linear Unit (ELU) activation function representation for the computational graph.
+    """
+
+    def __init__(self, a: float = 1.0):
+        """
+        Initializes the ELU activation function.
+        Consolidates the Java constructors using a default keyword argument.
+
+        :param a: The alpha parameter for the ELU function (defaults to 1.0).
+        """
+        self.a: float = a
+
+    def calculate(self, value: Tensor) -> Tensor:
+        """
+        Computes the ELU activation for the given tensor.
+
+        :param value: The tensor whose values are to be computed.
+        :return: A new Tensor containing ELU(x).
+        """
+        new_data = []
+        for old_value in value.getData():
+            if old_value < 0:
+                new_data.append(self.a * (math.exp(old_value) - 1.0))
+            else:
+                new_data.append(old_value)
+
+        return Tensor(new_data, value.getShape())
+
+    def derivative(self, value: Tensor, backward: Tensor) -> Tensor:
+        """
+        Computes the derivative of the ELU activation function.
+
+        :param value: The output tensor of the ELU(x) operation.
+        :param backward: The backward gradient tensor.
+        :return: A new Tensor containing the gradient values.
+        """
+        new_data = []
+
+        for old_value, backward_value in zip(value.getData(), backward.getData()):
+            # Because old_value is the output (y) of the ELU function,
+            # if x < 0, then y < 0, and dy/dx = y + a
+            if old_value < 0:
+                new_data.append((old_value + self.a) * backward_value)
+            else:
+                new_data.append(backward_value)
+
+        return Tensor(new_data, value.getShape())
+
+    def addEdge(self, input_nodes: List[ComputationalNode], is_biased: bool) -> ComputationalNode:
+        """
+        Adds an ELU node to the computational graph.
+        """
+        new_node = FunctionNode(function=self, is_biased=is_biased)
+        input_nodes[0].add(new_node)
+        return new_node

nlptoolkit_computationalgraph-1.0.1/ComputationalGraph/Function/Function.py

@@ -0,0 +1,44 @@
+from abc import ABC, abstractmethod
+from typing import List, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ComputationalGraph.Node.ComputationalNode import ComputationalNode
+    from Math.Tensor import Tensor
+
+
+class Function(ABC):
+    """
+    An interface representing a mathematical function in the computational graph.
+    """
+
+    @abstractmethod
+    def calculate(self, matrix: 'Tensor') -> 'Tensor':
+        """
+        Calculates the forward pass of the function.
+
+        :param matrix: The input tensor.
+        :return: The resulting tensor after applying the function.
+        """
+        pass
+
+    @abstractmethod
+    def derivative(self, value: 'Tensor', backward: 'Tensor') -> 'Tensor':
+        """
+        Calculates the derivative (backward pass) of the function.
+
+        :param value: The current tensor value.
+        :param backward: The backward gradient tensor.
+        :return: The resulting gradient tensor.
+        """
+        pass
+
+    @abstractmethod
+    def addEdge(self, input_nodes: List['ComputationalNode'], is_biased: bool) -> 'ComputationalNode':
+        """
+        Adds an edge to the computational graph.
+
+        :param input_nodes: A list of input computational nodes.
+        :param is_biased: Indicates whether the connection is biased.
+        :return: The resulting computational node.
+        """
+        pass

nlptoolkit_computationalgraph-1.0.1/ComputationalGraph/Function/Logarithm.py

@@ -0,0 +1,43 @@
+import math
+from typing import List
+
+from ComputationalGraph.Function.Function import Function
+from ComputationalGraph.Node.ComputationalNode import ComputationalNode
+from ComputationalGraph.Node.FunctionNode import FunctionNode
+from Math.Tensor import Tensor
+
+
+class Logarithm(Function):
+    """
+    Applies the natural logarithm function to a tensor.
+    """
+
+    def calculate(self, value: Tensor) -> Tensor:
+        """
+        Applies the natural logarithm to each element of the input tensor.
+        """
+
+        new_data = [math.log(x) for x in value.getData()]
+
+        return Tensor(new_data, value.getShape())
+
+    def derivative(self, value: Tensor, backward: Tensor) -> Tensor:
+        """
+        Computes the derivative of the Logarithm function.
+        Note: 'value' here is the output of the log function, so x = exp(value).
+        """
+        new_data = []
+
+        for val, back_val in zip(value.getData(), backward.getData()):
+            derivative_val = 1.0 / math.exp(val)
+            new_data.append(derivative_val * back_val)
+
+        return Tensor(new_data, value.getShape())
+
+    def addEdge(self, input_nodes: List[ComputationalNode], is_biased: bool) -> ComputationalNode:
+        """
+        Adds a Logarithm node to the graph.
+        """
+        new_node = FunctionNode(function=self, is_biased=is_biased)
+        input_nodes[0].add(new_node)
+        return new_node