retab 0.0.42__py3-none-any.whl → 0.0.43__py3-none-any.whl

retab/types/deprecated_evals.py

@@ -0,0 +1,195 @@
+import datetime
+from typing import Any, List, Literal, Optional
+
+import nanoid  # type: ignore
+from pydantic import BaseModel, Field, computed_field
+
+from ..utils.json_schema import generate_schema_data_id, generate_schema_id
+from .ai_models import Amount
+from .inference_settings import InferenceSettings
+from .mime import MIMEData
+
+# Define the type alias for MetricType
+MetricType = Literal["levenshtein", "jaccard", "hamming"]
+
+
+# Define the structure for an individual item metric
+class ItemMetric(BaseModel):
+    id: str = Field(description="The ID of the item being measured")
+    name: str = Field(description="The name of the item being measured")
+    similarity: float = Field(description="The similarity score between 0 and 1")
+    similarities: dict[str, Any] = Field(description="The similarity scores for each item in the list")
+    flat_similarities: dict[str, Optional[float]] = Field(description="The similarity scores for each item in the list in dot notation format")
+    aligned_similarity: float = Field(description="The similarity score between 0 and 1, after alignment")
+    aligned_similarities: dict[str, Any] = Field(description="The similarity scores for each item in the list, after alignment")
+    aligned_flat_similarities: dict[str, Optional[float]] = Field(description="The similarity scores for each item in the list in dot notation format, after alignment")
+
+
+# Define the main MetricResult model
+class MetricResult(BaseModel):
+    item_metrics: List[ItemMetric] = Field(description="List of similarity metrics for individual items")
+    mean_similarity: float = Field(description="The average similarity score across all items")
+    aligned_mean_similarity: float = Field(description="The average similarity score across all items, after alignment")
+    metric_type: MetricType = Field(description="The type of similarity metric used for comparison")
+
+
+class DistancesResult(BaseModel):
+    distances: dict[str, Any] = Field(description="List of distances for individual items")
+    mean_distance: float = Field(description="The average distance across all items")
+    metric_type: MetricType = Field(description="The type of distance metric used for comparison")
+
+
+class PredictionMetadata(BaseModel):
+    extraction_id: Optional[str] = Field(default=None, description="The ID of the extraction")
+    likelihoods: Optional[dict[str, Any]] = Field(default=None, description="The likelihoods of the extraction")
+    field_locations: Optional[dict[str, Any]] = Field(default=None, description="The field locations of the extraction")
+    agentic_field_locations: Optional[dict[str, Any]] = Field(default=None, description="The field locations of the extraction extracted by an llm")
+    consensus_details: Optional[list[dict[str, Any]]] = Field(default=None, description="The consensus details of the extraction")
+    api_cost: Optional[Amount] = Field(default=None, description="The cost of the API call for this document (if any -- ground truth for example)")
+
+
+class PredictionData(BaseModel):
+    prediction: dict[str, Any] = Field(default={}, description="The result of the extraction or manual annotation")
+    metadata: Optional[PredictionMetadata] = Field(default=None, description="The metadata of the prediction")
+
+
+class Iteration(BaseModel):
+    id: str = Field(default_factory=lambda: "eval_iter_" + nanoid.generate())
+    inference_settings: InferenceSettings
+    json_schema: dict[str, Any]
+    predictions: list[PredictionData] = Field(default_factory=list, description="The predictions of the iteration for all the documents")
+    metric_results: Optional[MetricResult] = Field(default=None, description="The metric results of the iteration")
+
+    @computed_field  # type: ignore
+    @property
+    def schema_data_id(self) -> str:
+        """Returns the SHA1 hash of the schema data, ignoring all prompt/description/default fields.
+
+        Returns:
+            str: A SHA1 hash string representing the schema data version.
+        """
+        return generate_schema_data_id(self.json_schema)
+
+    # This is a computed field, it is exposed when serializing the object
+    @computed_field  # type: ignore
+    @property
+    def schema_id(self) -> str:
+        """Returns the SHA1 hash of the complete schema.
+
+        Returns:
+            str: A SHA1 hash string representing the complete schema version.
+        """
+        return generate_schema_id(self.json_schema)
+
+
+class AnnotatedDocument(BaseModel):
+    mime_data: MIMEData = Field(
+        description="The mime data of the document. Can also be a BaseMIMEData, which is why we have this id field (to be able to identify the file, but id is equal to mime_data.id)"
+    )
+    annotation: dict[str, Any] = Field(default={}, description="The ground truth of the document")
+
+
+class DocumentItem(AnnotatedDocument):
+    annotation_metadata: Optional[PredictionMetadata] = Field(default=None, description="The metadata of the annotation when the annotation is a prediction")
+
+
+class EvaluationDocument(DocumentItem):
+    id: str = Field(description="The ID of the document. Equal to mime_data.id but robust to the case where mime_data is a BaseMIMEData")
+
+
+class CreateIterationRequest(BaseModel):
+    """
+    Request model for performing a new iteration with custom inference settings and optional JSON schema.
+    """
+
+    inference_settings: InferenceSettings
+    json_schema: Optional[dict[str, Any]] = None
+
+
+class UpdateEvaluationDocumentRequest(BaseModel):
+    annotation: Optional[dict[str, Any]] = Field(default=None, description="The ground truth of the document")
+    annotation_metadata: Optional[PredictionMetadata] = Field(default=None, description="The metadata of the annotation when the annotation is a prediction")
+
+
+class UpdateEvaluationRequest(BaseModel):
+    name: Optional[str] = Field(default=None, description="The name of the document")
+    documents: Optional[list[EvaluationDocument]] = Field(default=None, description="The documents of the evaluation")
+    iterations: Optional[list[Iteration]] = Field(default=None, description="The iterations of the evaluation")
+    json_schema: Optional[dict[str, Any]] = Field(default=None, description="The json schema of the evaluation")
+
+    project_id: Optional[str] = Field(default=None, description="The ID of the project")
+    default_inference_settings: Optional[InferenceSettings] = Field(default=None, description="The default inference properties for the evaluation (mostly used in the frontend)")
+
+    @computed_field  # type: ignore
+    @property
+    def schema_data_id(self) -> Optional[str]:
+        """Returns the SHA1 hash of the schema data, ignoring all prompt/description/default fields.
+
+        Returns:
+            str: A SHA1 hash string representing the schema data version.
+        """
+        if self.json_schema is None:
+            return None
+
+        return generate_schema_data_id(self.json_schema)
+
+    # This is a computed field, it is exposed when serializing the object
+    @computed_field  # type: ignore
+    @property
+    def schema_id(self) -> Optional[str]:
+        """Returns the SHA1 hash of the complete schema.
+
+        Returns:
+            str: A SHA1 hash string representing the complete schema version.
+        """
+        if self.json_schema is None:
+            return None
+        return generate_schema_id(self.json_schema)
+
+
+class Evaluation(BaseModel):
+    id: str = Field(default_factory=lambda: "eval_" + nanoid.generate())
+    updated_at: datetime.datetime = Field(default_factory=lambda: datetime.datetime.now(tz=datetime.timezone.utc))
+
+    name: str
+    old_documents: list[EvaluationDocument] | None = None
+    documents: list[EvaluationDocument]
+    iterations: list[Iteration]
+    json_schema: dict[str, Any]
+
+    project_id: str = Field(description="The ID of the project", default="default_spreadsheets")
+    default_inference_settings: Optional[InferenceSettings] = Field(default=None, description="The default inference properties for the evaluation (mostly used in the frontend)")
+
+    # @field_validator('iterations')
+    # def validate_iterations_content_length(cls: Any, v: list[Iteration], values: Any) -> list[Iteration]:
+    #     if 'ground_truth' in values:
+    #         ground_truth_length = len(values['ground_truth'])
+    #         for iteration in v:
+    #             if len(iteration.content) != ground_truth_length:
+    #                 raise ValueError(f"Iteration content length must match ground_truth length ({ground_truth_length})")
+    #     return v
+
+    @computed_field  # type: ignore
+    @property
+    def schema_data_id(self) -> str:
+        """Returns the SHA1 hash of the schema data, ignoring all prompt/description/default fields.
+
+        Returns:
+            str: A SHA1 hash string representing the schema data version.
+        """
+        return generate_schema_data_id(self.json_schema)
+
+    # This is a computed field, it is exposed when serializing the object
+    @computed_field  # type: ignore
+    @property
+    def schema_id(self) -> str:
+        """Returns the SHA1 hash of the complete schema.
+
+        Returns:
+            str: A SHA1 hash string representing the complete schema version.
+        """
+        return generate_schema_id(self.json_schema)
+
+
+class AddIterationFromJsonlRequest(BaseModel):
+    jsonl_gcs_path: str

retab/types/evaluations/__init__.py

@@ -1,6 +1,7 @@
-from .model import Evaluation, CreateEvaluation, PatchEvaluationRequest, ListEvaluationParams
+from .model import Evaluation, BaseEvaluation, CreateEvaluationRequest, PatchEvaluationRequest, ListEvaluationParams
 from .documents import AnnotatedDocument, DocumentItem, EvaluationDocument, CreateEvaluationDocumentRequest, PatchEvaluationDocumentRequest
 from .iterations import (
+    BaseIteration,
     Iteration,
     CreateIterationRequest,
     PatchIterationRequest,
@@ -13,7 +14,8 @@ from .iterations import (
 
 __all__ = [
     "Evaluation",
-    "CreateEvaluation",
+    "BaseEvaluation",
+    "CreateEvaluationRequest",
     "PatchEvaluationRequest",
     "ListEvaluationParams",
     "AnnotatedDocument",
@@ -21,6 +23,7 @@ __all__ = [
     "EvaluationDocument",
     "CreateEvaluationDocumentRequest",
     "PatchEvaluationDocumentRequest",
+    "BaseIteration",
     "Iteration",
     "CreateIterationRequest",
     "PatchIterationRequest",

retab/types/evaluations/iterations.py

@@ -1,59 +1,24 @@
-import copy
 import datetime
-import json
 from typing import Any, Optional, Self
 
 import nanoid  # type: ignore
-from pydantic import BaseModel, Field, computed_field, model_validator
+from pydantic import BaseModel, Field, model_validator
 
-from ...utils.json_schema import clean_schema
-from ...utils.mime import generate_blake2b_hash_from_string
 from ..inference_settings import InferenceSettings
-from ..metrics import MetricResult
 from ..predictions import PredictionData
 
 
-class Iteration(BaseModel):
+class BaseIteration(BaseModel):
     id: str = Field(default_factory=lambda: "eval_iter_" + nanoid.generate())
+    inference_settings: InferenceSettings
+    json_schema: dict[str, Any]
     updated_at: datetime.datetime = Field(
         default_factory=lambda: datetime.datetime.now(tz=datetime.timezone.utc),
         description="The last update date of inference settings or json schema",
     )
-    inference_settings: InferenceSettings
-    json_schema: dict[str, Any]
-    predictions: dict[str, PredictionData] = Field(default_factory=dict, description="The predictions of the iteration for all the documents")
-    metric_results: Optional[MetricResult] = Field(default=None, description="The metric results of the iteration")
-
-    @computed_field  # type: ignore
-    @property
-    def schema_data_id(self) -> str:
-        """Returns the SHA1 hash of the schema data, ignoring all prompt/description/default fields.
-
-        Returns:
-            str: A SHA1 hash string representing the schema data version.
-        """
-        return "sch_data_id_" + generate_blake2b_hash_from_string(
-            json.dumps(
-                clean_schema(
-                    copy.deepcopy(self.json_schema),
-                    remove_custom_fields=True,
-                    fields_to_remove=["description", "default", "title", "required", "examples", "deprecated", "readOnly", "writeOnly"],
-                ),
-                sort_keys=True,
-            ).strip()
-        )
-
-    # This is a computed field, it is exposed when serializing the object
-    @computed_field  # type: ignore
-    @property
-    def schema_id(self) -> str:
-        """Returns the SHA1 hash of the complete schema.
-
-        Returns:
-            str: A SHA1 hash string representing the complete schema version.
-        """
-        return "sch_id_" + generate_blake2b_hash_from_string(json.dumps(self.json_schema, sort_keys=True).strip())
 
+class Iteration(BaseIteration):
+    predictions: dict[str, PredictionData] = Field(default_factory=dict, description="The predictions of the iteration for all the documents")
 
 class CreateIterationRequest(BaseModel):
     """
@@ -71,13 +36,14 @@ class CreateIterationRequest(BaseModel):
     @model_validator(mode="after")
     def validate_one_of_from_iteration_id_or_json_schema(self) -> Self:
         if (self.from_iteration_id is None) ^ (self.json_schema is None):
-            raise ValueError("Exactly one of from_iteration_id or json_schema must be provided")
-        return self
+            return self
+        raise ValueError("Exactly one of from_iteration_id or json_schema must be provided")
 
 
 class PatchIterationRequest(BaseModel):
     inference_settings: Optional[InferenceSettings] = Field(default=None, description="The new inference settings of the iteration")
     json_schema: Optional[dict[str, Any]] = Field(default=None, description="The new json schema of the iteration")
+    version: Optional[int] = Field(default=None, description="Current version for optimistic locking")
 
 
 class ProcessIterationRequest(BaseModel):

retab/types/evaluations/model.py

@@ -1,31 +1,27 @@
 import datetime
-import json
 from typing import Any, Optional
 
 import nanoid  # type: ignore
 from pydantic import BaseModel, Field, computed_field
 
-from ...utils.json_schema import compute_schema_data_id
-from ...utils.mime import generate_blake2b_hash_from_string
+from ...utils.json_schema import generate_schema_data_id, generate_schema_id
 from ..inference_settings import InferenceSettings
 from .documents import EvaluationDocument
 from .iterations import Iteration
 
-
-# Actual Object stored in DB
-class Evaluation(BaseModel):
+class BaseEvaluation(BaseModel):
     id: str = Field(default_factory=lambda: "eval_" + nanoid.generate())
+    name: str = Field(default="", description="The name of the evaluation")
+    json_schema: dict[str, Any] = Field(default_factory=dict, description="The json schema of the evaluation")
+    project_id: str = Field(description="The ID of the project", default="default_spreadsheets")
+    default_inference_settings: InferenceSettings = Field(default=InferenceSettings(), description="The default inference properties for the evaluation.")
     updated_at: datetime.datetime = Field(default_factory=lambda: datetime.datetime.now(tz=datetime.timezone.utc))
 
-    name: str
+
+# Actual Object stored in DB
+class Evaluation(BaseEvaluation):
     documents: list[EvaluationDocument] = Field(default_factory=list)
     iterations: list[Iteration] = Field(default_factory=list)
-    json_schema: dict[str, Any]
-
-    project_id: str = Field(description="The ID of the project", default="default_spreadsheets")
-    default_inference_settings: InferenceSettings = Field(
-        default=InferenceSettings(), description="The default inference properties for the evaluation (mostly used in the frontend)"
-    )
 
     @computed_field  # type: ignore
     @property
@@ -35,7 +31,7 @@ class Evaluation(BaseModel):
         Returns:
             str: A SHA1 hash string representing the schema data version.
         """
-        return compute_schema_data_id(self.json_schema)
+        return generate_schema_data_id(self.json_schema)
 
     # This is a computed field, it is exposed when serializing the object
     @computed_field  # type: ignore
@@ -46,14 +42,7 @@ class Evaluation(BaseModel):
         Returns:
             str: A SHA1 hash string representing the complete schema version.
         """
-        return "sch_id_" + generate_blake2b_hash_from_string(json.dumps(self.json_schema, sort_keys=True).strip())
-
-
-class CreateEvaluation(BaseModel):
-    name: str
-    json_schema: dict[str, Any]
-    project_id: str = Field(description="The ID of the project", default="default_spreadsheets")
-    default_inference_settings: InferenceSettings = Field(default=InferenceSettings(), description="The default inference properties for the evaluation.")
+        return generate_schema_id(self.json_schema)
 
 
 class ListEvaluationParams(BaseModel):
@@ -62,6 +51,15 @@ class ListEvaluationParams(BaseModel):
     schema_data_id: Optional[str] = Field(default=None, description="The ID of the schema data")
 
 
+class CreateEvaluationRequest(BaseModel):
+    name: str
+    project_id: str
+    json_schema: dict[str, Any]
+    default_inference_settings: InferenceSettings
+
+
+# This is basically the same as BaseEvaluation, but everything is optional.
+# Could be achieved by convert_basemodel_to_partial_basemodel(BaseEvaluation) but we prefer explicitness
 class PatchEvaluationRequest(BaseModel):
     name: Optional[str] = Field(default=None, description="The name of the document")
     json_schema: Optional[dict[str, Any]] = Field(default=None, description="The json schema of the evaluation")

retab/types/extractions.py

@@ -77,6 +77,7 @@ class Extraction(BaseModel):
         default=None,
         description="The effort level for the model to reason about the input data.",
     )
+    n_consensus: int = Field(default=1, description="Number of consensus models used for the extraction")
     timings: list[ExtractionTimingStep] = Field(default_factory=list, description="Timings of the extraction")
 
     # Infered from the schema

retab/types/logs.py

@@ -7,8 +7,7 @@ from openai.types.chat.chat_completion import ChatCompletion
 from openai.types.chat.chat_completion_reasoning_effort import ChatCompletionReasoningEffort
 from pydantic import BaseModel, EmailStr, Field, HttpUrl, computed_field, field_validator
 
-from ..utils.json_schema import compute_schema_data_id
-from ..utils.mime import generate_blake2b_hash_from_string
+from ..utils.json_schema import generate_schema_data_id, generate_schema_id
 from ..utils.usage.usage import CostBreakdown, compute_cost_from_model, compute_cost_from_model_with_breakdown
 from .ai_models import Amount
 from .documents.extractions import RetabParsedChatCompletion
@@ -47,7 +46,7 @@ class ProcessorConfig(BaseModel):
         Returns:
             str: A SHA1 hash string representing the schema data version.
         """
-        return compute_schema_data_id(self.json_schema)
+        return generate_schema_data_id(self.json_schema)
 
     # This is a computed field, it is exposed when serializing the object
     @computed_field  # type: ignore
@@ -58,7 +57,7 @@ class ProcessorConfig(BaseModel):
         Returns:
             str: A SHA1 hash string representing the complete schema version.
         """
-        return "sch_id_" + generate_blake2b_hash_from_string(json.dumps(self.json_schema, sort_keys=True).strip())
+        return generate_schema_id(self.json_schema)
 
 
 class AutomationConfig(BaseModel):
@@ -111,7 +110,7 @@ class UpdateProcessorRequest(BaseModel):
         """
         if self.json_schema is None:
             return None
-        return compute_schema_data_id(self.json_schema)
+        return generate_schema_data_id(self.json_schema)
 
     @computed_field  # type: ignore
     @property
@@ -123,7 +122,7 @@ class UpdateProcessorRequest(BaseModel):
         """
         if self.json_schema is None:
             return None
-        return "sch_id_" + generate_blake2b_hash_from_string(json.dumps(self.json_schema, sort_keys=True).strip())
+        return generate_schema_id(self.json_schema)
 
 
 class UpdateAutomationRequest(BaseModel):

retab/types/mime.py

@@ -1,21 +1,12 @@
 import base64
 import datetime
 import gzip
-import hashlib
 import mimetypes
 import re
 from typing import Any, Optional, Self, Sequence
 
 from pydantic import BaseModel, Field, field_validator
-
-
-def generate_blake2b_hash_from_bytes(bytes_: bytes) -> str:
-    return hashlib.blake2b(bytes_, digest_size=8).hexdigest()
-
-
-def generate_blake2b_hash_from_base64(base64_string: str) -> str:
-    return generate_blake2b_hash_from_bytes(base64.b64decode(base64_string))
-
+from ..utils.hashing import generate_blake2b_hash_from_base64
 
 # **** OCR DATACLASSES (DocumentAI-compatible) ****
 class Point(BaseModel):

retab/types/schemas/enhance.py

@@ -1,6 +1,8 @@
 from typing import Any, Self, TypedDict
 
-from openai.types.chat.chat_completion_reasoning_effort import ChatCompletionReasoningEffort
+from openai.types.chat.chat_completion_reasoning_effort import (
+    ChatCompletionReasoningEffort,
+)
 from pydantic import BaseModel, Field, model_validator
 
 from ..mime import MIMEData
@@ -9,23 +11,35 @@ from ..browser_canvas import BrowserCanvas
 
 
 class EnhanceSchemaConfig(BaseModel):
+    allow_reasoning_fields_added: bool = True  # Whether to allow the llm to add reasoning fields
     allow_field_description_update: bool = False  # Whether to allow the llm to update the description of existing fields
     allow_system_prompt_update: bool = True  # Whether to allow the llm to update the system prompt
-    allow_reasoning_field_toggle: bool = False  # Whether to allow the llm to toggle the reasoning for fields
+    allow_field_simple_type_change: bool = False  # Whether to allow the llm to make simple type changes (optional, string to date, etc.)
+    allow_field_data_structure_breakdown: bool = False  # Whether to allow the llm to make complex data-structure changes (raw diff)
 
     # Model validator
     @model_validator(mode="after")
     def check_at_least_one_tool_allowed(self) -> Self:
-        if not any([self.allow_field_description_update, self.allow_system_prompt_update, self.allow_reasoning_field_toggle]):
+        if not any(
+            [
+                self.allow_reasoning_fields_added,
+                self.allow_field_description_update,
+                self.allow_system_prompt_update,
+                self.allow_field_simple_type_change,
+                self.allow_field_data_structure_breakdown,
+            ]
+        ):
             raise ValueError("At least one tool must be allowed")
         return self
 
 
 # Define a typed Dict for EnhanceSchemaConfig (for now it is kind static, but we will add more flexibility in the future)
 class EnhanceSchemaConfigDict(TypedDict, total=False):
+    allow_reasoning_fields_added: bool
     allow_field_description_update: bool
     allow_system_prompt_update: bool
-    allow_reasoning_field_toggle: bool
+    allow_field_simple_type_change: bool
+    allow_field_data_structure_breakdown: bool
 
 
 class EnhanceSchemaRequest(BaseModel):
@@ -48,7 +62,10 @@ class EnhanceSchemaRequest(BaseModel):
     stream: bool = False
     """Whether to stream the response."""
 
-    tools_config: EnhanceSchemaConfig = Field(default_factory=EnhanceSchemaConfig, description="The configuration for the tools to use")
+    tools_config: EnhanceSchemaConfig = Field(
+        default_factory=EnhanceSchemaConfig,
+        description="The configuration for the tools to use",
+    )
 
     json_schema: dict[str, Any]
     instructions: str | None = None

retab/types/schemas/evaluate.py

@@ -3,7 +3,7 @@ from typing import Any, Self
 from openai.types.chat.chat_completion_reasoning_effort import ChatCompletionReasoningEffort
 from pydantic import BaseModel, Field, model_validator
 
-from ..evals import ItemMetric
+from ..metrics import ItemMetric
 from ..mime import MIMEData
 from ..modalities import Modality
 from ..browser_canvas import BrowserCanvas

retab/types/schemas/object.py

@@ -233,6 +233,32 @@ When provided with a **JSON schema** and a **document**, you must:
 
 ---
 
+## Date and Time Formatting
+
+When extracting date, time, or datetime values:
+
+- **Always use ISO format** for dates and times (e.g., "2023-12-25", "14:30:00", "2023-12-25T14:30:00Z")
+- **Include timezone information** when available (e.g., "2023-12-25T14:30:00+02:00")
+- **Use UTC timezone** when timezone is not specified or unclear (e.g., "2023-12-25T14:30:00Z")
+- **Maintain precision** as found in the source document (seconds, milliseconds if present)
+
+**Examples:**
+
+```json
+// Correct ISO formats:
+{"date": "2023-12-25"}
+{"time": "14:30:00"}
+{"datetime": "2023-12-25T14:30:00Z"}
+{"datetime_with_tz": "2023-12-25T14:30:00+02:00"}
+
+// Incorrect formats:
+{"date": "12/25/2023"}
+{"time": "2:30 PM"}
+{"datetime": "Dec 25, 2023 at 2:30 PM"}
+```
+
+---
+
 ## Handling Missing and Nullable Fields
 
 ### Nullable Leaf Attributes

retab/types/standards.py

@@ -1,14 +1,14 @@
 from typing import Any, List, Literal, Optional, Tuple, TypeVar, TypedDict
 
 from pydantic import BaseModel, Field
-from pydantic.fields import _Unset
+from pydantic_core import PydanticUndefined
 
 # API Standards
 
 # Define a type variable to represent the content type
 T = TypeVar("T")
 
-FieldUnset = _Unset
+FieldUnset: Any = PydanticUndefined
 
 
 # Define the ErrorDetail model

retab/utils/__init__.py

@@ -0,0 +1,3 @@
+from .json_schema import filter_auxiliary_fields, flatten_dict, unflatten_dict
+
+__all__ = ["filter_auxiliary_fields", "flatten_dict", "unflatten_dict"]