sharedkernel 1.8.0__tar.gz → 2.0.0__tar.gz

{sharedkernel-1.8.0 → sharedkernel-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sharedkernel
-Version: 1.8.0
+Version: 2.0.0
 Summary: sharekernel is a shared package between all python projects
 Author: Smilinno
 Description-Content-Type: text/markdown
@@ -9,17 +9,26 @@ Requires-Dist: requests
 Requires-Dist: pymongo
 Requires-Dist: fastapi==0.111.0
 Requires-Dist: PyJWT
-Requires-Dist: chromadb
 Requires-Dist: persian_tools
 Requires-Dist: sentry-sdk
 Requires-Dist: jdatetime
 Requires-Dist: persiantools
 Requires-Dist: boto3
+Requires-Dist: python-docx
+Requires-Dist: mammoth
+Requires-Dist: markdown
+Requires-Dist: beautifulsoup4
 
 # SharedKernel
 this a shared kernel package
 
 # Change Log
+### Version 2.0.0
+- Update pydantic version
+- Delete vector databases
+- Implement JsonStringModel
+### Version 1.9.0
+- Implement DataFormatConverter
 ### Version 1.8.0
 - Implement persian number normalizer
 ### Version 1.7.3

{sharedkernel-1.8.0 → sharedkernel-2.0.0}/README.md

@@ -2,6 +2,12 @@
 this a shared kernel package
 
 # Change Log
+### Version 2.0.0
+- Update pydantic version
+- Delete vector databases
+- Implement JsonStringModel
+### Version 1.9.0
+- Implement DataFormatConverter
 ### Version 1.8.0
 - Implement persian number normalizer
 ### Version 1.7.3

{sharedkernel-1.8.0 → sharedkernel-2.0.0}/setup.py

@@ -13,7 +13,6 @@ setup(
     packages=[
         "sharedkernel",
         "sharedkernel.database",
-        "sharedkernel.database.vector_database_repository",
         "sharedkernel.enum",
         "sharedkernel.exception",
         "sharedkernel.objects",
@@ -26,16 +25,18 @@ setup(
         "pymongo",
         "fastapi==0.111.0",
         "PyJWT",
-        #"pymilvus",
-        "chromadb",
         "persian_tools",
         "sentry-sdk",
         "jdatetime",
         "persiantools",
-        "boto3"
+        "boto3",
+        "python-docx",
+        "mammoth",
+        "markdown",
+        "beautifulsoup4"
     ],
     # *strongly* suggested for sharing
-    version="1.8.0",
+    version="2.0.0",
     description="sharekernel is a shared package between all python projects",
     long_description=long_description,
     long_description_content_type="text/markdown",

sharedkernel-2.0.0/sharedkernel/data_format_converter.py

@@ -0,0 +1,108 @@
+from abc import ABC, abstractmethod
+import markdown
+
+from docx import Document
+from docx.enum.text import WD_ALIGN_PARAGRAPH
+from docx.oxml.ns import qn
+from docx.oxml import OxmlElement
+from bs4 import BeautifulSoup, NavigableString
+import mammoth
+
+
+class DataFormatConverter(ABC):
+    @abstractmethod
+    def convert(input_path, output_path):
+        pass
+
+    def _get_out_title(self, title: str, file_format: str = ".docx") -> str:
+        return ".".join(title.split(".")[:-1]) + file_format
+
+
+class Md2Docx(DataFormatConverter):
+
+    def __add_hyperlink(self, paragraph, url, text, color="0000FF", underline=True):
+        r_id = paragraph.part.relate_to(url, "hyperlink", is_external=True)
+        hyperlink = OxmlElement("w:hyperlink")
+        hyperlink.set(qn("r:id"), r_id)
+
+        new_run, rPr = OxmlElement("w:r"), OxmlElement("w:rPr")
+        if color:
+            color_element = OxmlElement("w:color")
+            color_element.set(qn("w:val"), color)
+            rPr.append(color_element)
+        if underline:
+            u = OxmlElement("w:u")
+            u.set(qn("w:val"), "single")
+            rPr.append(u)
+        new_run.append(rPr)
+
+        text_elem = OxmlElement("w:t")
+        text_elem.text = text
+        new_run.append(text_elem)
+        hyperlink.append(new_run)
+        paragraph._p.append(hyperlink)
+
+    def __set_right_to_left(self, paragraph):
+        paragraph.alignment = WD_ALIGN_PARAGRAPH.RIGHT
+        paragraph.paragraph_format.right_to_left = True
+
+    @staticmethod
+    def convert(input_path, output_path=None):
+        md2docx = Md2Docx()
+
+        with open(input_path, "r", encoding="utf-8") as f:
+            md_content = f.read()
+
+        output_file = output_path or md2docx._get_out_title(input_path)
+
+        html_content = markdown.markdown(md_content)
+        soup = BeautifulSoup(html_content, "html.parser")
+        doc = Document()
+
+        for element in soup:
+            if element.name == "h1":
+                paragraph = doc.add_paragraph(element.get_text(), style="Heading 1")
+            elif element.name == "h2":
+                paragraph = doc.add_paragraph(element.get_text(), style="Heading 2")
+            elif element.name == "p":
+                paragraph = doc.add_paragraph()
+                for part in element.contents:
+                    if isinstance(part, NavigableString):
+                        paragraph.add_run(str(part))
+                    elif part.name == "a":
+                        md2docx.__add_hyperlink(
+                            paragraph, part["href"], part.get_text()
+                        )
+                    elif part.name == "img":
+                        md2docx.__add_hyperlink(
+                            paragraph, part["src"], part.get("alt", "Image")
+                        )
+            md2docx.__set_right_to_left(paragraph)
+
+        doc.save(output_file)
+        return output_file
+
+
+class Docx2Md(DataFormatConverter):
+    def _get_out_title(self, title: str, file_format: str = ".txt") -> str:
+        return super()._get_out_title(title, file_format)
+
+    @staticmethod
+    def convert(input_path: str, output_path: str = None):
+        docx2md = Docx2Md()
+        with open(input_path, "rb") as docx:
+            result = mammoth.convert_to_markdown(
+                docx,
+                convert_image=mammoth.images.img_element(
+                    lambda image: {"src": "images/" + image.content_type.split("/")[-1]}
+                ),
+            )
+
+        output_path = output_path or docx2md._get_out_title(input_path)
+
+        with open(output_path, "w", encoding="utf-8") as md_file:
+            md_file.write(result.value)
+
+        if result.messages:
+            for message in result.messages:
+                print("Warning:", message)

sharedkernel-2.0.0/sharedkernel/database/__init__.py

@@ -0,0 +1 @@
+# from .mongo_repository_base import MongoRepositoryBase

{sharedkernel-1.8.0 → sharedkernel-2.0.0}/sharedkernel/database/mongo_generic_repository.py

@@ -1,7 +1,7 @@
 from pymongo import MongoClient
 from bson import ObjectId
 from typing import Generic, TypeVar, List, Type
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 from sharedkernel.string_extentions import camel_to_snake
 
 T = TypeVar("T", bound=BaseModel)
@@ -21,8 +21,7 @@ class MongoGenericRepository(Generic[T]):
     def find_one(self, id: str) -> T:
         query = {"_id": ObjectId(id), "is_deleted": False}
         result = self.collection.find_one(query)
-        return self._map_to_model(result) if result else None
-    
+        return self._map_to_model(result) if result else None   
 
     def insert_one(self, data: T) -> str:
         delattr(data, "id")

sharedkernel-2.0.0/sharedkernel/enum/error_code.py

@@ -0,0 +1,10 @@
+from enum import Enum
+
+
+class ErrorCode(str, Enum):
+    Item_NotFound = "یافت نشد"
+    Internal_Server = "خطایی در سیستم رخ داده است"
+    UnAuthorized = "توکن دسترسی معتبر نمی باشد"
+    Success = "با موفقیت انجام شد"
+    Intents_Count_Should_Equal_One = "فقط یک اینتنت میتوانید وارد نمایید"
+    Unsupported_Date_Type = "تاریخ داده شده پیشتیبانی نمیشود."

{sharedkernel-1.8.0 → sharedkernel-2.0.0}/sharedkernel/objects/base_document.py

@@ -1,10 +1,9 @@
 from pydantic import BaseModel, Field
-from typing import Optional
 from datetime import datetime
 
 
 class BaseDocument(BaseModel):
-    id: Optional[str]
-    is_deleted: bool = False
+    id: str | None = None
+    is_deleted: bool | None = False
     created_on: datetime = Field(default_factory=datetime.now)
     updated_on: datetime = Field(default_factory=datetime.now)

sharedkernel-2.0.0/sharedkernel/objects/json_string_model.py

@@ -0,0 +1,27 @@
+import json
+from typing import Any
+from pydantic import BaseModel, model_validator
+
+
+class JsonStringModel(BaseModel):
+    """`BaseModel` subclass used to validate objects passed to API as JSON strings.
+
+    The primary use case is treating one of the form fields as JSON payload. We need to do that
+    in case of endpoints that accept both file and rich set of input parameters. We cannot receive
+    both file and JSON body because they use conflicting Content-Type header. It is
+    "multipart/form-data" for the former and "application/json" for the latter. If we require
+    a client to send a file, we have to use the data sent in form for input parameters.
+
+    Theoretically we could use several form fields to gather all the required input parameters.
+    However, some of them are nested in their nature and forms don't support that. That's why we
+    consider it better to just use one form field and process its content as JSON payload.
+
+    If a form field model inherits from this class, we will get API documentation and input
+    validation - just the way we get it for JSON body defined with `BaseModel`.
+    """
+    @model_validator(mode='before')
+    @classmethod
+    def validate_to_json(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return cls(**json.loads(value))
+        return value

{sharedkernel-1.8.0 → sharedkernel-2.0.0}/sharedkernel/objects/result.py

@@ -1,10 +1,6 @@
-from pydantic import BaseModel
-
 from typing import Generic, TypeVar
-
 from pydantic import BaseModel
-from pydantic.generics import GenericModel
-
+from pydantic import BaseModel
 from sharedkernel.enum.error_code import ErrorCode
 
 ResultT = TypeVar("ResultT")
@@ -13,16 +9,16 @@ ResultT = TypeVar("ResultT")
 class BaseResult(BaseModel):
     isSucceed:bool = True
     message: str = ErrorCode.Success.value
-    errorCode: str = None
+    errorCode: str | None = None
 
-class Result(BaseResult,GenericModel, Generic[ResultT]):
-    data: ResultT =None
+class Result(BaseResult, Generic[ResultT]):
+    data: ResultT = None
 
     def __init__(
         self,
         isSucceed:bool = True,
-        data: object = None,
+        data: object | None = None,
         message: str = ErrorCode.Success.value,
-        errorCode: str = None
+        errorCode: str | None = None
         )-> None:
-        super().__init__(isSucceed= isSucceed, data= data, message= message, errorCode= errorCode)
+        super().__init__(isSucceed=isSucceed, data=data, message=message, errorCode=errorCode)

{sharedkernel-1.8.0 → sharedkernel-2.0.0}/sharedkernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sharedkernel
-Version: 1.8.0
+Version: 2.0.0
 Summary: sharekernel is a shared package between all python projects
 Author: Smilinno
 Description-Content-Type: text/markdown
@@ -9,17 +9,26 @@ Requires-Dist: requests
 Requires-Dist: pymongo
 Requires-Dist: fastapi==0.111.0
 Requires-Dist: PyJWT
-Requires-Dist: chromadb
 Requires-Dist: persian_tools
 Requires-Dist: sentry-sdk
 Requires-Dist: jdatetime
 Requires-Dist: persiantools
 Requires-Dist: boto3
+Requires-Dist: python-docx
+Requires-Dist: mammoth
+Requires-Dist: markdown
+Requires-Dist: beautifulsoup4
 
 # SharedKernel
 this a shared kernel package
 
 # Change Log
+### Version 2.0.0
+- Update pydantic version
+- Delete vector databases
+- Implement JsonStringModel
+### Version 1.9.0
+- Implement DataFormatConverter
 ### Version 1.8.0
 - Implement persian number normalizer
 ### Version 1.7.3

{sharedkernel-1.8.0 → sharedkernel-2.0.0}/sharedkernel.egg-info/SOURCES.txt

@@ -1,6 +1,7 @@
 README.md
 setup.py
 sharedkernel/common.py
+sharedkernel/data_format_converter.py
 sharedkernel/date_converter.py
 sharedkernel/jwt_service.py
 sharedkernel/regex_masking.py
@@ -13,13 +14,8 @@ sharedkernel.egg-info/requires.txt
 sharedkernel.egg-info/top_level.txt
 sharedkernel/database/__init__.py
 sharedkernel/database/mongo_generic_repository.py
-sharedkernel/database/vector_database_repository/__init__.py
-sharedkernel/database/vector_database_repository/chroma_startegy.py
-sharedkernel/database/vector_database_repository/vector_database_repository.py
-sharedkernel/database/vector_database_repository/vector_database_strategy.py
 sharedkernel/enum/__init__.py
 sharedkernel/enum/error_code.py
-sharedkernel/enum/vector_database_type.py
 sharedkernel/exception/__init__.py
 sharedkernel/exception/exception.py
 sharedkernel/exception/exception_handlers.py
@@ -28,5 +24,6 @@ sharedkernel/normalizer/number_normalizer.py
 sharedkernel/normalizer/phone_number_normalizer.py
 sharedkernel/objects/__init__.py
 sharedkernel/objects/base_document.py
+sharedkernel/objects/json_string_model.py
 sharedkernel/objects/jwt_model.py
 sharedkernel/objects/result.py

{sharedkernel-1.8.0 → sharedkernel-2.0.0}/sharedkernel.egg-info/requires.txt

@@ -3,9 +3,12 @@ requests
 pymongo
 fastapi==0.111.0
 PyJWT
-chromadb
 persian_tools
 sentry-sdk
 jdatetime
 persiantools
 boto3
+python-docx
+mammoth
+markdown
+beautifulsoup4

sharedkernel-1.8.0/sharedkernel/database/__init__.py

@@ -1,2 +0,0 @@
-from .vector_database_repository import VectorRepository
-# from .mongo_repository_base import MongoRepositoryBase

sharedkernel-1.8.0/sharedkernel/database/vector_database_repository/__init__.py

@@ -1,2 +0,0 @@
-from .chroma_startegy import ChromaStrategy
-from .vector_database_repository import VectorRepository

sharedkernel-1.8.0/sharedkernel/database/vector_database_repository/chroma_startegy.py

@@ -1,39 +0,0 @@
-import chromadb
-import numpy as np
-from chromadb.config import Settings
-from .vector_database_strategy import VectorDatabaseStrategy
-import uuid
-
-
-class ChromaStrategy(VectorDatabaseStrategy):
-    def __init__(self, collection_name: str):
-        self.collection_name = collection_name
-        self.collection = None
-
-    def connect(self, host: str = "localhost", port: int = 8000):
-        client = chromadb.Client(
-            Settings(
-                chroma_api_impl="rest",
-                chroma_server_host=host,
-                chroma_server_http_port=port,
-            )
-        )
-        self.collection = client.get_or_create_collection(self.collection_name)
-
-    def insert_vector(self, vector: np.ndarray, metadata: dict) -> str:
-        id = str(uuid.uuid4())
-        self.collection.upsert(ids=id, embeddings=vector.tolist(), metadatas=[metadata])
-
-        return id
-
-    def search_vector(self, vector: np.ndarray, top_k: int):
-        results = self.collection.query(vectors=[vector.tolist()], n_results=top_k)
-        return results
-
-    def get_vector_by_id(self, id: str):
-        result = self.collection.get(ids=id)
-
-        return result
-
-    def delete_vector(self, id: str):
-        self.collection.delete(ids=id)

sharedkernel-1.8.0/sharedkernel/database/vector_database_repository/vector_database_repository.py

@@ -1,28 +0,0 @@
-import numpy as np
-from .vector_database_strategy import VectorDatabaseStrategy
-# from .milvus_strategy import MilvusStrategy
-from .chroma_startegy import ChromaStrategy
-from sharedkernel.enum.vector_database_type import VectorDatabaseType
-
-class VectorRepository:
-    def __init__(self, database_type: VectorDatabaseType, collection_name: str, **connection_params):
-        self.strategy = self._get_strategy(database_type, collection_name)
-        self.strategy.connect(**connection_params)
-    
-    def _get_strategy(self, database_type: VectorDatabaseType, collection_name: str) -> VectorDatabaseStrategy:
-        # if database_type == VectorDatabaseType.MILVUS:
-        #     return MilvusStrategy(collection_name)
-        # else:
-        return ChromaStrategy(collection_name)
-    
-    def add_vector(self, vector: np.ndarray, metadata: dict) -> str:
-        return self.strategy.insert_vector(vector, metadata)
-    
-    def find_similar_vectors(self, vector: np.ndarray, top_k: int):
-        return self.strategy.search_vector(vector, top_k)
-    
-    def remove_vector(self, id: str):
-        self.strategy.delete_vector(id)
-
-    def get_vector_by_id(self, id: str):
-       return self.strategy.get_vector_by_id(id)

sharedkernel-1.8.0/sharedkernel/database/vector_database_repository/vector_database_strategy.py

@@ -1,22 +0,0 @@
-from abc import ABC, abstractmethod
-import numpy as np
-
-class VectorDatabaseStrategy(ABC):
-    @abstractmethod
-    def connect(self, **kwargs):
-        pass
-    @abstractmethod
-    def insert_vector(self, vector: np.ndarray, metadata: dict) -> str:
-        pass
-    
-    @abstractmethod
-    def search_vector(self, vector: np.ndarray, top_k: int):
-        pass
-    
-    @abstractmethod
-    def delete_vector(self, id: str):
-        pass
-    
-    @abstractmethod
-    def get_vector_by_id(self, id: str):
-        pass

sharedkernel-1.8.0/sharedkernel/enum/error_code.py

@@ -1,14 +0,0 @@
-from enum import Enum
-
-class ErrorCode(str, Enum):
-    Item_NotFound= "یافت نشد"
-
-    Internal_Server= "خطایی در سیستم رخ داده است"
-
-    UnAuthorized= "توکن دسترسی معتبر نمی باشد"
-
-    Success= "با موفقیت انجام شد"
-    
-    Intents_Count_Should_Equal_One= "فقط یک اینتنت میتوانید وارد نمایید"
-
-    Unsupported_Date_Type = "تاریخ داده شده پیشتیبانی نمیشود."

sharedkernel-1.8.0/sharedkernel/enum/vector_database_type.py

@@ -1,5 +0,0 @@
-from enum import Enum
-
-class VectorDatabaseType(Enum):
-    MILVUS = "milvus"
-    CHROMA = "chroma"