MindsDB 25.4.4.0__py3-none-any.whl → 25.5.3.0__py3-none-any.whl

mindsdb/interfaces/functions/to_markdown.py

@@ -1,13 +1,10 @@
-import base64
 from io import BytesIO
 import os
 from typing import Union
 from urllib.parse import urlparse
 
-import fitz  # PyMuPDF
-from markitdown import MarkItDown
+from aipdf import ocr
 import mimetypes
-from openai import OpenAI
 import requests
 
 
@@ -15,41 +12,22 @@ class ToMarkdown:
     """
     Extracts the content of documents of various formats in markdown format.
     """
-    def __init__(self, use_llm: bool, llm_client: OpenAI = None, llm_model: str = None):
+    def __init__(self):
         """
         Initializes the ToMarkdown class.
         """
-        # If use_llm is True, llm_client and llm_model must be provided.
-        if use_llm and (llm_client is None or llm_model is None):
-            raise ValueError('LLM client and model must be provided when use_llm is True.')
 
-        # If use_llm is False, set llm_client and llm_model to None even if they are provided.
-        if not use_llm:
-            llm_client = None
-            llm_model = None
-
-        # Only OpenAI is supported for now.
-        # TODO: Add support for other LLMs.
-        if llm_client is not None and not isinstance(llm_client, OpenAI):
-            raise ValueError('Only OpenAI models are supported at the moment.')
-
-        self.use_llm = use_llm
-        self.llm_client = llm_client
-        self.llm_model = llm_model
-
-    def call(self, file_path_or_url: str) -> str:
+    def call(self, file_path_or_url: str, **kwargs) -> str:
         """
         Converts a file to markdown.
         """
         file_extension = self._get_file_extension(file_path_or_url)
-        file = self._get_file_content(file_path_or_url)
+        file_content = self._get_file_content(file_path_or_url)
 
         if file_extension == '.pdf':
-            return self._pdf_to_markdown(file)
-        elif file_extension in ['.jpg', '.jpeg', '.png', '.gif']:
-            return self._image_to_markdown(file)
+            return self._pdf_to_markdown(file_content, **kwargs)
         else:
-            return self._other_to_markdown(file)
+            raise ValueError(f"Unsupported file type: {file_extension}.")
 
     def _get_file_content(self, file_path_or_url: str) -> str:
         """
@@ -90,105 +68,12 @@ class ToMarkdown:
         else:
             return os.path.splitext(file_path_or_url)[1]
 
-    def _pdf_to_markdown(self, file_content: Union[requests.Response, bytes]) -> str:
+    def _pdf_to_markdown(self, file_content: Union[requests.Response, bytes], **kwargs) -> str:
         """
         Converts a PDF file to markdown.
         """
-        if self.llm_client is None:
-            return self._pdf_to_markdown_no_llm(file_content)
-        else:
-            return self._pdf_to_markdown_llm(file_content)
-
-    def _pdf_to_markdown_llm(self, file_content: Union[requests.Response, BytesIO]) -> str:
-        """
-        Converts a PDF file to markdown using LLM.
-        The LLM is used mainly for the purpose of generating descriptions of any images in the PDF.
-        """
         if isinstance(file_content, requests.Response):
             file_content = BytesIO(file_content.content)
 
-        document = fitz.open(stream=file_content, filetype="pdf")
-
-        markdown_content = []
-        for page_num in range(len(document)):
-            page = document.load_page(page_num)
-
-            # Get text blocks with coordinates.
-            page_content = []
-            blocks = page.get_text("blocks")
-            for block in blocks:
-                x0, y0, x1, y1, text, _, _ = block
-                if text.strip():  # Skip empty or whitespace blocks.
-                    page_content.append((y0, text.strip()))
-
-            # Extract images from the page.
-            image_list = page.get_images(full=True)
-            for img_index, img in enumerate(image_list):
-                xref = img[0]
-                base_image = document.extract_image(xref)
-                image_bytes = base_image["image"]
-
-                # Use actual image y-coordinate if available.
-                y0 = float(base_image.get("y", 0))
-                image_description = self._generate_image_description(image_bytes)
-                page_content.append((y0, f"![{image_description}](image_{page_num + 1}_{img_index + 1}.png)"))
-
-            # Sort the content by y0 coordinate
-            page_content.sort(key=lambda x: x[0])
-
-            # Add sorted content to the markdown
-            for _, text in page_content:
-                markdown_content.append(text)
-            markdown_content.append("\n")
-
-        document.close()
-
-        return "\n".join(markdown_content)
-
-    def _generate_image_description(self, image_bytes: bytes) -> str:
-        """
-        Generates a description of the image using LLM.
-        """
-        image_base64 = base64.b64encode(image_bytes).decode("utf-8")
-
-        response = self.llm_client.chat.completions.create(
-            model=self.llm_model,
-            messages=[
-                {
-                    "role": "user",
-                    "content": [
-                        {"type": "text", "text": "Describe this image"},
-                        {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_base64}"}},
-                    ],
-                }
-            ],
-        )
-        description = response.choices[0].message.content
-        return description
-
-    def _pdf_to_markdown_no_llm(self, file_content: Union[requests.Response, BytesIO]) -> str:
-        """
-        Converts a PDF file to markdown without using LLM.
-        """
-        md = MarkItDown(enable_plugins=True)
-        result = md.convert(file_content)
-        return result.markdown
-
-    def _image_to_markdown(self, file_content: Union[requests.Response, BytesIO]) -> str:
-        """
-        Converts images to markdown.
-        """
-        if not self.use_llm or self.llm_client is None:
-            raise ValueError('LLM client must be enabled to convert images to markdown.')
-
-        md = MarkItDown(llm_client=self.llm_client, llm_model=self.llm_model, enable_plugins=True)
-        result = md.convert(file_content)
-        return result.markdown
-
-    def _other_to_markdown(self, file_content: Union[requests.Response, BytesIO]) -> str:
-        """
-        Converts other file formats to markdown.
-        """
-        md = MarkItDown(enable_plugins=True)
-        result = md.convert(file_content)
-        return result.markdown
+        markdown_pages = ocr(file_content, **kwargs)
+        return "\n\n---\n\n".join(markdown_pages)

mindsdb/interfaces/knowledge_base/controller.py

@@ -27,7 +27,7 @@ from mindsdb.integrations.libs.vectordatabase_handler import (
 from mindsdb.integrations.utilities.rag.rag_pipeline_builder import RAG
 from mindsdb.integrations.utilities.rag.config_loader import load_rag_config
 from mindsdb.integrations.utilities.handler_utils import get_api_key
-from mindsdb.integrations.handlers.langchain_embedding_handler.langchain_embedding_handler import construct_model_from_args, row_to_document
+from mindsdb.integrations.handlers.langchain_embedding_handler.langchain_embedding_handler import construct_model_from_args
 
 from mindsdb.interfaces.agents.constants import DEFAULT_EMBEDDINGS_MODEL_CLASS
 from mindsdb.interfaces.agents.langchain_agent import create_chat_model, get_llm_provider
@@ -37,21 +37,35 @@ from mindsdb.interfaces.knowledge_base.preprocessing.document_preprocessor impor
 from mindsdb.interfaces.model.functions import PredictorRecordNotFound
 from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
 from mindsdb.integrations.utilities.sql_utils import FilterCondition, FilterOperator
+from mindsdb.utilities.config import config
 from mindsdb.utilities.context import context as ctx
 
 from mindsdb.api.executor.command_executor import ExecuteCommands
+from mindsdb.api.executor.utilities.sql import query_df
 from mindsdb.utilities import log
-from mindsdb.integrations.utilities.rag.rerankers.reranker_compressor import LLMReranker
+from mindsdb.integrations.utilities.rag.rerankers.base_reranker import BaseLLMReranker
 
 logger = log.getLogger(__name__)
 
 KB_TO_VECTORDB_COLUMNS = {
-    'id': 'original_row_id',
+    'id': 'original_doc_id',
     'chunk_id': 'id',
     'chunk_content': 'content'
 }
 
 
+def get_model_params(model_params: dict, default_config_key: str):
+    """
+    Get model parameters by combining default config with user provided parameters.
+    """
+    combined_model_params = copy.deepcopy(config.get(default_config_key, {}))
+
+    if model_params:
+        combined_model_params.update(model_params)
+
+    return combined_model_params
+
+
 def get_embedding_model_from_params(embedding_model_params: dict):
     """
     Create embedding model from parameters.
@@ -89,7 +103,7 @@ def get_reranking_model_from_params(reranking_model_params: dict):
         params_copy["api_key"] = get_api_key(provider, params_copy, strict=False)
     params_copy['model'] = params_copy.pop('model_name', None)
 
-    return LLMReranker(**params_copy)
+    return BaseLLMReranker(**params_copy)
 
 
 class KnowledgeBaseTable:
@@ -137,13 +151,8 @@ class KnowledgeBaseTable:
         query.from_table = Identifier(parts=[self._kb.vector_database_table])
         logger.debug(f"Set table name to: {self._kb.vector_database_table}")
 
-        requested_kb_columns = []
-        for target in query.targets:
-            if isinstance(target, Star):
-                requested_kb_columns = None
-                break
-            else:
-                requested_kb_columns.append(target.parts[-1].lower())
+        # Copy query for complex execution via DuckDB: DISTINCT, GROUP BY etc.
+        query_copy = copy.deepcopy(query)
 
         query.targets = [
             Identifier(TableField.ID.value),
@@ -207,15 +216,23 @@ class KnowledgeBaseTable:
 
         df = self.add_relevance(df, query_text, relevance_threshold)
 
-        # filter by targets
-        if requested_kb_columns is not None:
-            df = df[requested_kb_columns]
+        if (
+            query.group_by is not None
+            or query.order_by is not None
+            or query.having is not None
+            or query.distinct is True
+            or len(query.targets) != 1
+            or not isinstance(query.targets[0], Star)
+        ):
+            query_copy.where = None
+            df = query_df(df, query_copy, session=self.session)
+
         return df
 
     def add_relevance(self, df, query_text, relevance_threshold=None):
         relevance_column = TableField.RELEVANCE.value
 
-        reranking_model_params = self._kb.params.get("reranking_model")
+        reranking_model_params = get_model_params(self._kb.params.get("reranking_model"), "default_llm")
         if reranking_model_params and query_text and len(df) > 0:
             # Use reranker for relevance score
             try:
@@ -277,7 +294,7 @@ class KnowledgeBaseTable:
         columns = list(df.columns)
         # update id, get from metadata
         df[TableField.ID.value] = df[TableField.METADATA.value].apply(
-            lambda m: None if m is None else m.get('original_row_id')
+            lambda m: None if m is None else m.get('original_doc_id')
         )
 
         # id on first place
@@ -466,12 +483,9 @@ class KnowledgeBaseTable:
                     # Use provided_id directly if it exists, otherwise generate one
                     doc_id = self._generate_document_id(content_str, col, provided_id)
 
-                    # Need provided ID to link chunks back to original source (e.g. database row).
-                    row_id = provided_id if provided_id else idx
-
                     metadata = {
                         **base_metadata,
-                        'original_row_id': str(row_id),
+                        'original_row_index': str(idx),  # provide link to original row index
                         'content_column': col,
                     }
 
@@ -657,47 +671,34 @@ class KnowledgeBaseTable:
         if df.empty:
             return pd.DataFrame([], columns=[TableField.EMBEDDINGS.value])
 
-        # keep only content
-        df = df[[TableField.CONTENT.value]]
-
         model_id = self._kb.embedding_model_id
-        if model_id:
-            # get the input columns
-            model_rec = db.session.query(db.Predictor).filter_by(id=model_id).first()
 
-            assert model_rec is not None, f"Model not found: {model_id}"
-            model_project = db.session.query(db.Project).filter_by(id=model_rec.project_id).first()
+        # get the input columns
+        model_rec = db.session.query(db.Predictor).filter_by(id=model_id).first()
 
-            project_datanode = self.session.datahub.get(model_project.name)
+        assert model_rec is not None, f"Model not found: {model_id}"
+        model_project = db.session.query(db.Project).filter_by(id=model_rec.project_id).first()
 
-            model_using = model_rec.learn_args.get('using', {})
-            input_col = model_using.get('question_column')
-            if input_col is None:
-                input_col = model_using.get('input_column')
+        project_datanode = self.session.datahub.get(model_project.name)
 
-            if input_col is not None and input_col != TableField.CONTENT.value:
-                df = df.rename(columns={TableField.CONTENT.value: input_col})
+        model_using = model_rec.learn_args.get('using', {})
+        input_col = model_using.get('question_column')
+        if input_col is None:
+            input_col = model_using.get('input_column')
 
-            df_out = project_datanode.predict(
-                model_name=model_rec.name,
-                df=df,
-                params=self.model_params
-            )
+        if input_col is not None and input_col != TableField.CONTENT.value:
+            df = df.rename(columns={TableField.CONTENT.value: input_col})
 
-            target = model_rec.to_predict[0]
-            if target != TableField.EMBEDDINGS.value:
-                # adapt output for vectordb
-                df_out = df_out.rename(columns={target: TableField.EMBEDDINGS.value})
-
-        elif self._kb.params.get('embedding_model'):
-            embedding_model = get_embedding_model_from_params(self._kb.params.get('embedding_model'))
-
-            df_texts = df.apply(row_to_document, axis=1)
-            embeddings = embedding_model.embed_documents(df_texts.tolist())
-            df_out = df.copy().assign(**{TableField.EMBEDDINGS.value: embeddings})
+        df_out = project_datanode.predict(
+            model_name=model_rec.name,
+            df=df,
+            params=self.model_params
+        )
 
-        else:
-            raise ValueError("No embedding model found for the knowledge base.")
+        target = model_rec.to_predict[0]
+        if target != TableField.EMBEDDINGS.value:
+            # adapt output for vectordb
+            df_out = df_out.rename(columns={target: TableField.EMBEDDINGS.value})
 
         df_out = df_out[[TableField.EMBEDDINGS.value]]
 
@@ -728,14 +729,15 @@ class KnowledgeBaseTable:
         """
         # Get embedding model from knowledge base
         embeddings_model = None
+        embedding_model_params = get_model_params(self._kb.params.get('embedding_model', {}), 'default_embedding_model')
         if self._kb.embedding_model:
             # Extract embedding model args from knowledge base table
             embedding_args = self._kb.embedding_model.learn_args.get('using', {})
             # Construct the embedding model directly
             embeddings_model = construct_model_from_args(embedding_args)
             logger.debug(f"Using knowledge base embedding model with args: {embedding_args}")
-        elif self._kb.params.get('embedding_model'):
-            embeddings_model = get_embedding_model_from_params(self._kb.params['embedding_model'])
+        elif embedding_model_params:
+            embeddings_model = get_embedding_model_from_params(embedding_model_params)
             logger.debug(f"Using knowledge base embedding model from params: {self._kb.params['embedding_model']}")
         else:
             embeddings_model = DEFAULT_EMBEDDINGS_MODEL_CLASS()
@@ -786,7 +788,7 @@ class KnowledgeBaseTable:
     def _generate_document_id(self, content: str, content_column: str, provided_id: str = None) -> str:
         """Generate a deterministic document ID using the utility function."""
         from mindsdb.interfaces.knowledge_base.utils import generate_document_id
-        return generate_document_id(content, content_column, provided_id)
+        return generate_document_id(content=content, provided_id=provided_id)
 
     def _convert_metadata_value(self, value):
         """
@@ -869,35 +871,33 @@ class KnowledgeBaseController:
                 return kb
             raise EntityExistsError("Knowledge base already exists", name)
 
-        embedding_model_params = params.get('embedding_model', None)
-        reranking_model_params = params.get('reranking_model', None)
+        embedding_params = copy.deepcopy(config.get('default_embedding_model', {}))
 
+        model_name = None
+        model_project = project
         if embedding_model:
             model_name = embedding_model.parts[-1]
+            if len(embedding_model.parts) > 1:
+                model_project = self.session.database_controller.get_project(embedding_model.parts[-2])
 
-        elif embedding_model_params:
-            # Get embedding model from params.
-            # This is called here to check validaity of the parameters.
-            get_embedding_model_from_params(
-                embedding_model_params
-            )
+        elif 'embedding_model' in params:
+            if isinstance(params['embedding_model'], str):
+                # it is model name
+                model_name = params['embedding_model']
+            else:
+                # it is params for model
+                embedding_params.update(params['embedding_model'])
 
-        else:
-            model_name = self._get_default_embedding_model(
+        if model_name is None:
+            model_name = self._create_embedding_model(
                 project.name,
-                params=params
+                params=embedding_params,
+                kb_name=name,
             )
-            params['default_embedding_model'] = model_name
-
-        model_project = None
-        if embedding_model is not None and len(embedding_model.parts) > 1:
-            # model project is set
-            model_project = self.session.database_controller.get_project(embedding_model.parts[-2])
-        elif not embedding_model_params:
-            model_project = project
+            params['created_embedding_model'] = model_name
 
         embedding_model_id = None
-        if model_project:
+        if model_name is not None:
             model = self.session.model_controller.get_model(
                 name=model_name,
                 project_name=model_project.name
@@ -905,6 +905,7 @@ class KnowledgeBaseController:
             model_record = db.Predictor.query.get(model['id'])
             embedding_model_id = model_record.id
 
+        reranking_model_params = get_model_params(params.get('reranking_model', {}), 'default_llm')
         if reranking_model_params:
             # Get reranking model from params.
             # This is called here to check validaity of the parameters.
@@ -989,38 +990,54 @@ class KnowledgeBaseController:
         self.session.integration_controller.add(vector_store_name, engine, connection_args)
         return vector_store_name
 
-    def _get_default_embedding_model(self, project_name, engine="langchain_embedding", params: dict = None):
+    def _create_embedding_model(self, project_name, engine="openai", params: dict = None, kb_name=''):
         """create a default embedding model for knowledge base, if not specified"""
-        model_name = "kb_default_embedding_model"
+        model_name = f"kb_embedding_{kb_name}"
 
-        # check exists
+        # drop if exists - parameters can be different
         try:
             model = self.session.model_controller.get_model(model_name, project_name=project_name)
             if model is not None:
-                return model_name
+                self.session.model_controller.delete_model(model_name, project_name)
         except PredictorRecordNotFound:
             pass
 
-        using_args = {
-            'engine': engine
-        }
-        if engine == 'langchain_embedding':
-            # Use default embeddings.
-            using_args['class'] = 'openai'
+        if 'provider' in params:
+            engine = params.pop('provider').lower()
+
+        api_key = get_api_key(engine, params, strict=False) or params.pop('api_key')
+
+        if engine == 'azure_openai':
+            engine = 'openai'
+            params['provider'] = 'azure'
+
+        if engine == 'openai':
+            if 'question_column' not in params:
+                params['question_column'] = 'content'
+            if api_key:
+                params[f"{engine}_api_key"] = api_key
+            if 'base_url' in params:
+                params['api_base'] = params.pop('base_url')
+
+        params['engine'] = engine
+        params['join_learn_process'] = True
+        params['mode'] = 'embedding'
 
         # Include API key if provided.
-        using_args.update({k: v for k, v in params.items() if 'api_key' in k})
         statement = CreatePredictor(
             name=Identifier(parts=[project_name, model_name]),
-            using=using_args,
+            using=params,
             targets=[
                 Identifier(parts=[TableField.EMBEDDINGS.value])
             ]
         )
 
         command_executor = ExecuteCommands(self.session)
-        command_executor.answer_create_predictor(statement, project_name)
-
+        resp = command_executor.answer_create_predictor(statement, project_name)
+        # check model status
+        record = resp.data.records[0]
+        if record['STATUS'] == 'error':
+            raise ValueError('Embedding model error:' + record['ERROR'])
         return model_name
 
     def delete(self, name: str, project_name: int, if_exists: bool = False) -> None:
@@ -1054,9 +1071,9 @@ class KnowledgeBaseController:
                 self.session.integration_controller.delete(kb.params['default_vector_storage'])
             except EntityNotExistsError:
                 pass
-        if 'default_embedding_model' in kb.params:
+        if 'created_embedding_model' in kb.params:
             try:
-                self.session.model_controller.delete_model(kb.params['default_embedding_model'], project_name)
+                self.session.model_controller.delete_model(kb.params['created_embedding_model'], project_name)
             except EntityNotExistsError:
                 pass
 

mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py

@@ -25,6 +25,8 @@ from langchain_core.documents import Document as LangchainDocument
 
 logger = log.getLogger(__name__)
 
+_DEFAULT_CONTENT_COLUMN_NAME = "content"
+
 
 class DocumentPreprocessor:
     """Base class for document preprocessing"""
@@ -90,14 +92,18 @@ class DocumentPreprocessor:
         start_char: Optional[int] = None,
         end_char: Optional[int] = None,
         provided_id: str = None,
+        content_column: str = None,
     ) -> str:
         """Generate human-readable deterministic ID for a chunk
-        Format: <doc_id>:<chunk_number>of<total_chunks>:<start_char>to<end_char>
+        Format: <doc_id>:<content_column>:<chunk_number>of<total_chunks>:<start_char>to<end_char>
         """
         if provided_id is None:
             raise ValueError("Document ID must be provided for chunk ID generation")
 
-        chunk_id = f"{provided_id}:{chunk_index + 1}of{total_chunks}:{start_char}to{end_char}"
+        if content_column is None:
+            raise ValueError("Content column must be provided for chunk ID generation")
+
+        chunk_id = f"{provided_id}:{content_column}:{chunk_index + 1}of{total_chunks}:{start_char}to{end_char}"
         logger.debug(f"Generated chunk ID: {chunk_id}")
         return chunk_id
 
@@ -254,8 +260,15 @@ Please give a short succinct context to situate this chunk within the overall do
             if doc.metadata:
                 metadata.update(doc.metadata)
 
+            # Get content_column from metadata or use default
+            content_column = metadata.get('content_column')
+            if content_column is None:
+                # If content_column is not in metadata, use the default column name
+                content_column = _DEFAULT_CONTENT_COLUMN_NAME
+                logger.debug(f"No content_column found in metadata, using default: {_DEFAULT_CONTENT_COLUMN_NAME}")
+
             chunk_id = self._generate_chunk_id(
-                chunk_index=chunk_index, provided_id=doc.id
+                chunk_index=chunk_index, provided_id=doc.id, content_column=content_column
             )
             processed_chunks.append(
                 ProcessedChunk(
@@ -324,13 +337,23 @@ class TextChunkingPreprocessor(DocumentPreprocessor):
                 metadata["start_char"] = start_char
                 metadata["end_char"] = end_char
 
-                # Generate chunk ID with total chunks
+                # Get content_column from metadata or use default
+                content_column = None
+                if doc.metadata:
+                    content_column = doc.metadata.get('content_column')
+
+                if content_column is None:
+                    # If content_column is not in metadata, use the default column name
+                    content_column = _DEFAULT_CONTENT_COLUMN_NAME
+                    logger.debug(f"No content_column found in metadata, using default: {_DEFAULT_CONTENT_COLUMN_NAME}")
+
                 chunk_id = self._generate_chunk_id(
                     chunk_index=i,
                     total_chunks=total_chunks,
                     start_char=start_char,
                     end_char=end_char,
-                    provided_id=doc.id
+                    provided_id=doc.id,
+                    content_column=content_column
                 )
 
                 processed_chunks.append(

mindsdb/interfaces/knowledge_base/utils.py

@@ -2,27 +2,22 @@
 import hashlib
 
 
-def generate_document_id(content: str, content_column: str, provided_id: str = None) -> str:
+def generate_document_id(content: str, content_column: str = None, provided_id: str = None) -> str:
     """
-    Generate a deterministic document ID from content and column name.
-    If provided_id exists, combines it with content_column.
-    For generated IDs, uses a short hash of just the content to ensure
-    same content gets same base ID across different columns.
+    Generate a deterministic document ID from content.
+    If provided_id exists, returns it directly.
+    For generated IDs, uses a short hash of just the content.
 
     Args:
         content: The content string
-        content_column: Name of the content column
+        content_column: Name of the content column (not used in ID generation, kept for backward compatibility)
         provided_id: Optional user-provided ID
     Returns:
-        Deterministic document ID in format: <base_id>_<column>
-        where base_id is either the provided_id or a 16-char hash of content
+        Deterministic document ID (either provided_id or a 16-char hash of content)
     """
     if provided_id is not None:
-        base_id = provided_id
-    else:
-        # Generate a shorter 16-character hash based only on content
-        hash_obj = hashlib.md5(content.encode())
-        base_id = hash_obj.hexdigest()[:16]
+        return provided_id
 
-    # Append column name to maintain uniqueness across columns
-    return f"{base_id}_{content_column}"
+    # Generate a shorter 16-character hash based only on content
+    hash_obj = hashlib.md5(content.encode())
+    return hash_obj.hexdigest()[:16]

mindsdb/interfaces/model/model_controller.py

@@ -1,4 +1,3 @@
-import sys
 import copy
 import datetime as dt
 from copy import deepcopy
@@ -28,7 +27,6 @@ from mindsdb.utilities import log
 
 logger = log.getLogger(__name__)
 
-IS_PY36 = sys.version_info[1] <= 6
 
 default_project = config.get('default_project')