tgshops-integrations 2.4__py3-none-any.whl → 3.0__py3-none-any.whl

tgshops_integrations/middlewares/gateway.py

@@ -1,36 +1,68 @@
-from typing import List,Optional
-import importlib.util
+from typing import List, Optional
 from pathlib import Path
+import importlib.util
 
 from aiocache import cached
-from tgshops_integrations.models.products import ProductModel
+from models.products import ProductModel
+
+# TODO: For test purposes, remove in production
+import sys
+sys.path.append('/home/latoff/Desktop/MarketBots')
+
 from tgshops_integrations.nocodb_connector.client import NocodbClient
+from tgshops_integrations.nocodb_connector.model_mapping import (
+    dump_product_data,
+    dump_product_data_with_check,
+    get_pagination_info,
+    ID_FIELD,
+    parse_product_data,
+    PRODUCT_CATEGORY_ID_LOOKUP_FIELD,
+    PRODUCT_NAME_FIELD,
+    PRODUCT_PRICE_FIELD,
+    PRODUCT_STOCK_FIELD,
+    PRODUCT_IMAGES_LOOKUP_FIELD,
+    PRODUCT_EXTERNAL_ID,
+)
+from services.nocodb_connector.categories import CategoryManager
+from services.nocodb_connector.products import ProductManager
+from loguru import logger
 
-from tgshops_integrations.nocodb_connector.model_mapping import dump_product_data,dump_product_data_with_check, get_pagination_info, ID_FIELD, \
-    parse_product_data, PRODUCT_CATEGORY_ID_LOOKUP_FIELD, PRODUCT_NAME_FIELD, PRODUCT_PRICE_FIELD, \
-    PRODUCT_STOCK_FIELD, PRODUCT_IMAGES_LOOKUP_FIELD 
 
-from tgshops_integrations.nocodb_connector.categories import CategoryManager
-from tgshops_integrations.nocodb_connector.products import ProductManager
-from tgshops_integrations.nocodb_connector.tables import *
-# from .config import NOCODB_CATEGORIES,NOCODB_PRODUCTS,NOCODB_STATUSES,NOCODB_BOT_MESSAGES,NOCODB_ORDERS
+def custom_key_builder(func, *args, **kwargs):
+    """
+    Key builder function for caching.
+    Excludes 'self' by processing args from args[1:].
+    """
+    args_key_part = "-".join(str(arg) for arg in args[1:])
+    kwargs_key_part = "-".join(f"{key}-{value}" for key, value in sorted(kwargs.items()))
+    return f"{func.__name__}-{args_key_part}-{kwargs_key_part}"
 
-from loguru import logger
 
 class Gateway(NocodbClient):
 
-    def __init__(self,logging=False,NOCODB_HOST=None,NOCODB_API_KEY=None,SOURCE=None,filter_buttons=[],config_path=None,special_attributes=False):
-        super().__init__(NOCODB_HOST=NOCODB_HOST,NOCODB_API_KEY=NOCODB_API_KEY,SOURCE=SOURCE)
-        if config_path:
-            self.load_config_from_path(config_path)
+    def __init__(
+        self,
+        logging: bool = False,
+        NOCODB_HOST: Optional[str] = None,
+        NOCODB_API_KEY: Optional[str] = None,
+        SOURCE: Optional[str] = None,
+        filter_buttons: Optional[List[str]] = [],
+        config_path: Optional[Path] = None,
+        special_attributes: bool = False,
+    ):
+        super().__init__(NOCODB_HOST=NOCODB_HOST, NOCODB_API_KEY=NOCODB_API_KEY, SOURCE=SOURCE)
 
         self.logging = logging
-        self.required_fields = [self.config.PRODUCT_NAME_FIELD, self.config.PRODUCT_PRICE_FIELD]
+        self.required_fields = []
         self.projection = []
         self.special_attributes = special_attributes
         self.filter_buttons = filter_buttons
 
-    def load_config_from_path(self,config_path):
+        if config_path:
+            self.load_config_from_path(config_path)
+
+    def load_config_from_path(self, config_path: Path):
+        """Loads configuration from the specified path."""
         if config_path.exists():
             spec = importlib.util.spec_from_file_location("config", config_path)
             self.config = importlib.util.module_from_spec(spec)
@@ -38,80 +70,74 @@ class Gateway(NocodbClient):
         else:
             raise FileNotFoundError(f"Configuration file not found at {config_path}")
 
-    async def load_data(self,SOURCE=None):
-        self.SOURCE=SOURCE
-        await self.get_all_tables()
-        self.category_manager=CategoryManager(table_id=self.tables_list[self.config.NOCODB_CATEGORIES],NOCODB_HOST=self.NOCODB_HOST,NOCODB_API_KEY=self.NOCODB_API_KEY,logging=True,filter_buttons=self.filter_buttons)
-        self.product_manager=ProductManager(table_id=self.tables_list[self.config.NOCODB_PRODUCTS],NOCODB_HOST=self.NOCODB_HOST,NOCODB_API_KEY=self.NOCODB_API_KEY,logging=True)
-
-    async def create_product(self,product: ProductModel) -> ProductModel:
-        products_table = self.tables_list[self.config.NOCODB_PRODUCTS]
-        data = dump_product_data_with_check(data=product ,data_check=self.category_manager.categories)
-        # product_json = dump_product_data_with_check(data=product,data_check=self.categories)
-        external_id = data.pop("ID")
-        metadata = await self.get_table_meta(self.tables_list["Products"])
-        images_column=[column["id"] for column in metadata["columns"] if column["column_name"] == "Images"][0]
-        data[PRODUCT_IMAGES_LOOKUP_FIELD]=[image['title'] for image in data['Images']]
-
-        for num,item in enumerate(data['Images']):
-            url_before=item['url']
-            image_name=item['title']
-            data['Images'][num]['url']=await self.save_image_to_nocodb(source_column_id=self.SOURCE,image_url=url_before,image_name=image_name,product_table_name=products_table,images_column_id=images_column)
-
-        record = await self.create_table_record(table_name=products_table, record=data)
-        # logger.info(f"Created product {record['id']}")
-        logger.info(f"Created product {external_id}")
-
-    async def get_all_products(self):
-        actual_products=[]
-        products_portion=[]
-        portion=200
-        # TODO Check once busy
-        for i in range(10):
-            products_portion=await self.product_manager.get_products_v2(offset=i*portion,limit=portion)
-            actual_products.extend(products_portion)
-            if len(products_portion) < 200:
-                break
-        return actual_products
-    
-    async def update_products(self, external_products: List[ProductModel]):
-        products_table = self.tables_list[self.config.NOCODB_PRODUCTS]
-        await self.product_manager.update_attributes(products=external_products)
-        # Updates categories if there were a new ones created
-        # external_products=await self.category_manager.map_categories(external_products=external_products)
-        self.product_manager.actual_products=await self.get_all_products()
-        # self.product_manager.actual_products = await self.product_manager.get_products_v2(offset=0,limit=200)
+    async def load_data(self, SOURCE: Optional[str] = None):
+        """Loads necessary data including tables, categories, and products."""
+        self.SOURCE = SOURCE
+        self.tables_list = await self.get_all_tables()
+        self.products_table = self.tables_list[self.config.NOCODB_PRODUCTS]
+        self.category_manager = CategoryManager(
+            table_id=self.tables_list[self.config.NOCODB_CATEGORIES],
+            NOCODB_HOST=self.NOCODB_HOST,
+            NOCODB_API_KEY=self.NOCODB_API_KEY,
+            logging=True,
+            filter_buttons=self.filter_buttons,
+        )
+        self.product_manager = ProductManager(
+            table_id=self.tables_list[self.config.NOCODB_PRODUCTS],
+            NOCODB_HOST=self.NOCODB_HOST,
+            NOCODB_API_KEY=self.NOCODB_API_KEY,
+            logging=True,
+        )
+
+    @cached(ttl=60, key_builder=custom_key_builder)
+    async def update_attributes(self, products: List[ProductModel]):
+        """Updates attributes for the product table."""
+        system_attributes = [PRODUCT_EXTERNAL_ID, PRODUCT_IMAGES_LOOKUP_FIELD]
+        attributes = await self.get_table_meta(table_name=self.products_table)
+        self.columns = [item['title'].lower() for item in attributes.get('columns', [])]
+
+        # Ensure system attributes exist
+        for attribute_name in system_attributes:
+            if attribute_name.lower() not in self.columns:
+                await self.create_table_column(table_name=self.products_table, name=attribute_name)
+                logger.info(f"Created attribute: {attribute_name}")
+
+        # Validate and add extra attributes
+        for item in products:
+            attributes = await self.get_table_meta(table_name=self.products_table)
+            self.columns = [col['title'].lower() for col in attributes.get('columns', [])]
+
+            for attribute in item.extra_attributes:
+                if attribute.name.rstrip().lower() not in self.columns:
+                    await self.create_table_column(table_name=self.products_table, name=attribute.name.lower())
+                    logger.info(f"Created attribute: {attribute.name.lower()}")
 
-        self.ids_mapping={product.external_id : product.id for product in self.product_manager.actual_products}
-        products_meta= {product.external_id : product for product in self.product_manager.actual_products}
+    async def update_products(self, external_products: List[ProductModel]):
+        """Updates product data by comparing with existing records."""
+        await self.update_attributes(products=external_products)
+        self.actual_products = await self.product_manager.get_all_products()
+        self.ids_mapping = {product.external_id: product.id for product in self.actual_products}
+        self.products_meta = {product.external_id: product for product in self.actual_products}
 
         for product in external_products:
-            if product.external_id in self.ids_mapping.keys():
-                product.id=self.ids_mapping[product.external_id]
-                if self.product_manager.hash_product(product,special_attributes=self.special_attributes)!=self.product_manager.hash_product(products_meta[product.external_id],special_attributes=self.special_attributes):
-                    await self.update_product(product=product)
+            if product.external_id in self.ids_mapping:
+                product.id = self.ids_mapping[product.external_id]
+                current_hash = self.product_manager.hash_product(product, special_attributes=self.special_attributes)
+                existing_hash = self.product_manager.hash_product(
+                    self.products_meta[product.external_id],
+                    special_attributes=self.special_attributes,
+                )
+                if current_hash != existing_hash:
+                    await self.product_manager.update_product(product=product)
             else:
-                await self.create_product(product=product)
-        
-    async def update_product(self, product: ProductModel):
-        products_table = self.tables_list[self.config.NOCODB_PRODUCTS]
-        data = dump_product_data_with_check(data=product ,data_check=self.category_manager.categories)
-
-        await self.update_table_record(
-            table_name=products_table,
-            record_id=product.id,
-            updated_data=data)
-        logger.info(f"Updated product {product.external_id}")
-        
-        
-    def find_product_id_by_name(self,name: str):
-        for product in self.product_manager.actual_products.products:
-            if product.name == name:
-                return product.id
-        return None  # Return None if no product is found with the given name
-    
-    async def delete_all_products(self):  
-        items = await self.product_manager.get_products_v2(offset=0,limit=200)
-        products_table = self.tables_list[self.config.NOCODB_PRODUCTS]    
-        for num,item in enumerate(items):
-            await self.delete_table_record(products_table, item.id)
+                checked_data = dump_product_data_with_check(
+                    data=product,
+                    data_check=self.category_manager.categories,
+                )
+                await self.product_manager.create_product(product=product, checked_data=checked_data)
+
+    async def delete_all_products(self):
+        """Deletes all products in the table."""
+        items = await self.product_manager.get_products_v2(offset=0, limit=200)
+        for item in items:
+            await self.product_manager.delete_table_record(self.products_table, item.id)

tgshops_integrations/models/products.py

@@ -4,7 +4,6 @@ from datetime import datetime
 
 from pydantic import BaseModel, Field, schema, validator
 
-
 class ExtraAttribute(BaseModel):
     name: str
     description: Optional[str]
@@ -29,12 +28,12 @@ class ExternalProductModel(BaseModel):
 class ProductModel(BaseModel):
     id: Optional[str]
     external_id: Optional[str]
-    category: Optional[List[str]]
+    category: Optional[List[int]]
     category_name: Optional[List[str]]
     name: str
     description: Optional[str]
     price: Optional[float]
-    final_price: Optional[float]
+    final_price: Optional[int]
     currency: Optional[str]
     stock_qty: int
     orders_qty: int = Field(0, hidden_field=True)

tgshops_integrations/nocodb_connector/categories_management.py

@@ -0,0 +1,149 @@
+from typing import List
+from loguru import logger
+from aiocache import cached
+from models.categories import CategoryModel, CategoryResponseModel, CategoryListResponseModel
+from models.products import ProductModel
+from tgshops_integrations.nocodb_connector.client import custom_key_builder, NocodbClient
+from tgshops_integrations.nocodb_connector.model_mapping import (
+    CATEGORY_IMAGE_FIELD,
+    CATEGORY_NAME_FIELD,
+    CATEGORY_PARENT_FIELD,
+    CATEGORY_PARENT_ID_FIELD,
+    PRODUCT_NAME_FIELD,
+    CATEGORY_ID_OF_CATEGORY_FIELD,
+    dump_category_data,
+    get_pagination_info,
+    parse_category_data,
+)
+
+class CategoryManager(NocodbClient):
+    def __init__(
+        self,
+        table_id=None,
+        logging=False,
+        config_type=None,
+        NOCODB_HOST=None,
+        NOCODB_API_KEY=None,
+        SOURCE=None,
+        filter_buttons=None,
+    ):
+        super().__init__(NOCODB_HOST=NOCODB_HOST, NOCODB_API_KEY=NOCODB_API_KEY, SOURCE=SOURCE)
+        self.NOCODB_HOST = NOCODB_HOST
+        self.NOCODB_API_KEY = NOCODB_API_KEY
+        self.SOURCE = SOURCE
+        self.CONFIG_TYPE = config_type
+        self.categories_table = table_id
+        self.external_categories = {}
+        self.logging = logging
+        self.filter_categories = []
+        self.filter_buttons = filter_buttons or []
+        self.required_fields = [CATEGORY_NAME_FIELD]
+        self.projection = ["Id", CATEGORY_NAME_FIELD, CATEGORY_PARENT_ID_FIELD, CATEGORY_ID_OF_CATEGORY_FIELD]
+
+    @cached(ttl=30, key_builder=custom_key_builder)
+    async def get_categories(self, table_id: str) -> List[CategoryModel]:
+        records = await self.get_table_records(table_id, self.required_fields, self.projection)
+        return [parse_category_data(record) for record in records]
+
+    @cached(ttl=30, key_builder=custom_key_builder)
+    async def get_categories_v2(self, table_id: str, offset: int = None, limit: int = None) -> CategoryListResponseModel:
+        response = await self.get_table_records_v2(
+            table_name=self.categories_table,
+            required_fields=self.required_fields,
+            projection=self.projection,
+            offset=offset,
+            limit=limit,
+        )
+        page_info = get_pagination_info(page_info=response['pageInfo'])
+        categories = [parse_category_data(record) for record in response['list']]
+        return CategoryListResponseModel(categories=categories, page_info=page_info)
+
+    @cached(ttl=30, key_builder=custom_key_builder)
+    async def get_category(self, table_id: str, category_id: str) -> CategoryModel:
+        record = await self.get_table_record(self.categories_table, category_id, self.required_fields, self.projection)
+        return parse_category_data(record)
+
+    async def create_category(self, table_id: str, category: CategoryModel) -> CategoryModel:
+        category_json = dump_category_data(category)
+        record = await self.create_table_record(self.categories_table, category_json)
+        return parse_category_data(record)
+
+    @cached(ttl=30, key_builder=custom_key_builder)
+    async def get_categories_in_category(self, table_id: str, category_id: str) -> List[CategoryModel]:
+        extra_where = (
+            f"({CATEGORY_PARENT_ID_FIELD},eq,{category_id})"
+            if category_id
+            else f"({CATEGORY_PARENT_FIELD},eq,0)"
+        )
+        records = await self.get_table_records(
+            table_name=self.categories_table,
+            required_fields=self.required_fields,
+            projection=self.projection,
+            extra_where=extra_where,
+        )
+        return [parse_category_data(record) for record in records]
+
+    async def update_categories(self, external_products: List[ProductModel]) -> None:
+        self.categories = await self.get_product_categories(table_id=self.categories_table, table_name=PRODUCT_NAME_FIELD)
+        categories_list = list(self.categories.keys())
+        properties_to_create = []
+
+        for product in external_products:
+            for num, product_property in enumerate(product.product_properties):
+                if product_property not in categories_list:
+                    parent_id = (
+                        self.categories[self.filter_buttons[num]]
+                        if self.filter_buttons
+                        else self.categories[product.category_I_name[0]]
+                    )
+                    properties_to_create.append([product_property, parent_id])
+                    categories_list.append(product_property)
+
+        if properties_to_create:
+            properties_to_create.sort(key=lambda x: x[0])
+            new_id = max(self.categories.values(), default=0) + 1
+
+            for product_property, parent_id in properties_to_create:
+                new_property = await self.create_product_category(
+                    table_id=self.categories_table,
+                    property_name=product_property,
+                    category_id=new_id,
+                    table_name=PRODUCT_NAME_FIELD,
+                )
+                if self.logging:
+                    logger.info(f"New Category: {new_property}")
+                new_id += 1
+
+    async def link_categories(self, parent_id: int, child_id: int):
+        metadata = await self.get_table_meta(self.categories_table)
+        linked_column = next((col for col in metadata['columns'] if col["title"] == "Set parent category" and col["uidt"] == "Links"), None)
+
+        if linked_column:
+            await self.link_table_record(
+                linked_column["base_id"],
+                linked_column["fk_model_id"],
+                child_id,
+                linked_column["id"],
+                parent_id,
+            )
+
+    async def unlink_categories(self, parent_id: int, child_id: int):
+        metadata = await self.get_table_meta(self.categories_table)
+        linked_column = next((col for col in metadata['columns'] if col["uidt"] == "Links"), None)
+
+        if linked_column:
+            await self.unlink_table_record(
+                linked_column["base_id"],
+                linked_column["fk_model_id"],
+                parent_id,
+                linked_column["id"],
+                child_id,
+            )
+
+    async def map_categories(self, external_products: List[ProductModel]) -> List[ProductModel]:
+        for num, product in enumerate(external_products):
+            if not product.category and product.product_properties:
+                external_products[num].category = [
+                    str(self.categories[property_name]) for property_name in product.product_properties
+                ]
+        return external_products