amochka 0.4.6__tar.gz → 0.4.8__tar.gz

{amochka-0.4.6 → amochka-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: amochka
-Version: 0.4.6
+Version: 0.4.8
 Summary: Python library for working with amoCRM API with ETL capabilities
 Author-email: Timur <timurdt@gmail.com>
 License: MIT

{amochka-0.4.6 → amochka-0.4.8}/amochka/__init__.py

@@ -2,7 +2,7 @@
 amochka: Библиотека для работы с API amoCRM.
 """
 
-__version__ = "0.4.6"
+__version__ = "0.4.8"
 
 from .client import AmoCRMClient, CacheConfig
 from .errors import (

{amochka-0.4.6 → amochka-0.4.8}/amochka.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: amochka
-Version: 0.4.6
+Version: 0.4.8
 Summary: Python library for working with amoCRM API with ETL capabilities
 Author-email: Timur <timurdt@gmail.com>
 License: MIT

{amochka-0.4.6 → amochka-0.4.8}/etl/config.py

@@ -32,6 +32,12 @@ def _load_env_file(path: Path) -> Dict[str, str]:
     return env
 
 
+def _get_bool(value: Optional[str], default: bool = False) -> bool:
+    if value is None:
+        return default
+    return value.strip().lower() in {"1", "true", "yes", "y", "on"}
+
+
 @dataclass
 class DatabaseConfig:
     """Конфигурация подключения к PostgreSQL."""
@@ -139,6 +145,7 @@ class ETLConfig:
     batch_size: int = 100
     window_minutes: int = 120  # Окно выгрузки по умолчанию (2 часа)
     log_level: str = "INFO"
+    include_companies: bool = False
 
     @classmethod
     def from_env(cls, env_path: Optional[Path] = None) -> "ETLConfig":
@@ -180,12 +187,15 @@ class ETLConfig:
                 )
             )
 
+        include_companies = _get_bool(os.environ.get("ETL_INCLUDE_COMPANIES"), False)
+
         return cls(
             database=db_config,
             accounts=accounts,
             batch_size=int(os.environ.get("ETL_BATCH_SIZE", "100") or "100"),
             window_minutes=int(os.environ.get("ETL_WINDOW_MINUTES", "120") or "120"),
             log_level=os.environ.get("ETL_LOG_LEVEL", "INFO") or "INFO",
+            include_companies=include_companies,
         )
 
 
@@ -213,6 +223,7 @@ DEFAULT_CONFIG = ETLConfig(
     ],
     batch_size=100,
     window_minutes=120,
+    include_companies=False,
 )
 
 

{amochka-0.4.6 → amochka-0.4.8}/etl/loaders.py

@@ -415,12 +415,26 @@ class PostgresLoader:
         result = cursor.fetchone()
         return result[0] if result else None
 
-    def load_transformed_contact(self, cursor, transformed: TransformedContact) -> int:
+    def load_transformed_contact(
+        self,
+        cursor,
+        transformed: TransformedContact,
+        user_id_map: Optional[Dict[int, int]] = None,
+    ) -> int:
         """Загружает полностью трансформированный контакт."""
         contacts_id = self.upsert_contact(cursor, transformed.contact)
 
         # Факты по контакту
         transformed.contact_facts["contacts_id"] = contacts_id
+
+        # Преобразуем users_id из amoCRM ID во внутренний ID (если найден).
+        # Если пользователя нет в маппинге — оставляем внешний ID как есть.
+        if user_id_map:
+            amo_user_id = transformed.contact_facts.get("users_id")
+            if amo_user_id is not None:
+                internal_user_id = user_id_map.get(amo_user_id)
+                if internal_user_id is not None:
+                    transformed.contact_facts["users_id"] = internal_user_id
         registered_id = self._get_or_create_date_id(cursor, transformed.contact_facts.get("created_date"))
 
         cursor.execute(

{amochka-0.4.6 → amochka-0.4.8}/etl/run_etl.py

@@ -88,6 +88,7 @@ def sync_leads_with_contacts(
     updated_from: Optional[datetime] = None,
     updated_to: Optional[datetime] = None,
     pipeline_ids: Optional[List[int]] = None,
+    include_companies: bool = False,
     batch_size: int = 100,
 ) -> dict:
     """
@@ -101,11 +102,16 @@ def sync_leads_with_contacts(
     Returns:
         dict с ключами: leads_count, contacts_count
     """
-    contact_transformer = ContactTransformer(mybi_account_id)
+    contact_transformer = ContactTransformer(mybi_account_id, include_companies=include_companies)
 
     # Загружаем справочники для денормализации
     pipelines_map, statuses_map = extractor.load_pipelines_and_statuses()
-    lead_transformer = LeadTransformer(mybi_account_id, pipelines_map, statuses_map)
+    lead_transformer = LeadTransformer(
+        mybi_account_id,
+        pipelines_map,
+        statuses_map,
+        include_companies=include_companies,
+    )
 
     # 1. Собираем сделки и ID контактов из API
     leads_iter = extractor.iter_leads(
@@ -134,6 +140,7 @@ def sync_leads_with_contacts(
             with conn.cursor() as cursor:
                 # Проверяем какие контакты уже есть в БД
                 existing_contacts = loader.build_contact_id_map(cursor, mybi_account_id)
+                user_id_map = loader.build_user_id_map(cursor, mybi_account_id)
                 missing_contact_ids = contact_ids - set(existing_contacts.keys())
 
                 if missing_contact_ids:
@@ -141,7 +148,7 @@ def sync_leads_with_contacts(
 
                     for contact in extractor.iter_contacts(contact_ids=list(missing_contact_ids)):
                         transformed = contact_transformer.transform(contact)
-                        loader.load_transformed_contact(cursor, transformed)
+                        loader.load_transformed_contact(cursor, transformed, user_id_map)
                         contacts_loaded += 1
 
                         if contacts_loaded % batch_size == 0:
@@ -260,6 +267,7 @@ def sync_contacts(
     contact_ids: Optional[Set[int]] = None,
     updated_from: Optional[datetime] = None,
     updated_to: Optional[datetime] = None,
+    include_companies: bool = False,
     batch_size: int = 100,
 ) -> int:
     """
@@ -278,7 +286,7 @@ def sync_contacts(
     Returns:
         Количество загруженных контактов
     """
-    contact_transformer = ContactTransformer(mybi_account_id)
+    contact_transformer = ContactTransformer(mybi_account_id, include_companies=include_companies)
 
     # Загружаем все обновлённые контакты за период
     contacts_iter = extractor.iter_contacts(
@@ -292,6 +300,7 @@ def sync_contacts(
 
     with loader.connection() as conn:
         with conn.cursor() as cursor:
+            user_id_map = loader.build_user_id_map(cursor, mybi_account_id)
             for i, contact in enumerate(contacts_iter):
                 contact_id = contact.get("id")
 
@@ -301,7 +310,7 @@ def sync_contacts(
                     continue
 
                 transformed = contact_transformer.transform(contact)
-                loader.load_transformed_contact(cursor, transformed)
+                loader.load_transformed_contact(cursor, transformed, user_id_map)
                 loaded_count += 1
 
                 # Отслеживаем максимальный updated_at
@@ -482,6 +491,7 @@ def run_etl_for_account(
     entities: List[str],
     window_minutes: int,
     full_sync: bool = False,
+    include_companies: bool = False,
     batch_size: int = 100,
 ) -> dict:
     """
@@ -535,6 +545,7 @@ def run_etl_for_account(
             updated_from=leads_updated_from,
             updated_to=updated_to,
             pipeline_ids=account.pipeline_ids,
+            include_companies=include_companies,
             batch_size=batch_size,
         )
         stats["leads"] = result["leads_count"]
@@ -576,6 +587,7 @@ def run_etl_for_account(
                 contact_ids=contact_ids_in_pipelines,  # Только контакты из наших воронок
                 updated_from=contacts_updated_from,
                 updated_to=updated_to,
+                include_companies=include_companies,
                 batch_size=batch_size,
             )
         else:
@@ -675,6 +687,7 @@ def main():
                 entities,
                 config.window_minutes,
                 full_sync=args.full,
+                include_companies=config.include_companies,
                 batch_size=config.batch_size,
             )
             all_stats[account.name] = stats

{amochka-0.4.6 → amochka-0.4.8}/etl/transformers.py

@@ -78,7 +78,13 @@ class TransformedEvent:
 class LeadTransformer:
     """Трансформер для сделок (leads)."""
 
-    def __init__(self, account_id: int, pipelines_map: Optional[Dict[int, str]] = None, statuses_map: Optional[Dict[int, Dict]] = None):
+    def __init__(
+        self,
+        account_id: int,
+        pipelines_map: Optional[Dict[int, str]] = None,
+        statuses_map: Optional[Dict[int, Dict]] = None,
+        include_companies: bool = False,
+    ):
         """
         Инициализирует трансформер.
 
@@ -90,6 +96,7 @@ class LeadTransformer:
         self.account_id = account_id
         self.pipelines_map = pipelines_map or {}
         self.statuses_map = statuses_map or {}
+        self.include_companies = include_companies
 
     def transform(self, lead: Dict[str, Any]) -> TransformedLead:
         """
@@ -145,7 +152,7 @@ class LeadTransformer:
             main_contact_id = contacts[0].get("id")
 
         # Компания из _embedded.companies
-        companies = embedded.get("companies", [])
+        companies = embedded.get("companies", []) if self.include_companies else []
         company_id = companies[0].get("id") if companies else None
 
         lead_facts_record = {
@@ -221,8 +228,9 @@ class LeadTransformer:
 class ContactTransformer:
     """Трансформер для контактов."""
 
-    def __init__(self, account_id: int):
+    def __init__(self, account_id: int, include_companies: bool = False):
         self.account_id = account_id
+        self.include_companies = include_companies
 
     def transform(self, contact: Dict[str, Any]) -> TransformedContact:
         """Преобразует контакт из amoCRM в структуру mybi."""
@@ -249,7 +257,7 @@ class ContactTransformer:
 
         # Компания из _embedded
         embedded = contact.get("_embedded", {})
-        companies = embedded.get("companies", [])
+        companies = embedded.get("companies", []) if self.include_companies else []
         if companies:
             contact_record["company"] = companies[0].get("name")
 

{amochka-0.4.6 → amochka-0.4.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "amochka"
-version = "0.4.6"
+version = "0.4.8"
 description = "Python library for working with amoCRM API with ETL capabilities"
 readme = "README.md"
 authors = [