amochka 0.4.7__py3-none-any.whl → 0.4.8__py3-none-any.whl

amochka/__init__.py

@@ -2,7 +2,7 @@
 amochka: Библиотека для работы с API amoCRM.
 """
 
-__version__ = "0.4.7"
+__version__ = "0.4.8"
 
 from .client import AmoCRMClient, CacheConfig
 from .errors import (

{amochka-0.4.7.dist-info → amochka-0.4.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: amochka
-Version: 0.4.7
+Version: 0.4.8
 Summary: Python library for working with amoCRM API with ETL capabilities
 Author-email: Timur <timurdt@gmail.com>
 License: MIT

{amochka-0.4.7.dist-info → amochka-0.4.8.dist-info}/RECORD

@@ -1,15 +1,15 @@
-amochka/__init__.py,sha256=1IrHrHoIJWMJgXlmxVzX93HXxTgDL7agZRR7GeP6AiY,925
+amochka/__init__.py,sha256=yvD3BuCLtzagE2oTYt36fPKGYpcImv2mY590XqBsXwI,925
 amochka/client.py,sha256=zQUy6DhoQfLDnO9q7CxXd28cjLQmwE15MtTGsGYYVSs,91571
 amochka/errors.py,sha256=Rg4U9srjboQwgU3wwhAed0p3vGcc0ZhuyKHIDhYFnp8,1371
 amochka/etl.py,sha256=tzdGPRGxR49aSAjLksic-FqechvDUTfK8ZUUJGuIU7M,8960
 etl/__init__.py,sha256=bp9fPqbKlOc7xzs27diHEvysy1FgBrwlpX6GnR6GL9U,255
-etl/config.py,sha256=BvaGn5BSGMIfvUNNsnap04iy3BHyMOuRX81G7EiLUfE,9032
+etl/config.py,sha256=eBQ5U9Kulyho-XMu8gRSJ47WrnONiiWyL2l3GtZhrNs,9416
 etl/extractors.py,sha256=PqjzlmUa8FLZa1z85mP6Y0-s_TH3REqW58632JzRKUc,13047
 etl/loaders.py,sha256=AA4XDCbre-Y8SrIji5Sk8olW9WvREM9CMuuknFVe5j0,32589
-etl/run_etl.py,sha256=6ieidJiiXOILI1hzFzCCyVjUHX6wCJ-D6556NUqKOxw,26847
-etl/transformers.py,sha256=OwYJ_9l3oqvy2Y3-umXjAGweOIqlfRI0iSiCFPrcQ8E,17867
+etl/run_etl.py,sha256=wCpCs2vL9cxucxHjcpPQydja9JzFxaRlfc9CwpS9OGU,27270
+etl/transformers.py,sha256=ML9iBtLRh8OgFtSK-1ngmRw6vJu_GXHCbln1U8d7sHc,18150
 etl/migrations/001_create_tables.sql,sha256=YrSaZjpofC1smjYx0bM4eHQumboruIBY3fwRDlJLLSo,15749
-amochka-0.4.7.dist-info/METADATA,sha256=W_0rYdYd14YBxn0hp625NmZJtVzPVOY-ERR9hdKsrBQ,7530
-amochka-0.4.7.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-amochka-0.4.7.dist-info/top_level.txt,sha256=grRX8aLFG-yYKPsAqCD6sUBmdLSQeOMHsc9Dl6S7Lzo,12
-amochka-0.4.7.dist-info/RECORD,,
+amochka-0.4.8.dist-info/METADATA,sha256=gB8ZgvtYOkXy6wmxyQbV6gxhs8eHOx9H5-XTV9yrYao,7530
+amochka-0.4.8.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+amochka-0.4.8.dist-info/top_level.txt,sha256=grRX8aLFG-yYKPsAqCD6sUBmdLSQeOMHsc9Dl6S7Lzo,12
+amochka-0.4.8.dist-info/RECORD,,

etl/config.py

@@ -32,6 +32,12 @@ def _load_env_file(path: Path) -> Dict[str, str]:
     return env
 
 
+def _get_bool(value: Optional[str], default: bool = False) -> bool:
+    if value is None:
+        return default
+    return value.strip().lower() in {"1", "true", "yes", "y", "on"}
+
+
 @dataclass
 class DatabaseConfig:
     """Конфигурация подключения к PostgreSQL."""
@@ -139,6 +145,7 @@ class ETLConfig:
     batch_size: int = 100
     window_minutes: int = 120  # Окно выгрузки по умолчанию (2 часа)
     log_level: str = "INFO"
+    include_companies: bool = False
 
     @classmethod
     def from_env(cls, env_path: Optional[Path] = None) -> "ETLConfig":
@@ -180,12 +187,15 @@ class ETLConfig:
                 )
             )
 
+        include_companies = _get_bool(os.environ.get("ETL_INCLUDE_COMPANIES"), False)
+
         return cls(
             database=db_config,
             accounts=accounts,
             batch_size=int(os.environ.get("ETL_BATCH_SIZE", "100") or "100"),
             window_minutes=int(os.environ.get("ETL_WINDOW_MINUTES", "120") or "120"),
             log_level=os.environ.get("ETL_LOG_LEVEL", "INFO") or "INFO",
+            include_companies=include_companies,
         )
 
 
@@ -213,6 +223,7 @@ DEFAULT_CONFIG = ETLConfig(
     ],
     batch_size=100,
     window_minutes=120,
+    include_companies=False,
 )
 
 

etl/run_etl.py

@@ -88,6 +88,7 @@ def sync_leads_with_contacts(
     updated_from: Optional[datetime] = None,
     updated_to: Optional[datetime] = None,
     pipeline_ids: Optional[List[int]] = None,
+    include_companies: bool = False,
     batch_size: int = 100,
 ) -> dict:
     """
@@ -101,11 +102,16 @@ def sync_leads_with_contacts(
     Returns:
         dict с ключами: leads_count, contacts_count
     """
-    contact_transformer = ContactTransformer(mybi_account_id)
+    contact_transformer = ContactTransformer(mybi_account_id, include_companies=include_companies)
 
     # Загружаем справочники для денормализации
     pipelines_map, statuses_map = extractor.load_pipelines_and_statuses()
-    lead_transformer = LeadTransformer(mybi_account_id, pipelines_map, statuses_map)
+    lead_transformer = LeadTransformer(
+        mybi_account_id,
+        pipelines_map,
+        statuses_map,
+        include_companies=include_companies,
+    )
 
     # 1. Собираем сделки и ID контактов из API
     leads_iter = extractor.iter_leads(
@@ -261,6 +267,7 @@ def sync_contacts(
     contact_ids: Optional[Set[int]] = None,
     updated_from: Optional[datetime] = None,
     updated_to: Optional[datetime] = None,
+    include_companies: bool = False,
     batch_size: int = 100,
 ) -> int:
     """
@@ -279,7 +286,7 @@ def sync_contacts(
     Returns:
         Количество загруженных контактов
     """
-    contact_transformer = ContactTransformer(mybi_account_id)
+    contact_transformer = ContactTransformer(mybi_account_id, include_companies=include_companies)
 
     # Загружаем все обновлённые контакты за период
     contacts_iter = extractor.iter_contacts(
@@ -484,6 +491,7 @@ def run_etl_for_account(
     entities: List[str],
     window_minutes: int,
     full_sync: bool = False,
+    include_companies: bool = False,
     batch_size: int = 100,
 ) -> dict:
     """
@@ -537,6 +545,7 @@ def run_etl_for_account(
             updated_from=leads_updated_from,
             updated_to=updated_to,
             pipeline_ids=account.pipeline_ids,
+            include_companies=include_companies,
             batch_size=batch_size,
         )
         stats["leads"] = result["leads_count"]
@@ -578,6 +587,7 @@ def run_etl_for_account(
                 contact_ids=contact_ids_in_pipelines,  # Только контакты из наших воронок
                 updated_from=contacts_updated_from,
                 updated_to=updated_to,
+                include_companies=include_companies,
                 batch_size=batch_size,
             )
         else:
@@ -677,6 +687,7 @@ def main():
                 entities,
                 config.window_minutes,
                 full_sync=args.full,
+                include_companies=config.include_companies,
                 batch_size=config.batch_size,
             )
             all_stats[account.name] = stats

etl/transformers.py

@@ -78,7 +78,13 @@ class TransformedEvent:
 class LeadTransformer:
     """Трансформер для сделок (leads)."""
 
-    def __init__(self, account_id: int, pipelines_map: Optional[Dict[int, str]] = None, statuses_map: Optional[Dict[int, Dict]] = None):
+    def __init__(
+        self,
+        account_id: int,
+        pipelines_map: Optional[Dict[int, str]] = None,
+        statuses_map: Optional[Dict[int, Dict]] = None,
+        include_companies: bool = False,
+    ):
         """
         Инициализирует трансформер.
 
@@ -90,6 +96,7 @@ class LeadTransformer:
         self.account_id = account_id
         self.pipelines_map = pipelines_map or {}
         self.statuses_map = statuses_map or {}
+        self.include_companies = include_companies
 
     def transform(self, lead: Dict[str, Any]) -> TransformedLead:
         """
@@ -145,7 +152,7 @@ class LeadTransformer:
             main_contact_id = contacts[0].get("id")
 
         # Компания из _embedded.companies
-        companies = embedded.get("companies", [])
+        companies = embedded.get("companies", []) if self.include_companies else []
         company_id = companies[0].get("id") if companies else None
 
         lead_facts_record = {
@@ -221,8 +228,9 @@ class LeadTransformer:
 class ContactTransformer:
     """Трансформер для контактов."""
 
-    def __init__(self, account_id: int):
+    def __init__(self, account_id: int, include_companies: bool = False):
         self.account_id = account_id
+        self.include_companies = include_companies
 
     def transform(self, contact: Dict[str, Any]) -> TransformedContact:
         """Преобразует контакт из amoCRM в структуру mybi."""
@@ -249,7 +257,7 @@ class ContactTransformer:
 
         # Компания из _embedded
         embedded = contact.get("_embedded", {})
-        companies = embedded.get("companies", [])
+        companies = embedded.get("companies", []) if self.include_companies else []
         if companies:
             contact_record["company"] = companies[0].get("name")