amochka 0.1.9__py3-none-any.whl → 0.3.0__py3-none-any.whl

amochka/etl.py

@@ -0,0 +1,302 @@
+import json
+from pathlib import Path
+from typing import Callable, Iterable, List, Optional, Sequence, Set, Union
+
+from .client import AmoCRMClient
+
+
+def _ensure_path(path: Union[str, Path]) -> Path:
+    output_path = Path(path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    return output_path
+
+
+def _resolve_timestamp(record: dict, timestamp_fields: Sequence[str]) -> Optional[Union[int, float, str]]:
+    for field in timestamp_fields:
+        if not field:
+            continue
+        value = record.get(field)
+        if value is not None:
+            return value
+    return None
+
+
+def write_ndjson(
+    records: Iterable[dict],
+    output_path: Union[str, Path],
+    *,
+    entity: str,
+    account_id: Optional[Union[int, str]] = None,
+    timestamp_fields: Sequence[str] = ("updated_at", "created_at"),
+    transform: Optional[Callable[[dict], dict]] = None,
+    on_record: Optional[Callable[[dict], None]] = None,
+) -> int:
+    """
+    Записывает переданные записи в формат NDJSON.
+
+    Возвращает количество записанных строк.
+    """
+    path = _ensure_path(output_path)
+    count = 0
+    with path.open("w", encoding="utf-8") as handler:
+        for original in records:
+            payload = transform(original) if transform else original
+            timestamp = _resolve_timestamp(original, timestamp_fields)
+            line = {
+                "entity": entity,
+                "account_id": account_id,
+                "updated_at": timestamp,
+                "payload": payload,
+            }
+            handler.write(json.dumps(line, ensure_ascii=False))
+            handler.write("\n")
+            count += 1
+            if on_record:
+                on_record(original)
+    return count
+
+
+def export_leads_to_ndjson(
+    client: AmoCRMClient,
+    output_path: Union[str, Path],
+    account_id: Union[int, str],
+    *,
+    start=None,
+    end=None,
+    pipeline_ids=None,
+    include_contacts: bool = True,
+    include=None,
+    limit: int = 250,
+    extra_params: Optional[dict] = None,
+    on_record: Optional[Callable[[dict], None]] = None,
+) -> int:
+    """
+    Выгружает сделки и записывает их в NDJSON.
+    """
+    records = client.iter_leads(
+        updated_from=start,
+        updated_to=end,
+        pipeline_ids=pipeline_ids,
+        include_contacts=include_contacts,
+        include=include,
+        limit=limit,
+        extra_params=extra_params,
+    )
+    return write_ndjson(
+        records,
+        output_path,
+        entity="lead",
+        account_id=account_id,
+        timestamp_fields=("updated_at", "created_at"),
+        on_record=on_record,
+    )
+
+
+def export_contacts_to_ndjson(
+    client: AmoCRMClient,
+    output_path: Union[str, Path],
+    account_id: Union[int, str],
+    *,
+    start=None,
+    end=None,
+    contact_ids=None,
+    limit: int = 250,
+    extra_params: Optional[dict] = None,
+    on_record: Optional[Callable[[dict], None]] = None,
+) -> int:
+    """
+    Выгружает контакты и записывает их в NDJSON.
+    """
+    contact_id_list: Optional[List[int]] = None
+    if contact_ids is not None:
+        if isinstance(contact_ids, (list, tuple, set)):
+            contact_id_list = [int(cid) for cid in contact_ids if cid is not None]
+        else:
+            contact_id_list = [int(contact_ids)]
+
+    def _iter_contacts():
+        seen: Set[int] = set()
+        if contact_id_list:
+            params = dict(extra_params or {})
+            params["filter[id][]"] = [str(cid) for cid in contact_id_list]
+            params["page"] = 1
+            params["limit"] = limit
+            while True:
+                response = client._make_request("GET", "/api/v4/contacts", params=params)
+                embedded = (response or {}).get("_embedded", {})
+                contacts = embedded.get("contacts") or []
+                if not contacts:
+                    break
+                for contact in contacts:
+                    cid = contact.get("id")
+                    if cid is not None:
+                        seen.add(int(cid))
+                    yield contact
+                total_pages = response.get("_page_count", params["page"])
+                if params["page"] >= total_pages:
+                    break
+                params["page"] += 1
+        else:
+            for contact in client.iter_contacts(
+                updated_from=start,
+                updated_to=end,
+                contact_ids=None,
+                limit=limit,
+                extra_params=extra_params,
+            ):
+                cid = contact.get("id")
+                if cid is not None:
+                    seen.add(int(cid))
+                yield contact
+
+        if contact_id_list:
+            missing = [cid for cid in contact_id_list if cid not in seen]
+            for cid in missing:
+                try:
+                    contact = client.get_contact_by_id(cid)
+                except Exception:
+                    continue
+                retrieved_id = contact.get("id")
+                if retrieved_id is not None and int(retrieved_id) not in seen:
+                    seen.add(int(retrieved_id))
+                    yield contact
+
+    return write_ndjson(
+        _iter_contacts(),
+        output_path,
+        entity="contact",
+        account_id=account_id,
+        timestamp_fields=("updated_at", "created_at"),
+        on_record=on_record,
+    )
+
+
+def export_notes_to_ndjson(
+    client: AmoCRMClient,
+    output_path: Union[str, Path],
+    account_id: Union[int, str],
+    *,
+    entity: str = "lead",
+    start=None,
+    end=None,
+    note_type=None,
+    entity_ids=None,
+    limit: int = 250,
+    extra_params: Optional[dict] = None,
+    on_record: Optional[Callable[[dict], None]] = None,
+) -> int:
+    """
+    Выгружает примечания и записывает их в NDJSON.
+    """
+    records = client.iter_notes(
+        entity=entity,
+        updated_from=start,
+        updated_to=end,
+        note_type=note_type,
+        entity_ids=entity_ids,
+        limit=limit,
+        extra_params=extra_params,
+    )
+    entity_name = f"{entity}_note" if entity else "note"
+    return write_ndjson(
+        records,
+        output_path,
+        entity=entity_name,
+        account_id=account_id,
+        timestamp_fields=("updated_at", "created_at"),
+        on_record=on_record,
+    )
+
+
+def export_events_to_ndjson(
+    client: AmoCRMClient,
+    output_path: Union[str, Path],
+    account_id: Union[int, str],
+    *,
+    entity: Optional[str] = "lead",
+    start=None,
+    end=None,
+    event_type=None,
+    entity_ids=None,
+    limit: int = 250,
+    extra_params: Optional[dict] = None,
+    on_record: Optional[Callable[[dict], None]] = None,
+) -> int:
+    """
+    Выгружает события и записывает их в NDJSON.
+    """
+    records = client.iter_events(
+        entity=entity,
+        entity_ids=entity_ids,
+        event_type=event_type,
+        created_from=start,
+        created_to=end,
+        limit=limit,
+        extra_params=extra_params,
+    )
+    entity_name = f"{entity}_event" if entity else "event"
+    return write_ndjson(
+        records,
+        output_path,
+        entity=entity_name,
+        account_id=account_id,
+        timestamp_fields=("created_at", "updated_at"),
+        on_record=on_record,
+    )
+
+
+def export_users_to_ndjson(
+    client: AmoCRMClient,
+    output_path: Union[str, Path],
+    account_id: Union[int, str],
+    *,
+    limit: int = 250,
+    extra_params: Optional[dict] = None,
+    on_record: Optional[Callable[[dict], None]] = None,
+) -> int:
+    """
+    Выгружает пользователей и записывает их в NDJSON.
+    """
+    records = client.iter_users(limit=limit, extra_params=extra_params)
+    return write_ndjson(
+        records,
+        output_path,
+        entity="user",
+        account_id=account_id,
+        timestamp_fields=("updated_at", "created_at"),
+        on_record=on_record,
+    )
+
+
+def export_pipelines_to_ndjson(
+    client: AmoCRMClient,
+    output_path: Union[str, Path],
+    account_id: Union[int, str],
+    *,
+    limit: int = 250,
+    extra_params: Optional[dict] = None,
+    on_record: Optional[Callable[[dict], None]] = None,
+) -> int:
+    """
+    Выгружает воронки и записывает их в NDJSON.
+    """
+    records = client.iter_pipelines(limit=limit, extra_params=extra_params)
+    return write_ndjson(
+        records,
+        output_path,
+        entity="pipeline",
+        account_id=account_id,
+        timestamp_fields=("updated_at", "created_at"),
+        on_record=on_record,
+    )
+
+
+__all__ = [
+    "write_ndjson",
+    "export_leads_to_ndjson",
+    "export_contacts_to_ndjson",
+    "export_notes_to_ndjson",
+    "export_events_to_ndjson",
+    "export_users_to_ndjson",
+    "export_pipelines_to_ndjson",
+]

{amochka-0.1.9.dist-info → amochka-0.3.0.dist-info}/METADATA

@@ -1,14 +1,14 @@
 Metadata-Version: 2.4
 Name: amochka
-Version: 0.1.9
-Summary: Python library for working with amoCRM API
+Version: 0.3.0
+Summary: Python library for working with amoCRM API with ETL capabilities
 Author-email: Timur <timurdt@gmail.com>
 License: MIT
 Project-URL: Homepage, https://github.com/yourusername/amochka
 Project-URL: Documentation, https://github.com/yourusername/amochka
 Project-URL: Repository, https://github.com/yourusername/amochka
 Project-URL: Bug Tracker, https://github.com/yourusername/amochka/issues
-Keywords: amocrm,crm,api,client,automation
+Keywords: amocrm,crm,api,client,automation,etl
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -27,6 +27,8 @@ Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 Requires-Dist: requests>=2.25.0
 Requires-Dist: ratelimit>=2.2.0
+Requires-Dist: psycopg2-binary>=2.9.0
+Requires-Dist: python-dotenv>=1.0.0
 
 # amochka
 

amochka-0.3.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+amochka/__init__.py,sha256=NFAgMbhBnrx3nF--MeY9Chpu5gtZ5kVn-QYnmO3Nhpk,620
+amochka/client.py,sha256=mGoDEE0XOt-c9f2FmqFVzDMrXQel_zComtyAFXdCHjg,66155
+amochka/etl.py,sha256=N8rXNFbtmlKfsYpgr7HDcP4enoj63XQPWuTDxGuMhw4,8901
+etl/__init__.py,sha256=bp9fPqbKlOc7xzs27diHEvysy1FgBrwlpX6GnR6GL9U,255
+etl/config.py,sha256=YY6M7pib_XD7bjPW8J_iWzt2hoXzDpx6yIvZHmlUlrE,8873
+etl/extractors.py,sha256=-QCBZ6PoJ51j0drNQaH5bLfvjPqAQmfVgaT1D_ZSwjI,12909
+etl/loaders.py,sha256=nQx6TDwnuHVWzjmuKY9v-23hmmk4Ex6ZoGtb3PApO1k,31974
+etl/run_etl.py,sha256=p_2NxJwXMiACMETvRsjrozMgz66U9ezDNSWZXUieNMs,26262
+etl/transformers.py,sha256=OwYJ_9l3oqvy2Y3-umXjAGweOIqlfRI0iSiCFPrcQ8E,17867
+etl/migrations/001_create_tables.sql,sha256=YrSaZjpofC1smjYx0bM4eHQumboruIBY3fwRDlJLLSo,15749
+amochka-0.3.0.dist-info/METADATA,sha256=5dZFPOs2wupzc0VsYVXyk1pJ-F4Kwm997N9EA_Z7htE,6371
+amochka-0.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+amochka-0.3.0.dist-info/top_level.txt,sha256=grRX8aLFG-yYKPsAqCD6sUBmdLSQeOMHsc9Dl6S7Lzo,12
+amochka-0.3.0.dist-info/RECORD,,

amochka-0.3.0.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+amochka
+etl

etl/__init__.py

@@ -0,0 +1,7 @@
+"""
+ETL-коннектор для выгрузки данных из amoCRM в PostgreSQL.
+
+Структура таблиц совместима с mybi.ru для работы с существующими dbt-моделями.
+"""
+
+__version__ = "0.1.0"

etl/config.py

@@ -0,0 +1,236 @@
+"""
+Конфигурация ETL для amoCRM.
+
+Настройки загружаются из переменных окружения или .env файла.
+"""
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+
+def _load_env_file(path: Path) -> Dict[str, str]:
+    """Загружает переменные из .env файла."""
+    if not path.exists():
+        return {}
+
+    env: Dict[str, str] = {}
+    with path.open("r", encoding="utf-8") as handler:
+        for raw_line in handler:
+            line = raw_line.strip()
+            if not line or line.startswith("#"):
+                continue
+            if "=" not in line:
+                continue
+            key, value = line.split("=", 1)
+            key = key.strip()
+            value = value.strip()
+            if value.startswith(('"', "'")) and value.endswith(('"', "'")) and len(value) >= 2:
+                value = value[1:-1]
+            env[key] = value
+    return env
+
+
+@dataclass
+class DatabaseConfig:
+    """Конфигурация подключения к PostgreSQL."""
+
+    host: str
+    port: int
+    dbname: str
+    user: str
+    password: str
+    schema: str = "public"
+    sslmode: Optional[str] = None
+    connect_timeout: int = 30
+
+    def connection_kwargs(self) -> Dict[str, Any]:
+        """Возвращает kwargs для psycopg.connect()."""
+        kwargs: Dict[str, Any] = {
+            "host": self.host,
+            "port": self.port,
+            "dbname": self.dbname,
+            "user": self.user,
+            "password": self.password,
+            "connect_timeout": self.connect_timeout,
+        }
+        if self.sslmode:
+            kwargs["sslmode"] = self.sslmode
+        return kwargs
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> "DatabaseConfig":
+        """Создаёт конфигурацию из словаря (для Airflow DAG)."""
+        return cls(
+            host=d.get("host", "localhost"),
+            port=int(d.get("port", 5432)),
+            dbname=d.get("dbname", "amocrm"),
+            user=d.get("user", "postgres"),
+            password=d.get("password", ""),
+            schema=d.get("schema", "public"),
+            sslmode=d.get("sslmode"),
+            connect_timeout=int(d.get("connect_timeout", 30)),
+        )
+
+    @classmethod
+    def from_env(cls, env_path: Optional[Path] = None) -> "DatabaseConfig":
+        """Создаёт конфигурацию из переменных окружения."""
+        if env_path:
+            file_env = _load_env_file(env_path)
+            for key, value in file_env.items():
+                os.environ.setdefault(key, value)
+
+        def _get(key: str, default: Optional[str] = None) -> Optional[str]:
+            return os.environ.get(key, default)
+
+        return cls(
+            host=_get("ETL_DB_HOST", "localhost") or "localhost",
+            port=int(_get("ETL_DB_PORT", "5432") or "5432"),
+            dbname=_get("ETL_DB_NAME", "amocrm") or "amocrm",
+            user=_get("ETL_DB_USER", "postgres") or "postgres",
+            password=_get("ETL_DB_PASSWORD", "") or "",
+            schema=_get("ETL_DB_SCHEMA", "public") or "public",
+            sslmode=_get("ETL_DB_SSLMODE"),
+            connect_timeout=int(_get("ETL_DB_CONNECT_TIMEOUT", "30") or "30"),
+        )
+
+
+@dataclass
+class AmoCRMAccount:
+    """Конфигурация одного аккаунта amoCRM."""
+
+    id: int  # ID аккаунта в amoCRM (из URL или API)
+    name: str
+    base_url: str
+    token_path: Path
+    mybi_account_id: int  # Внутренний account_id как в mybi.ru (для совместимости)
+    pipeline_ids: Optional[List[int]] = None  # None = все воронки
+    cache_dir: Optional[Path] = None
+
+    def __post_init__(self):
+        if isinstance(self.token_path, str):
+            self.token_path = Path(self.token_path)
+        if isinstance(self.cache_dir, str):
+            self.cache_dir = Path(self.cache_dir)
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> "AmoCRMAccount":
+        """Создаёт конфигурацию аккаунта из словаря (для Airflow DAG)."""
+        return cls(
+            id=int(d.get("id", 0)),
+            name=d.get("name", "account"),
+            base_url=d.get("base_url", ""),
+            token_path=Path(d.get("token_path", "token.json")),
+            mybi_account_id=int(d.get("mybi_account_id", 0)),
+            pipeline_ids=d.get("pipeline_ids"),
+            cache_dir=Path(d.get("cache_dir", ".cache")) if d.get("cache_dir") else None,
+        )
+
+
+@dataclass
+class ETLConfig:
+    """Главная конфигурация ETL."""
+
+    database: DatabaseConfig
+    accounts: List[AmoCRMAccount]
+    batch_size: int = 100
+    window_minutes: int = 120  # Окно выгрузки по умолчанию (2 часа)
+    log_level: str = "INFO"
+
+    @classmethod
+    def from_env(cls, env_path: Optional[Path] = None) -> "ETLConfig":
+        """Создаёт конфигурацию из переменных окружения и файла настроек."""
+        if env_path:
+            file_env = _load_env_file(env_path)
+            for key, value in file_env.items():
+                os.environ.setdefault(key, value)
+
+        db_config = DatabaseConfig.from_env(env_path)
+
+        # Парсим аккаунты из переменных окружения
+        # Формат: AMO_ACCOUNT_1_ID, AMO_ACCOUNT_1_NAME, AMO_ACCOUNT_1_URL, etc.
+        accounts = []
+        for i in range(1, 10):  # Поддерживаем до 9 аккаунтов
+            prefix = f"AMO_ACCOUNT_{i}_"
+            account_id = os.environ.get(f"{prefix}ID")
+            if not account_id:
+                continue
+
+            pipeline_ids_str = os.environ.get(f"{prefix}PIPELINE_IDS", "")
+            pipeline_ids = None
+            if pipeline_ids_str:
+                pipeline_ids = [int(pid.strip()) for pid in pipeline_ids_str.split(",") if pid.strip()]
+
+            mybi_id = os.environ.get(f"{prefix}MYBI_ACCOUNT_ID")
+            if not mybi_id:
+                raise ValueError(f"Не указан {prefix}MYBI_ACCOUNT_ID для аккаунта {account_id}")
+
+            accounts.append(
+                AmoCRMAccount(
+                    id=int(account_id),
+                    name=os.environ.get(f"{prefix}NAME", f"account_{i}") or f"account_{i}",
+                    base_url=os.environ.get(f"{prefix}URL", "") or "",
+                    token_path=Path(os.environ.get(f"{prefix}TOKEN_PATH", f"token_{i}.json") or f"token_{i}.json"),
+                    mybi_account_id=int(mybi_id),
+                    pipeline_ids=pipeline_ids if pipeline_ids else None,
+                    cache_dir=Path(os.environ.get(f"{prefix}CACHE_DIR", ".cache") or ".cache"),
+                )
+            )
+
+        return cls(
+            database=db_config,
+            accounts=accounts,
+            batch_size=int(os.environ.get("ETL_BATCH_SIZE", "100") or "100"),
+            window_minutes=int(os.environ.get("ETL_WINDOW_MINUTES", "120") or "120"),
+            log_level=os.environ.get("ETL_LOG_LEVEL", "INFO") or "INFO",
+        )
+
+
+# Пример конфигурации для разработки (можно переопределить в .env)
+DEFAULT_CONFIG = ETLConfig(
+    database=DatabaseConfig(
+        host="localhost",
+        port=5432,
+        dbname="amocrm",
+        user="postgres",
+        password="",
+        schema="public",
+    ),
+    accounts=[
+        AmoCRMAccount(
+            id=30019651,
+            name="bneginskogo",
+            base_url="https://bneginskogo.amocrm.ru",
+            token_path=Path("token.json"),
+            mybi_account_id=53859,  # Внутренний ID из mybi.ru
+            pipeline_ids=[5987164, 6241334],
+            cache_dir=Path(".cache"),
+        ),
+        # Добавьте остальные аккаунты здесь
+    ],
+    batch_size=100,
+    window_minutes=120,
+)
+
+
+def get_config(env_path: Optional[Union[str, Path]] = None) -> ETLConfig:
+    """
+    Получает конфигурацию ETL.
+
+    Если указан env_path, загружает настройки из файла.
+    Иначе использует переменные окружения или DEFAULT_CONFIG.
+    """
+    if env_path:
+        return ETLConfig.from_env(Path(env_path))
+
+    # Проверяем наличие .env в текущей директории
+    default_env = Path(".env")
+    if default_env.exists():
+        return ETLConfig.from_env(default_env)
+
+    # Проверяем наличие переменных окружения
+    if os.environ.get("ETL_DB_HOST") or os.environ.get("AMO_ACCOUNT_1_ID"):
+        return ETLConfig.from_env()
+
+    return DEFAULT_CONFIG