amochka 0.4.0__tar.gz → 0.4.1__tar.gz

{amochka-0.4.0 → amochka-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: amochka
-Version: 0.4.0
+Version: 0.4.1
 Summary: Python library for working with amoCRM API with ETL capabilities
 Author-email: Timur <timurdt@gmail.com>
 License: MIT

{amochka-0.4.0 → amochka-0.4.1}/amochka/__init__.py

@@ -2,7 +2,7 @@
 amochka: Библиотека для работы с API amoCRM.
 """
 
-__version__ = "0.4.0"
+__version__ = "0.4.1"
 
 from .client import AmoCRMClient, CacheConfig
 from .errors import (

{amochka-0.4.0 → amochka-0.4.1}/amochka/client.py

@@ -1591,6 +1591,9 @@ class AmoCRMClient:
         notes = []
         while True:
             response = self._make_request("GET", endpoint, params=params)
+            if not response:
+                self.logger.warning(f"Empty response for notes {entity} {entity_id}, stopping pagination")
+                break
             if response and "_embedded" in response and "notes" in response["_embedded"]:
                 notes.extend(response["_embedded"]["notes"])
             if not get_all:
@@ -1670,6 +1673,9 @@ class AmoCRMClient:
         events = []
         while True:
             response = self._make_request("GET", "/api/v4/events", params=params)
+            if not response:
+                self.logger.warning(f"Empty response for events {entity} {entity_id}, stopping pagination")
+                break
             if response and "_embedded" in response and "events" in response["_embedded"]:
                 events.extend(response["_embedded"]["events"])
             # Если не нужно получать все страницы, выходим

{amochka-0.4.0 → amochka-0.4.1}/amochka.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: amochka
-Version: 0.4.0
+Version: 0.4.1
 Summary: Python library for working with amoCRM API with ETL capabilities
 Author-email: Timur <timurdt@gmail.com>
 License: MIT

{amochka-0.4.0 → amochka-0.4.1}/amochka.egg-info/SOURCES.txt

@@ -16,4 +16,10 @@ etl/loaders.py
 etl/run_etl.py
 etl/transformers.py
 etl/migrations/001_create_tables.sql
-tests/test_client.py
+tests/test_cache.py
+tests/test_client.py
+tests/test_etl.py
+tests/test_http.py
+tests/test_notes_events.py
+tests/test_security.py
+tests/test_utils.py

{amochka-0.4.0 → amochka-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "amochka"
-version = "0.4.0"
+version = "0.4.1"
 description = "Python library for working with amoCRM API with ETL capabilities"
 readme = "README.md"
 authors = [

amochka-0.4.1/tests/test_cache.py

@@ -0,0 +1,87 @@
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+import amochka.client as client_module
+from amochka import AmoCRMClient, CacheConfig
+
+
+def make_client(cache_config):
+    token_data = {"access_token": "token", "expires_at": str(time.time() + 3600)}
+    return AmoCRMClient(
+        base_url="https://example.amocrm.ru",
+        token_file=json.dumps(token_data),
+        cache_config=cache_config,
+        disable_logging=True,
+    )
+
+
+def make_file_client(tmp_path, lifetime_hours=1):
+    token_path = tmp_path / "token.json"
+    token_path.write_text(json.dumps({"access_token": "token", "expires_at": str(time.time() + 3600)}))
+    cache_dir = tmp_path / "cache"
+    cache_config = CacheConfig.file_cache(base_dir=str(cache_dir), lifetime_hours=lifetime_hours)
+    return AmoCRMClient(
+        base_url="https://example.amocrm.ru",
+        token_file=str(token_path),
+        cache_config=cache_config,
+        disable_logging=True,
+    )
+
+
+def test_memory_cache_ttl(monkeypatch):
+    client = make_client(CacheConfig.memory_only())
+    client._memory_cache_ttl = 10
+
+    now = {"value": 100.0}
+
+    def fake_time():
+        return now["value"]
+
+    monkeypatch.setattr(client_module.time, "time", fake_time)
+
+    calls = {"count": 0}
+
+    def fetch():
+        calls["count"] += 1
+        return [calls["count"]]
+
+    assert client._get_cached_resource("users", fetch) == [1]
+    now["value"] = 105.0
+    assert client._get_cached_resource("users", fetch) == [1]
+    assert calls["count"] == 1
+    now["value"] = 120.0
+    assert client._get_cached_resource("users", fetch) == [2]
+    assert calls["count"] == 2
+
+
+def test_file_cache_save_and_load(tmp_path):
+    client = make_file_client(tmp_path, lifetime_hours=1)
+    payload = [{"id": 1}]
+    client._save_cache("users", payload)
+    loaded = client._load_cache("users")
+    assert loaded == payload
+
+
+def test_file_cache_expired_returns_none(tmp_path):
+    client = make_file_client(tmp_path, lifetime_hours=1)
+    payload = [{"id": 1}]
+    client._save_cache("users", payload)
+    cache_path = client._get_cache_file_path("users")
+    with open(cache_path, "r", encoding="utf-8") as handle:
+        cache_data = json.loads(handle.read())
+    cache_data["last_updated"] = time.time() - 7200
+    with open(cache_path, "w", encoding="utf-8") as handle:
+        json.dump(cache_data, handle)
+    assert client._load_cache("users") is None
+
+
+def test_get_lifetime_default_and_none():
+    config = CacheConfig(lifetime_hours={"users": 5})
+    assert config.get_lifetime("users") == 5
+    assert config.get_lifetime("unknown") == 24
+    config = CacheConfig(lifetime_hours=None)
+    assert config.get_lifetime("users") is None

amochka-0.4.1/tests/test_etl.py

@@ -0,0 +1,34 @@
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+from amochka.etl import write_ndjson
+
+
+def test_write_ndjson_transform_and_on_record(tmp_path):
+    records = [{"id": 1, "updated_at": 123}]
+    seen = []
+
+    def transform(record):
+        return {"id": record["id"], "flag": True}
+
+    def on_record(record):
+        seen.append(record["id"])
+
+    output_path = tmp_path / "out.ndjson"
+    count = write_ndjson(
+        records,
+        output_path,
+        entity="lead",
+        account_id=42,
+        transform=transform,
+        on_record=on_record,
+    )
+    assert count == 1
+    payload = json.loads(output_path.read_text().strip())
+    assert payload["entity"] == "lead"
+    assert payload["account_id"] == 42
+    assert payload["payload"]["flag"] is True
+    assert seen == [1]

amochka-0.4.1/tests/test_http.py

@@ -0,0 +1,168 @@
+import json
+import os
+import sys
+import time
+
+import pytest
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+import amochka.client as client_module
+from amochka import AmoCRMClient, CacheConfig, APIError, NotFoundError, RateLimitError
+
+
+class FakeResponse:
+    def __init__(self, status_code, json_data=None, text="", headers=None):
+        self.status_code = status_code
+        self._json_data = json_data
+        self.text = text
+        self.headers = headers or {}
+
+    def json(self):
+        return self._json_data
+
+
+def make_client(max_retries=0):
+    token_data = {"access_token": "token", "expires_at": str(time.time() + 3600)}
+    return AmoCRMClient(
+        base_url="https://example.amocrm.ru",
+        token_file=json.dumps(token_data),
+        cache_config=CacheConfig.disabled(),
+        disable_logging=True,
+        max_retries=max_retries,
+    )
+
+
+def test_make_request_success_json(monkeypatch):
+    client = make_client()
+
+    def fake_request(*args, **kwargs):
+        return FakeResponse(200, json_data={"ok": True})
+
+    monkeypatch.setattr(client_module.requests, "request", fake_request)
+    assert client._make_request("GET", "/api/v4/leads") == {"ok": True}
+
+
+def test_make_request_204_returns_none(monkeypatch):
+    client = make_client()
+
+    def fake_request(*args, **kwargs):
+        return FakeResponse(204)
+
+    monkeypatch.setattr(client_module.requests, "request", fake_request)
+    assert client._make_request("DELETE", "/api/v4/leads/1") is None
+
+
+def test_make_request_404_raises(monkeypatch):
+    client = make_client()
+
+    def fake_request(*args, **kwargs):
+        return FakeResponse(404, text="not found")
+
+    monkeypatch.setattr(client_module.requests, "request", fake_request)
+    with pytest.raises(NotFoundError):
+        client._make_request("GET", "/api/v4/leads/404")
+
+
+def test_make_request_bad_request_non_sensitive(monkeypatch):
+    client = make_client()
+
+    def fake_request(*args, **kwargs):
+        return FakeResponse(400, text="bad request")
+
+    monkeypatch.setattr(client_module.requests, "request", fake_request)
+    with pytest.raises(APIError) as excinfo:
+        client._make_request("GET", "/api/v4/leads")
+    assert "bad request" in str(excinfo.value)
+
+
+def test_make_request_bad_request_sensitive(monkeypatch):
+    client = make_client()
+
+    def fake_request(*args, **kwargs):
+        return FakeResponse(400, text="secret token")
+
+    monkeypatch.setattr(client_module.requests, "request", fake_request)
+    with pytest.raises(APIError) as excinfo:
+        client._make_request("POST", "/oauth2/access_token")
+    assert "secret token" not in str(excinfo.value)
+
+
+def test_rate_limit_error_includes_retry_after(monkeypatch):
+    client = make_client(max_retries=0)
+
+    def fake_request(*args, **kwargs):
+        return FakeResponse(429, text="too many", headers={"Retry-After": "3"})
+
+    monkeypatch.setattr(client_module.requests, "request", fake_request)
+    with pytest.raises(RateLimitError) as excinfo:
+        client._make_request("GET", "/api/v4/leads")
+    assert excinfo.value.retry_after == "3"
+    assert "too many" in str(excinfo.value)
+
+
+def test_rate_limit_sensitive_hides_body(monkeypatch):
+    client = make_client(max_retries=0)
+
+    def fake_request(*args, **kwargs):
+        return FakeResponse(429, text="secret", headers={"Retry-After": "1"})
+
+    monkeypatch.setattr(client_module.requests, "request", fake_request)
+    with pytest.raises(RateLimitError) as excinfo:
+        client._make_request("GET", "/oauth2/access_token")
+    assert "secret" not in str(excinfo.value)
+
+
+def test_retries_then_succeeds(monkeypatch):
+    client = make_client(max_retries=1)
+    responses = [
+        FakeResponse(500, text="server error"),
+        FakeResponse(200, json_data={"ok": True}),
+    ]
+    calls = []
+
+    def fake_request(*args, **kwargs):
+        calls.append(1)
+        return responses.pop(0)
+
+    monkeypatch.setattr(client_module.requests, "request", fake_request)
+    monkeypatch.setattr(client_module.time, "sleep", lambda *_: None)
+    assert client._make_request("GET", "/api/v4/leads") == {"ok": True}
+    assert len(calls) == 2
+
+
+def test_401_triggers_refresh(monkeypatch):
+    client = make_client(max_retries=1)
+    client.refresh_token = "refresh"
+    client.client_id = "id"
+    client.client_secret = "secret"
+    client.redirect_uri = "https://example.amocrm.ru"
+
+    responses = [
+        FakeResponse(401, text="unauthorized"),
+        FakeResponse(200, json_data={"ok": True}),
+    ]
+    refreshed = {"called": 0}
+
+    def fake_request(*args, **kwargs):
+        return responses.pop(0)
+
+    def fake_refresh():
+        refreshed["called"] += 1
+        client.token = "new-token"
+
+    monkeypatch.setattr(client_module.requests, "request", fake_request)
+    monkeypatch.setattr(client, "_refresh_access_token", fake_refresh)
+    assert client._make_request("GET", "/api/v4/leads") == {"ok": True}
+    assert refreshed["called"] == 1
+
+
+def test_timeout_raises_api_error(monkeypatch):
+    client = make_client(max_retries=0)
+
+    def fake_request(*args, **kwargs):
+        raise client_module.requests.exceptions.Timeout("timeout")
+
+    monkeypatch.setattr(client_module.requests, "request", fake_request)
+    with pytest.raises(APIError):
+        client._make_request("GET", "/api/v4/leads")

amochka-0.4.1/tests/test_notes_events.py

@@ -0,0 +1,30 @@
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+from amochka import AmoCRMClient, CacheConfig
+
+
+def make_client():
+    token_data = {"access_token": "token", "expires_at": str(time.time() + 3600)}
+    return AmoCRMClient(
+        base_url="https://example.amocrm.ru",
+        token_file=json.dumps(token_data),
+        cache_config=CacheConfig.disabled(),
+        disable_logging=True,
+    )
+
+
+def test_get_entity_notes_handles_none_response(monkeypatch):
+    client = make_client()
+    monkeypatch.setattr(client, "_make_request", lambda *args, **kwargs: None)
+    assert client.get_entity_notes("lead", 1, get_all=True) == []
+
+
+def test_get_entity_events_handles_none_response(monkeypatch):
+    client = make_client()
+    monkeypatch.setattr(client, "_make_request", lambda *args, **kwargs: None)
+    assert client.get_entity_events("lead", 1, get_all=True) == []

amochka-0.4.1/tests/test_security.py

@@ -0,0 +1,179 @@
+import json
+import os
+import sys
+import time
+
+import pytest
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+import amochka.client as client_module
+from amochka import AmoCRMClient, CacheConfig, AuthenticationError
+
+
+class FakeResponse:
+    def __init__(self, status_code, json_data=None, text="", headers=None):
+        self.status_code = status_code
+        self._json_data = json_data or {}
+        self.text = text
+        self.headers = headers or {}
+
+    def json(self):
+        return self._json_data
+
+
+def _token_payload(expires_at):
+    return {
+        "access_token": "token",
+        "refresh_token": "refresh",
+        "expires_at": expires_at,
+        "client_id": "client-id",
+        "client_secret": "client-secret",
+        "redirect_uri": "https://example.amocrm.ru",
+    }
+
+
+def test_validate_base_url_accepts_allowed_domains():
+    assert AmoCRMClient._validate_base_url("https://example.amocrm.ru/") == "https://example.amocrm.ru"
+    assert AmoCRMClient._validate_base_url("https://sub.amocrm.com") == "https://sub.amocrm.com"
+    assert AmoCRMClient._validate_base_url("https://demo.kommo.com/") == "https://demo.kommo.com"
+
+
+def test_validate_base_url_rejects_invalid():
+    with pytest.raises(ValueError):
+        AmoCRMClient._validate_base_url("http://example.amocrm.ru")
+    with pytest.raises(ValueError):
+        AmoCRMClient._validate_base_url("https://example.com")
+    with pytest.raises(ValueError):
+        AmoCRMClient._validate_base_url("https://example.amocrm.ru:8443")
+
+
+def test_mask_sensitive():
+    assert AmoCRMClient._mask_sensitive(None) == "***"
+    assert AmoCRMClient._mask_sensitive("abcd") == "***"
+    assert AmoCRMClient._mask_sensitive("abcdefghijkl") == "abcd...ijkl"
+
+
+def test_env_token_overrides_file(monkeypatch, tmp_path):
+    token_path = tmp_path / "token.json"
+    token_path.write_text(json.dumps(_token_payload(time.time() + 3600)))
+
+    monkeypatch.delenv("AMOCRM_ACCESS_TOKEN", raising=False)
+    monkeypatch.delenv("AMOCRM_EXPIRES_AT", raising=False)
+    monkeypatch.setenv("AMOCRM_ACCESS_TOKEN", "env-token")
+    monkeypatch.setenv("AMOCRM_EXPIRES_AT", str(time.time() + 3600))
+
+    client = AmoCRMClient(
+        base_url="https://example.amocrm.ru",
+        token_file=str(token_path),
+        cache_config=CacheConfig.disabled(),
+        disable_logging=True,
+    )
+    assert client.token == "env-token"
+
+
+def test_load_token_from_json_string():
+    payload = _token_payload(time.time() + 3600)
+    client = AmoCRMClient(
+        base_url="https://example.amocrm.ru",
+        token_file=json.dumps(payload),
+        cache_config=CacheConfig.disabled(),
+        disable_logging=True,
+    )
+    assert client.token == "token"
+
+
+def test_invalid_token_string_raises():
+    with pytest.raises(AuthenticationError):
+        AmoCRMClient(
+            base_url="https://example.amocrm.ru",
+            token_file="{bad json",
+            cache_config=CacheConfig.disabled(),
+            disable_logging=True,
+        )
+
+
+def test_refresh_access_token_success(monkeypatch, tmp_path):
+    token_path = tmp_path / "token.json"
+    token_path.write_text(json.dumps(_token_payload(time.time() - 10)))
+
+    def fake_post(*args, **kwargs):
+        return FakeResponse(
+            200,
+            json_data={
+                "access_token": "new-token",
+                "refresh_token": "new-refresh",
+                "expires_in": 3600,
+            },
+        )
+
+    monkeypatch.setattr(client_module.requests, "post", fake_post)
+
+    client = AmoCRMClient(
+        base_url="https://example.amocrm.ru",
+        token_file=str(token_path),
+        cache_config=CacheConfig.disabled(),
+        disable_logging=True,
+    )
+
+    assert client.token == "new-token"
+    saved = json.loads(token_path.read_text())
+    assert saved["access_token"] == "new-token"
+
+
+def test_refresh_access_token_failure(monkeypatch, tmp_path):
+    token_path = tmp_path / "token.json"
+    token_path.write_text(json.dumps(_token_payload(time.time() - 10)))
+
+    def fake_post(*args, **kwargs):
+        return FakeResponse(400, text="bad")
+
+    monkeypatch.setattr(client_module.requests, "post", fake_post)
+
+    with pytest.raises(AuthenticationError):
+        AmoCRMClient(
+            base_url="https://example.amocrm.ru",
+            token_file=str(token_path),
+            cache_config=CacheConfig.disabled(),
+            disable_logging=True,
+        )
+
+
+def test_cache_path_validation():
+    with pytest.raises(ValueError):
+        CacheConfig._validate_path("../secret", "base_dir")
+    with pytest.raises(ValueError):
+        CacheConfig._validate_path("", "file")
+
+
+def test_cache_file_path_stays_in_base_dir(tmp_path):
+    token_path = tmp_path / "acct..name.json"
+    token_path.write_text(json.dumps(_token_payload(time.time() + 3600)))
+    cache_dir = tmp_path / "cache"
+    cache_config = CacheConfig.file_cache(base_dir=str(cache_dir))
+
+    client = AmoCRMClient(
+        base_url="https://example.amocrm.ru",
+        token_file=str(token_path),
+        cache_config=cache_config,
+        disable_logging=True,
+    )
+
+    cache_path = client._get_cache_file_path("users")
+    assert os.path.realpath(cache_path).startswith(os.path.realpath(str(cache_dir)))
+    assert ".." not in os.path.basename(cache_path)
+
+
+def test_extract_account_name_from_token_file(tmp_path):
+    token_path = tmp_path / "accounts" / "bneginskogo_eng.json"
+    token_path.parent.mkdir(parents=True, exist_ok=True)
+    token_path.write_text(json.dumps(_token_payload(time.time() + 3600)))
+
+    client = AmoCRMClient(
+        base_url="https://example.amocrm.ru",
+        token_file=str(token_path),
+        cache_config=CacheConfig.disabled(),
+        disable_logging=True,
+    )
+
+    assert client._extract_account_name() == "eng"

amochka-0.4.1/tests/test_utils.py

@@ -0,0 +1,89 @@
+import json
+import os
+import sys
+import time
+from datetime import datetime
+
+import pytest
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+from amochka import AmoCRMClient, CacheConfig
+
+
+class PaginationClient(AmoCRMClient):
+    def __init__(self, responses):
+        token_data = {"access_token": "token", "expires_at": str(time.time() + 3600)}
+        super().__init__(
+            base_url="https://example.amocrm.ru",
+            token_file=json.dumps(token_data),
+            cache_config=CacheConfig.disabled(),
+            disable_logging=True,
+        )
+        self._responses = list(responses)
+        self.calls = 0
+
+    def _make_request(self, method, endpoint, params=None, data=None, timeout=10):
+        self.calls += 1
+        return self._responses.pop(0)
+
+
+def make_client():
+    token_data = {"access_token": "token", "expires_at": str(time.time() + 3600)}
+    return AmoCRMClient(
+        base_url="https://example.amocrm.ru",
+        token_file=json.dumps(token_data),
+        cache_config=CacheConfig.disabled(),
+        disable_logging=True,
+    )
+
+
+def test_to_timestamp():
+    client = make_client()
+    dt = datetime(2020, 1, 1, 0, 0, 0)
+    assert client._to_timestamp(dt) == int(dt.timestamp())
+    assert client._to_timestamp(1700000000) == 1700000000
+    assert client._to_timestamp(1700000000.5) == 1700000000
+    iso = "2020-01-01T00:00:00"
+    assert client._to_timestamp(iso) == int(datetime.fromisoformat(iso).timestamp())
+    with pytest.raises(ValueError):
+        client._to_timestamp("bad-date")
+    with pytest.raises(TypeError):
+        client._to_timestamp(object())
+
+
+def test_format_filter_values():
+    client = make_client()
+    assert client._format_filter_values(None) is None
+    assert client._format_filter_values(123) == "123"
+    assert client._format_filter_values(["1", 2]) == ["1", "2"]
+
+
+def test_extract_collection():
+    client = make_client()
+    response = {"_embedded": {"items": [{"id": 1}]}}
+    assert client._extract_collection(response, ("_embedded", "items")) == [{"id": 1}]
+    assert client._extract_collection(response, ("_embedded", "missing")) == []
+    assert client._extract_collection({"_embedded": {"items": {"id": 1}}}, ("_embedded", "items")) == []
+
+
+def test_iterate_paginated_links_next():
+    responses = [
+        {"_embedded": {"items": [{"id": 1}]}, "_links": {"next": {"href": "next"}}},
+        {"_embedded": {"items": [{"id": 2}]}, "_links": {}},
+    ]
+    client = PaginationClient(responses)
+    items = list(client._iterate_paginated("/api/v4/items", data_path=("_embedded", "items")))
+    assert [item["id"] for item in items] == [1, 2]
+    assert client.calls == 2
+
+
+def test_iterate_paginated_max_pages():
+    responses = [
+        {"_embedded": {"items": [{"id": 1}]}, "_page_count": 3},
+        {"_embedded": {"items": [{"id": 2}]}, "_page_count": 3},
+    ]
+    client = PaginationClient(responses)
+    items = list(client._iterate_paginated("/api/v4/items", data_path=("_embedded", "items"), max_pages=1))
+    assert [item["id"] for item in items] == [1]
+    assert client.calls == 1