linkmerce 0.2.1__tar.gz → 0.2.2__tar.gz

{linkmerce-0.2.1 → linkmerce-0.2.2}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.3
 Name: linkmerce
-Version: 0.2.1
+Version: 0.2.2
 Summary: E-commerce API integration management
 Requires-Dist: aiohttp>=3.12.14
-Requires-Dist: asyncio>=3.4.3
 Requires-Dist: bs4>=0.0.2
 Requires-Dist: duckdb>=1.3.2
 Requires-Dist: jinja2>=3.1.6
+Requires-Dist: pyarrow>=21.0.0
 Requires-Dist: pytz>=2025.2
 Requires-Dist: requests>=2.32.4
 Requires-Dist: tqdm>=4.67.1

{linkmerce-0.2.1 → linkmerce-0.2.2}/pyproject.toml

@@ -1,15 +1,15 @@
 [project]
 name = "linkmerce"
-version = "0.2.1"
+version = "0.2.2"
 description = "E-commerce API integration management"
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
     "aiohttp>=3.12.14",
-    "asyncio>=3.4.3",
     "bs4>=0.0.2",
     "duckdb>=1.3.2",
     "jinja2>=3.1.6",
+    "pyarrow>=21.0.0",
     "pytz>=2025.2",
     "requests>=2.32.4",
     "tqdm>=4.67.1",

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/collect/base.py

@@ -9,8 +9,8 @@ if TYPE_CHECKING:
     from typing import Any, Coroutine, Hashable, IO, Literal, TypeVar
     from linkmerce.collect.tasks import RequestLoop, RequestEach
     from linkmerce.collect.tasks import PaginateAll, RequestEachPages
-    from linkmerce.types import JsonObject, JsonSerialize, ResponseComponent
-    from linkmerce.types import TaskOption, TaskOptions
+    from linkmerce.common import JsonObject, JsonSerialize, ResponseComponent
+    from linkmerce.common import TaskOption, TaskOptions
     _KT = TypeVar("_KT", Hashable)
     _VT = TypeVar("_VT", Any)
 
@@ -221,23 +221,6 @@ class RequestSessionClient(BaseSessionClient):
         response = self.request_text(method, url, params=params, data=data, json=json, headers=headers, cookies=cookies, **kwargs)
         return BeautifulSoup(response, features)
 
-    def request_table(
-            self,
-            method: str,
-            url: str,
-            params: dict | list[tuple] | bytes | None = None,
-            data: dict | list[tuple] | bytes | IO | None = None,
-            json: JsonSerialize | None = None,
-            headers: dict[str,str] = None,
-            cookies: dict | RequestsCookieJar = None,
-            content_type: Literal["excel", "csv", "html", "xml"] | Sequence = "xlsx",
-            table_options: dict = dict(),
-            **kwargs
-        ):
-        from linkmerce.extensions.pandas import read_table
-        response = self.request_content(method, url, params=params, data=data, json=json, headers=headers, cookies=cookies, **kwargs)
-        return read_table(response, table_format=content_type, **table_options)
-
     def with_session(func):
         @functools.wraps(func)
         def wrapper(self: RequestSessionClient, *args, init_session: bool = False, **kwargs):
@@ -365,23 +348,6 @@ class AiohttpSessionClient(BaseSessionClient):
         response = await self.request_async_text(method, url, params=params, data=data, json=json, headers=headers, cookies=cookies, **kwargs)
         return BeautifulSoup(response, features)
 
-    async def request_async_table(
-            self,
-            method: str,
-            url: str,
-            params: dict | list[tuple] | bytes | None = None,
-            data: dict | list[tuple] | bytes | IO | None = None,
-            json: JsonSerialize | None = None,
-            headers: dict[str,str] = None,
-            cookies: dict | LooseCookies = None,
-            content_type: Literal["excel", "csv", "html", "xml"] | Sequence = "xlsx",
-            table_options: dict = dict(),
-            **kwargs
-        ):
-        from linkmerce.extensions.pandas import read_table
-        response = await self.request_async_content(method, url, params=params, data=data, json=json, headers=headers, cookies=cookies, **kwargs)
-        return read_table(response, table_format=content_type, **table_options)
-
     def async_with_session(func):
         @functools.wraps(func)
         async def wrapper(self: AiohttpSessionClient, *args, init_session: bool = False, **kwargs):

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/collect/naver/api/__init__.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Any, Callable, Hashable, IO, Literal, TypeVar
-    from linkmerce.types import JsonSerialize
+    from linkmerce.common import JsonSerialize
     _KT = TypeVar("_KT", Hashable)
     _VT = TypeVar("_VT", Any)
 

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/collect/naver/api/search.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Iterable, Literal
-    from linkmerce.types import JsonObject, TaskOptions
+    from linkmerce.common import JsonObject, TaskOptions
 
 
 class _SearchCollector(NaverOpenAPI):

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/collect/searchad/manage/__init__.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Any, Callable, Hashable, IO, Literal, TypeVar
-    from linkmerce.types import JsonSerialize
+    from linkmerce.common import JsonSerialize
     _KT = TypeVar("_KT", Hashable)
     _VT = TypeVar("_VT", Any)
 
@@ -62,7 +62,7 @@ class SearchAdManager(Collector):
         headers = super().get_request_headers(referer=redirect_url, origin=self.origin)
         response = self.request_text("GET", url, headers=headers)
         if response.strip() != "true":
-            from linkmerce.exceptions import AuthenticationError
+            from linkmerce.common import AuthenticationError
             raise AuthenticationError("Authentication failed: cookies are invalid.")
 
     def authorize(self):

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/collect/searchad/manage/exposure.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Iterable, Literal
-    from linkmerce.types import JsonObject, TaskOptions
+    from linkmerce.common import JsonObject, TaskOptions
 
 
 class ExposureDiagnosis(SearchAdManager):

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/collect/smartstore/hcenter/catalog.py

@@ -6,7 +6,7 @@ from typing import Iterable, TYPE_CHECKING
 if TYPE_CHECKING:
     from typing import Callable, Coroutine, Literal, Sequence
     from linkmerce.collect.tasks import RequestEachPages
-    from linkmerce.types import JsonObject, TaskOptions
+    from linkmerce.common import JsonObject, TaskOptions
 
 
 class _CatalogCollector(PartnerCenter):

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/collect/smartstore/hcenter/sales.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Iterable, Literal
-    from linkmerce.types import JsonObject, TaskOptions
+    from linkmerce.common import JsonObject, TaskOptions
     import datetime as dt
 
 

linkmerce-0.2.1/src/linkmerce/types.py → linkmerce-0.2.2/src/linkmerce/common.py

@@ -12,3 +12,19 @@ ResponseComponent = Union[int, bytes, str, JsonObject, dict[str,str]]
 
 TaskOption = Dict[_KT,_VT]
 TaskOptions = Dict[_KT,TaskOption]
+
+
+class AuthenticationError(RuntimeError):
+    ...
+
+
+class ParseError(ValueError):
+    ...
+
+
+class RequestError(RuntimeError):
+    ...
+
+
+class UnauthorizedError(RuntimeError):
+    ...

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/parse/base.py

@@ -21,7 +21,7 @@ class Parser(metaclass=ABCMeta):
         raise NotImplementedError("The 'parse' method must be implemented.")
 
     def raise_parse_error(self, msg: str):
-        from linkmerce.exceptions import ParseError
+        from linkmerce.common import ParseError
         raise ParseError(msg)
 
 

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/parse/naver/api/search.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Literal
-    from linkmerce.types import JsonObject
+    from linkmerce.common import JsonObject
 
 
 class _SearchParser(QueryParser):
@@ -27,7 +27,7 @@ class _SearchParser(QueryParser):
         return wrapper
 
     def raise_request_error(self, response: dict):
-        from linkmerce.exceptions import RequestError
+        from linkmerce.common import RequestError
         raise RequestError(response.get("errorMessage") or str())
 
     @check_errors

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/parse/searchad/manage/exposure.py

@@ -6,7 +6,7 @@ import functools
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from linkmerce.types import JsonObject
+    from linkmerce.common import JsonObject
 
 
 class ExposureDiagnosis(QueryParser):
@@ -23,7 +23,7 @@ class ExposureDiagnosis(QueryParser):
         return wrapper
 
     def raise_request_error(self, response: dict):
-        from linkmerce.exceptions import RequestError, UnauthorizedError
+        from linkmerce.common import RequestError, UnauthorizedError
         msg = response.get("title") or response.get("message") or str()
         if (msg == "Forbidden") or ("권한이 없습니다." in msg) or ("인증이 만료됐습니다." in msg):
             raise UnauthorizedError(msg)
@@ -47,7 +47,7 @@ class ExposureDiagnosis(QueryParser):
             NULLIF(fmpBrand, '') AS mallName,
             NULLIF(fmpMaker, '') AS makerName,
             imageUrl,
-            CAST(COALESCE(lowPrice, mobileLowPrice, NULL) AS INT64) AS salesPrice,
+            -- CAST(COALESCE(lowPrice, mobileLowPrice, NULL) AS INT64) AS salesPrice,
             {{ created_at }} AS createdAt
         FROM {{ table }}
         {{ where }}

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/parse/smartstore/hcenter/catalog.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Literal
-    from linkmerce.types import JsonObject
+    from linkmerce.common import JsonObject
 
 
 class _CatalogParser(QueryParser):
@@ -27,7 +27,7 @@ class _CatalogParser(QueryParser):
 
     def raise_request_error(self, response: dict):
         from linkmerce.utils.map import hier_get
-        from linkmerce.exceptions import RequestError
+        from linkmerce.common import RequestError
         msg = hier_get(response, ["errors",0,"message"]) or "null"
         raise RequestError(f"An error occurred during the request: {msg}")
 

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/parse/smartstore/hcenter/sales.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Literal
-    from linkmerce.types import JsonObject
+    from linkmerce.common import JsonObject
     import datetime as dt
 
 
@@ -30,10 +30,10 @@ class _SalesParser(QueryParser):
         from linkmerce.utils.map import hier_get
         msg = hier_get(response, ["error","error"]) or "null"
         if msg == "Unauthorized":
-            from linkmerce.exceptions import UnauthorizedError
+            from linkmerce.common import UnauthorizedError
             raise UnauthorizedError("Unauthorized request")
         else:
-            from linkmerce.exceptions import RequestError
+            from linkmerce.common import RequestError
             raise RequestError(f"An error occurred during the request: {msg}")
 
     @check_errors

{linkmerce-0.2.1 → linkmerce-0.2.2}/src/linkmerce/utils/duckdb.py

@@ -6,8 +6,9 @@ import functools
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Any, Generator, Literal, Sequence
+    from typing import Any, Literal, Sequence
     from duckdb import DuckDBPyConnection
+    from linkmerce.utils.pyarrow import Table
 
 DEFAULT_TEMP_TABLE = "temp_table"
 
@@ -79,6 +80,17 @@ def select_to_json(
     return [dict(zip(columns, row)) for row in relation.fetchall()]
 
 
+def select_to_arrow(
+        query: str,
+        table: tuple[str,Table],
+        params: dict | None = None,
+        conn: DuckDBPyConnection | None = None,
+    ) -> Table:
+    with duckdb.connect() as conn:
+        conn.register(*table)
+        return conn.execute(query, parameters=params).arrow()
+
+
 ###################################################################
 ############################# Datetime ############################
 ###################################################################
@@ -200,52 +212,68 @@ def combine_first(
 ########################### Partition By ##########################
 ###################################################################
 
-def partition_by(
-        data: list[dict],
-        field: str,
-        type: str | None = None,
-        condition: str | None = None,
-        sort: bool = True,
-    ) -> Generator[list[dict], None, None]:
-    conn = duckdb.connect()
-    try:
-        create_table(conn, DEFAULT_TEMP_TABLE, data, option="ignore", temp=True)
-        if field not in get_columns(conn, DEFAULT_TEMP_TABLE):
-            field = _add_partition(conn, DEFAULT_TEMP_TABLE, field, type)
-        exclude = "EXCLUDE (_PARTITIONFIELD)" if field == "_PARTITIONFIELD" else str()
-        for partition in _select_partition(conn, DEFAULT_TEMP_TABLE, field, condition, sort):
-            yield select_to_json(
-                f"SELECT * {exclude} FROM temp_table WHERE {field} = {_quote(partition)};", conn=conn)
-    finally:
-        conn.close()
-
-
-def _add_partition(
-        conn: DuckDBPyConnection,
-        table: str,
-        expr: str,
-        type: str | None = None,
-    ) -> str:
-    field = "_PARTITIONFIELD"
-    if not type:
-        type = conn.execute(f"SELECT {expr} FROM {table} LIMIT 1").description[0][TYPE]
-    conn.execute(f"ALTER TABLE {table} ADD COLUMN {field} {type};")
-    conn.execute(f"UPDATE {table} SET {field} = {expr};")
-    return field
+class Partition:
+    def __init__(
+            self,
+            data: list[dict],
+            field: str,
+            type: str | None = None,
+            condition: str | None = None,
+            sort: bool = True,
+            temp_table: str = DEFAULT_TEMP_TABLE,
+        ):
+        self.conn = duckdb.connect()
+        self.table = temp_table
+        try:
+            self.set_data(data)
+            self.set_field(field, type)
+            self.set_partitions(condition, sort)
+        except:
+            ...
+
+    def set_data(self, data: list[dict]):
+        create_table(self.conn, self.table, data, option="ignore", temp=True)
+
+    def set_field(self, field: str, type: str | None = None):
+        if field not in get_columns(self.conn, self.table):
+            field = "_PARTITIONFIELD"
+            if not type:
+                type = self.conn.execute(f"SELECT {field} FROM {self.table} LIMIT 1").description[0][TYPE]
+            self.conn.execute(f"ALTER TABLE {self.table} ADD COLUMN {field} {type};")
+            self.conn.execute(f"UPDATE {self.table} SET {field} = {field};")
+        self.field = field
+
+    def set_partitions(self,  condition: str | None = None, sort: bool = True):
+        query = f"SELECT DISTINCT {self.field} FROM {self.table} {_where(condition, self.field)};"
+        if sort:
+            self.partitions = sorted(map(lambda x: x[0], self.conn.execute(query).fetchall()))
+        else:
+            self.partitions = [row[0] for row in self.conn.execute(query).fetchall()]
+
+    def __iter__(self) -> Partition:
+        self.index = 0
+        return self
+
+    def __next__(self) -> list[dict]:
+        if self.index < len(self):
+            exclude = "EXCLUDE (_PARTITIONFIELD)" if self.field == "_PARTITIONFIELD" else str()
+            query = f"SELECT * {exclude} FROM temp_table WHERE {self.field} = {_quote(self.partitions[self.index])};"
+            self.index += 1
+            return select_to_json(query, conn=self.conn)
+        else:
+            raise StopIteration
 
+    def __len__(self) -> int:
+        return len(self.partitions)
 
-def _select_partition(
-        conn: DuckDBPyConnection,
-        table: str,
-        field: str,
-        condition: str | None = None,
-        sort: bool = True
-    ) -> list[Any]:
-    query = f"SELECT DISTINCT {field} FROM {table} {_where(condition, field)};"
-    if sort:
-        return sorted(map(lambda x: x[0], conn.execute(query).fetchall()))
-    else:
-        return [row[0] for row in conn.execute(query).fetchall()]
+    def __exit__(self):
+        self.close()
+
+    def close(self):
+        try:
+            self.conn.close()
+        except:
+            pass
 
 
 def _where(condition: str | None = None, field: str | None = None, **kwargs) -> str: