linkmerce 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {linkmerce-0.3.0 → linkmerce-0.3.2}/PKG-INFO +1 -1
- {linkmerce-0.3.0 → linkmerce-0.3.2}/pyproject.toml +1 -1
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/api/naver/openapi.py +25 -16
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/api/searchad/manage.py +7 -4
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/api/smartstore/brand.py +43 -14
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/common/api.py +36 -6
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/common/extract.py +2 -1
- linkmerce-0.3.2/src/linkmerce/common/load.py +598 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/common/tasks.py +1 -2
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/common/transform.py +58 -32
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/naver/openapi/search/extract.py +1 -1
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/naver/openapi/search/models.sql +2 -2
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/naver/openapi/search/transform.py +8 -9
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/searchad/manage/exposure/transform.py +2 -1
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/brand/catalog/extract.py +4 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/brand/catalog/models.sql +21 -1
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/brand/catalog/transform.py +15 -1
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/brand/sales/transform.py +2 -1
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/utils/tqdm.py +11 -3
- linkmerce-0.3.0/src/linkmerce/common/load.py +0 -195
- linkmerce-0.3.0/src/linkmerce/utils/duckdb.py +0 -274
- {linkmerce-0.3.0 → linkmerce-0.3.2}/README.md +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/.DS_Store +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/api/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/api/naver/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/api/searchad/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/api/smartstore/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/common/.DS_Store +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/common/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/common/exceptions.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/common/models.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/.DS_Store +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/naver/.DS_Store +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/naver/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/naver/openapi/.DS_Store +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/naver/openapi/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/naver/openapi/common.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/naver/openapi/search/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/searchad/.DS_Store +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/searchad/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/searchad/manage/.DS_Store +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/searchad/manage/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/searchad/manage/common.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/searchad/manage/exposure/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/searchad/manage/exposure/extract.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/searchad/manage/exposure/models.sql +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/.DS_Store +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/brand/.DS_Store +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/brand/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/brand/catalog/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/brand/common.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/brand/sales/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/brand/sales/extract.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/core/smartstore/brand/sales/models.sql +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/utils/.DS_Store +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/utils/__init__.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/utils/cast.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/utils/date.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/utils/graphql.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/utils/headers.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/utils/jinja.py +0 -0
- {linkmerce-0.3.0 → linkmerce-0.3.2}/src/linkmerce/utils/map.py +0 -0
|
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING
|
|
|
8
8
|
if TYPE_CHECKING:
|
|
9
9
|
from typing import Iterable, Literal
|
|
10
10
|
from linkmerce.common.extract import JsonObject
|
|
11
|
+
from linkmerce.common.load import DuckDBConnection
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
def get_module(name: str) -> str:
|
|
@@ -21,8 +22,9 @@ def search_blog(
|
|
|
21
22
|
start: int | Iterable[int] = 1,
|
|
22
23
|
display: int = 100,
|
|
23
24
|
sort: Literal["sim","date"] = "sim",
|
|
25
|
+
connection: DuckDBConnection | None = None,
|
|
24
26
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
25
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
27
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
26
28
|
extract_options: dict | None = None,
|
|
27
29
|
transform_options: dict | None = None,
|
|
28
30
|
) -> JsonObject:
|
|
@@ -30,7 +32,7 @@ def search_blog(
|
|
|
30
32
|
table = get_table(transform_options, "table")
|
|
31
33
|
extract_options = dict(extract_options or dict(), variables=dict(client_id=client_id, client_secret=client_secret))
|
|
32
34
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
33
|
-
return run_with_duckdb(get_module(".search"), "BlogSearch", "BlogSearch", how, table, return_type, args, **options)
|
|
35
|
+
return run_with_duckdb(get_module(".search"), "BlogSearch", "BlogSearch", connection, how, table, return_type, args, **options)
|
|
34
36
|
|
|
35
37
|
|
|
36
38
|
def search_news(
|
|
@@ -40,8 +42,9 @@ def search_news(
|
|
|
40
42
|
start: int | Iterable[int] = 1,
|
|
41
43
|
display: int = 100,
|
|
42
44
|
sort: Literal["sim","date"] = "sim",
|
|
45
|
+
connection: DuckDBConnection | None = None,
|
|
43
46
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
44
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
47
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
45
48
|
extract_options: dict | None = None,
|
|
46
49
|
transform_options: dict | None = None,
|
|
47
50
|
) -> JsonObject:
|
|
@@ -49,7 +52,7 @@ def search_news(
|
|
|
49
52
|
table = get_table(transform_options, "table")
|
|
50
53
|
extract_options = dict(extract_options or dict(), variables=dict(client_id=client_id, client_secret=client_secret))
|
|
51
54
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
52
|
-
return run_with_duckdb(get_module(".search"), "NewsSearch", "NewsSearch", how, table, return_type, args, **options)
|
|
55
|
+
return run_with_duckdb(get_module(".search"), "NewsSearch", "NewsSearch", connection, how, table, return_type, args, **options)
|
|
53
56
|
|
|
54
57
|
|
|
55
58
|
def search_book(
|
|
@@ -59,8 +62,9 @@ def search_book(
|
|
|
59
62
|
start: int | Iterable[int] = 1,
|
|
60
63
|
display: int = 100,
|
|
61
64
|
sort: Literal["sim","date"] = "sim",
|
|
65
|
+
connection: DuckDBConnection | None = None,
|
|
62
66
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
63
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
67
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
64
68
|
extract_options: dict | None = None,
|
|
65
69
|
transform_options: dict | None = None,
|
|
66
70
|
) -> JsonObject:
|
|
@@ -68,7 +72,7 @@ def search_book(
|
|
|
68
72
|
table = get_table(transform_options, "table")
|
|
69
73
|
extract_options = dict(extract_options or dict(), variables=dict(client_id=client_id, client_secret=client_secret))
|
|
70
74
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
71
|
-
return run_with_duckdb(get_module(".search"), "BookSearch", "BookSearch", how, table, return_type, args, **options)
|
|
75
|
+
return run_with_duckdb(get_module(".search"), "BookSearch", "BookSearch", connection, how, table, return_type, args, **options)
|
|
72
76
|
|
|
73
77
|
|
|
74
78
|
def search_cafe(
|
|
@@ -78,8 +82,9 @@ def search_cafe(
|
|
|
78
82
|
start: int | Iterable[int] = 1,
|
|
79
83
|
display: int = 100,
|
|
80
84
|
sort: Literal["sim","date"] = "sim",
|
|
85
|
+
connection: DuckDBConnection | None = None,
|
|
81
86
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
82
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
87
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
83
88
|
extract_options: dict | None = None,
|
|
84
89
|
transform_options: dict | None = None,
|
|
85
90
|
) -> JsonObject:
|
|
@@ -87,7 +92,7 @@ def search_cafe(
|
|
|
87
92
|
table = get_table(transform_options, "table")
|
|
88
93
|
extract_options = dict(extract_options or dict(), variables=dict(client_id=client_id, client_secret=client_secret))
|
|
89
94
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
90
|
-
return run_with_duckdb(get_module(".search"), "CafeSearch", "CafeSearch", how, table, return_type, args, **options)
|
|
95
|
+
return run_with_duckdb(get_module(".search"), "CafeSearch", "CafeSearch", connection, how, table, return_type, args, **options)
|
|
91
96
|
|
|
92
97
|
|
|
93
98
|
def search_kin(
|
|
@@ -97,8 +102,9 @@ def search_kin(
|
|
|
97
102
|
start: int | Iterable[int] = 1,
|
|
98
103
|
display: int = 100,
|
|
99
104
|
sort: Literal["sim","date","point"] = "sim",
|
|
105
|
+
connection: DuckDBConnection | None = None,
|
|
100
106
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
101
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
107
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
102
108
|
extract_options: dict | None = None,
|
|
103
109
|
transform_options: dict | None = None,
|
|
104
110
|
) -> JsonObject:
|
|
@@ -106,7 +112,7 @@ def search_kin(
|
|
|
106
112
|
table = get_table(transform_options, "table")
|
|
107
113
|
extract_options = dict(extract_options or dict(), variables=dict(client_id=client_id, client_secret=client_secret))
|
|
108
114
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
109
|
-
return run_with_duckdb(get_module(".search"), "KiNSearch", "KiNSearch", how, table, return_type, args, **options)
|
|
115
|
+
return run_with_duckdb(get_module(".search"), "KiNSearch", "KiNSearch", connection, how, table, return_type, args, **options)
|
|
110
116
|
|
|
111
117
|
|
|
112
118
|
def search_image(
|
|
@@ -117,8 +123,9 @@ def search_image(
|
|
|
117
123
|
display: int = 100,
|
|
118
124
|
sort: Literal["sim","date"] = "sim",
|
|
119
125
|
filter: Literal["all","large","medium","small"] = "all",
|
|
126
|
+
connection: DuckDBConnection | None = None,
|
|
120
127
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
121
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
128
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
122
129
|
extract_options: dict | None = None,
|
|
123
130
|
transform_options: dict | None = None,
|
|
124
131
|
) -> JsonObject:
|
|
@@ -126,7 +133,7 @@ def search_image(
|
|
|
126
133
|
table = get_table(transform_options, "table")
|
|
127
134
|
extract_options = dict(extract_options or dict(), variables=dict(client_id=client_id, client_secret=client_secret))
|
|
128
135
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
129
|
-
return run_with_duckdb(get_module(".search"), "ImageSearch", "ImageSearch", how, table, return_type, args, **options)
|
|
136
|
+
return run_with_duckdb(get_module(".search"), "ImageSearch", "ImageSearch", connection, how, table, return_type, args, **options)
|
|
130
137
|
|
|
131
138
|
|
|
132
139
|
def search_shop(
|
|
@@ -136,8 +143,9 @@ def search_shop(
|
|
|
136
143
|
start: int | Iterable[int] = 1,
|
|
137
144
|
display: int = 100,
|
|
138
145
|
sort: Literal["sim","date","asc","dsc"] = "sim",
|
|
146
|
+
connection: DuckDBConnection | None = None,
|
|
139
147
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
140
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
148
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
141
149
|
extract_options: dict | None = None,
|
|
142
150
|
transform_options: dict | None = None,
|
|
143
151
|
) -> JsonObject:
|
|
@@ -145,7 +153,7 @@ def search_shop(
|
|
|
145
153
|
table = get_table(transform_options, "table")
|
|
146
154
|
extract_options = dict(extract_options or dict(), variables=dict(client_id=client_id, client_secret=client_secret))
|
|
147
155
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
148
|
-
return run_with_duckdb(get_module(".search"), "ShoppingSearch", "ShoppingSearch", how, table, return_type, args, **options)
|
|
156
|
+
return run_with_duckdb(get_module(".search"), "ShoppingSearch", "ShoppingSearch", connection, how, table, return_type, args, **options)
|
|
149
157
|
|
|
150
158
|
|
|
151
159
|
def rank_shop(
|
|
@@ -155,8 +163,9 @@ def rank_shop(
|
|
|
155
163
|
start: int | Iterable[int] = 1,
|
|
156
164
|
display: int = 100,
|
|
157
165
|
sort: Literal["sim","date","asc","dsc"] = "sim",
|
|
166
|
+
connection: DuckDBConnection | None = None,
|
|
158
167
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
159
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
168
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
160
169
|
extract_options: dict | None = None,
|
|
161
170
|
transform_options: dict | None = None,
|
|
162
171
|
) -> dict[str,JsonObject]:
|
|
@@ -164,4 +173,4 @@ def rank_shop(
|
|
|
164
173
|
table = [get_table(transform_options, "rank_table"), get_table(transform_options, "product_table", "product")]
|
|
165
174
|
extract_options = dict(extract_options or dict(), variables=dict(client_id=client_id, client_secret=client_secret))
|
|
166
175
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
167
|
-
return run_with_duckdb(get_module(".search"), "ShoppingRank", "ShoppingRank", how, table, return_type, args, **options)
|
|
176
|
+
return run_with_duckdb(get_module(".search"), "ShoppingRank", "ShoppingRank", connection, how, table, return_type, args, **options)
|
|
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING
|
|
|
8
8
|
if TYPE_CHECKING:
|
|
9
9
|
from typing import Iterable, Literal
|
|
10
10
|
from linkmerce.common.extract import JsonObject
|
|
11
|
+
from linkmerce.common.load import DuckDBConnection
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
def get_module(name: str) -> str:
|
|
@@ -21,7 +22,8 @@ def exposure_diagnosis(
|
|
|
21
22
|
domain: Literal["search","shopping"] = "search",
|
|
22
23
|
mobile: bool = True,
|
|
23
24
|
is_own: bool | None = None,
|
|
24
|
-
|
|
25
|
+
connection: DuckDBConnection | None = None,
|
|
26
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
25
27
|
extract_options: dict | None = None,
|
|
26
28
|
transform_options: dict | None = None,
|
|
27
29
|
) -> JsonObject:
|
|
@@ -29,7 +31,7 @@ def exposure_diagnosis(
|
|
|
29
31
|
table = get_table(transform_options, "table")
|
|
30
32
|
extract_options = dict(extract_options or dict(), headers=dict(cookies=cookies), variables=dict(customer_id=customer_id))
|
|
31
33
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
32
|
-
return run_with_duckdb(get_module(".exposure"), "ExposureDiagnosis", "ExposureDiagnosis", "sync", table, return_type, args, **options)
|
|
34
|
+
return run_with_duckdb(get_module(".exposure"), "ExposureDiagnosis", "ExposureDiagnosis", connection, "sync", table, return_type, args, **options)
|
|
33
35
|
|
|
34
36
|
|
|
35
37
|
def exposure_rank(
|
|
@@ -39,7 +41,8 @@ def exposure_rank(
|
|
|
39
41
|
domain: Literal["search","shopping"] = "search",
|
|
40
42
|
mobile: bool = True,
|
|
41
43
|
is_own: bool | None = None,
|
|
42
|
-
|
|
44
|
+
connection: DuckDBConnection | None = None,
|
|
45
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
43
46
|
extract_options: dict | None = None,
|
|
44
47
|
transform_options: dict | None = None,
|
|
45
48
|
) -> JsonObject:
|
|
@@ -47,4 +50,4 @@ def exposure_rank(
|
|
|
47
50
|
table = [get_table(transform_options, "rank_table"), get_table(transform_options, "product_table", "product")]
|
|
48
51
|
extract_options = dict(extract_options or dict(), headers=dict(cookies=cookies), variables=dict(customer_id=customer_id))
|
|
49
52
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
50
|
-
return run_with_duckdb(get_module(".exposure"), "ExposureRank", "ExposureRank", "sync", table, return_type, args, **options)
|
|
53
|
+
return run_with_duckdb(get_module(".exposure"), "ExposureRank", "ExposureRank", connection, "sync", table, return_type, args, **options)
|
|
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING
|
|
|
8
8
|
if TYPE_CHECKING:
|
|
9
9
|
from typing import Iterable, Literal
|
|
10
10
|
from linkmerce.common.extract import JsonObject
|
|
11
|
+
from linkmerce.common.load import DuckDBConnection
|
|
11
12
|
import datetime as dt
|
|
12
13
|
|
|
13
14
|
|
|
@@ -22,8 +23,9 @@ def brand_catalog(
|
|
|
22
23
|
is_brand_catalog: bool | None = None,
|
|
23
24
|
page: int | list[int] | None = 0,
|
|
24
25
|
page_size: int = 100,
|
|
26
|
+
connection: DuckDBConnection | None = None,
|
|
25
27
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
26
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
28
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
27
29
|
extract_options: dict | None = None,
|
|
28
30
|
transform_options: dict | None = None,
|
|
29
31
|
) -> JsonObject:
|
|
@@ -31,7 +33,7 @@ def brand_catalog(
|
|
|
31
33
|
table = get_table(transform_options, "table")
|
|
32
34
|
extract_options = dict(extract_options or dict(), headers=dict(cookies=cookies))
|
|
33
35
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
34
|
-
return run_with_duckdb(get_module(".catalog"), "BrandCatalog", "BrandCatalog", how, table, return_type, args, **options)
|
|
36
|
+
return run_with_duckdb(get_module(".catalog"), "BrandCatalog", "BrandCatalog", connection, how, table, return_type, args, **options)
|
|
35
37
|
|
|
36
38
|
|
|
37
39
|
def brand_product(
|
|
@@ -42,8 +44,9 @@ def brand_product(
|
|
|
42
44
|
is_brand_catalog: bool | None = None,
|
|
43
45
|
page: int | list[int] | None = 0,
|
|
44
46
|
page_size: int = 100,
|
|
47
|
+
connection: DuckDBConnection | None = None,
|
|
45
48
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
46
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
49
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
47
50
|
extract_options: dict | None = None,
|
|
48
51
|
transform_options: dict | None = None,
|
|
49
52
|
) -> JsonObject:
|
|
@@ -51,7 +54,7 @@ def brand_product(
|
|
|
51
54
|
table = get_table(transform_options, "table")
|
|
52
55
|
extract_options = dict(extract_options or dict(), headers=dict(cookies=cookies))
|
|
53
56
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
54
|
-
return run_with_duckdb(get_module(".catalog"), "BrandProduct", "BrandProduct", how, table, return_type, args, **options)
|
|
57
|
+
return run_with_duckdb(get_module(".catalog"), "BrandProduct", "BrandProduct", connection, how, table, return_type, args, **options)
|
|
55
58
|
|
|
56
59
|
|
|
57
60
|
def brand_price(
|
|
@@ -62,8 +65,9 @@ def brand_price(
|
|
|
62
65
|
is_brand_catalog: bool | None = None,
|
|
63
66
|
page: int | list[int] | None = 0,
|
|
64
67
|
page_size: int = 100,
|
|
68
|
+
connection: DuckDBConnection | None = None,
|
|
65
69
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
66
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
70
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
67
71
|
extract_options: dict | None = None,
|
|
68
72
|
transform_options: dict | None = None,
|
|
69
73
|
) -> JsonObject:
|
|
@@ -71,7 +75,28 @@ def brand_price(
|
|
|
71
75
|
table = [get_table(transform_options, "price_table"), get_table(transform_options, "product_table", "product")]
|
|
72
76
|
extract_options = dict(extract_options or dict(), headers=dict(cookies=cookies))
|
|
73
77
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
74
|
-
return run_with_duckdb(get_module(".catalog"), "BrandPrice", "BrandPrice", how, table, return_type, args, **options)
|
|
78
|
+
return run_with_duckdb(get_module(".catalog"), "BrandPrice", "BrandPrice", connection, how, table, return_type, args, **options)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def match_catalog(
|
|
82
|
+
cookies: str,
|
|
83
|
+
brand_ids: str | Iterable[str],
|
|
84
|
+
mall_seq: int | str | Iterable[int | str],
|
|
85
|
+
sort_type: Literal["popular","recent","price"] = "recent",
|
|
86
|
+
is_brand_catalog: bool | None = None,
|
|
87
|
+
page: int | list[int] | None = 0,
|
|
88
|
+
page_size: int = 100,
|
|
89
|
+
connection: DuckDBConnection | None = None,
|
|
90
|
+
how: Literal["sync","async","async_loop"] = "sync",
|
|
91
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
92
|
+
extract_options: dict | None = None,
|
|
93
|
+
transform_options: dict | None = None,
|
|
94
|
+
) -> JsonObject:
|
|
95
|
+
args = (brand_ids, mall_seq, sort_type, is_brand_catalog, page, page_size)
|
|
96
|
+
table = get_table(transform_options, "table")
|
|
97
|
+
extract_options = dict(extract_options or dict(), headers=dict(cookies=cookies))
|
|
98
|
+
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
99
|
+
return run_with_duckdb(get_module(".catalog"), "MatchCatalog", "MatchCatalog", connection, how, table, return_type, args, **options)
|
|
75
100
|
|
|
76
101
|
|
|
77
102
|
def store_sales(
|
|
@@ -82,8 +107,9 @@ def store_sales(
|
|
|
82
107
|
date_type: Literal["daily","weekly","monthly"] = "daily",
|
|
83
108
|
page: int | Iterable[int] = 1,
|
|
84
109
|
page_size: int = 1000,
|
|
110
|
+
connection: DuckDBConnection | None = None,
|
|
85
111
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
86
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
112
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
87
113
|
extract_options: dict | None = None,
|
|
88
114
|
transform_options: dict | None = None,
|
|
89
115
|
) -> JsonObject:
|
|
@@ -91,7 +117,7 @@ def store_sales(
|
|
|
91
117
|
table = get_table(transform_options, "table")
|
|
92
118
|
extract_options = dict(extract_options or dict(), headers=dict(cookies=cookies))
|
|
93
119
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
94
|
-
return run_with_duckdb(get_module(".sales"), "StoreSales", "StoreSales", how, table, return_type, args, **options)
|
|
120
|
+
return run_with_duckdb(get_module(".sales"), "StoreSales", "StoreSales", connection, how, table, return_type, args, **options)
|
|
95
121
|
|
|
96
122
|
|
|
97
123
|
def category_sales(
|
|
@@ -102,8 +128,9 @@ def category_sales(
|
|
|
102
128
|
date_type: Literal["daily","weekly","monthly"] = "daily",
|
|
103
129
|
page: int | Iterable[int] = 1,
|
|
104
130
|
page_size: int = 1000,
|
|
131
|
+
connection: DuckDBConnection | None = None,
|
|
105
132
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
106
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
133
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
107
134
|
extract_options: dict | None = None,
|
|
108
135
|
transform_options: dict | None = None,
|
|
109
136
|
) -> JsonObject:
|
|
@@ -111,7 +138,7 @@ def category_sales(
|
|
|
111
138
|
table = get_table(transform_options, "table")
|
|
112
139
|
extract_options = dict(extract_options or dict(), headers=dict(cookies=cookies))
|
|
113
140
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
114
|
-
return run_with_duckdb(get_module(".sales"), "CategorySales", "CategorySales", how, table, return_type, args, **options)
|
|
141
|
+
return run_with_duckdb(get_module(".sales"), "CategorySales", "CategorySales", connection, how, table, return_type, args, **options)
|
|
115
142
|
|
|
116
143
|
|
|
117
144
|
def product_sales(
|
|
@@ -122,8 +149,9 @@ def product_sales(
|
|
|
122
149
|
date_type: Literal["daily","weekly","monthly"] = "daily",
|
|
123
150
|
page: int | Iterable[int] = 1,
|
|
124
151
|
page_size: int = 1000,
|
|
152
|
+
connection: DuckDBConnection | None = None,
|
|
125
153
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
126
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
154
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
127
155
|
extract_options: dict | None = None,
|
|
128
156
|
transform_options: dict | None = None,
|
|
129
157
|
) -> JsonObject:
|
|
@@ -131,7 +159,7 @@ def product_sales(
|
|
|
131
159
|
table = get_table(transform_options, "table")
|
|
132
160
|
extract_options = dict(extract_options or dict(), headers=dict(cookies=cookies))
|
|
133
161
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
134
|
-
return run_with_duckdb(get_module(".sales"), "ProductSales", "ProductSales", how, table, return_type, args, **options)
|
|
162
|
+
return run_with_duckdb(get_module(".sales"), "ProductSales", "ProductSales", connection, how, table, return_type, args, **options)
|
|
135
163
|
|
|
136
164
|
|
|
137
165
|
def aggregated_sales(
|
|
@@ -142,8 +170,9 @@ def aggregated_sales(
|
|
|
142
170
|
date_type: Literal["daily","weekly","monthly"] = "daily",
|
|
143
171
|
page: int | Iterable[int] = 1,
|
|
144
172
|
page_size: int = 1000,
|
|
173
|
+
connection: DuckDBConnection | None = None,
|
|
145
174
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
146
|
-
return_type: Literal["csv","json","parquet","raw"] = "json",
|
|
175
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
147
176
|
extract_options: dict | None = None,
|
|
148
177
|
transform_options: dict | None = None,
|
|
149
178
|
) -> JsonObject:
|
|
@@ -151,4 +180,4 @@ def aggregated_sales(
|
|
|
151
180
|
table = [get_table(transform_options, "sales_table"), get_table(transform_options, "product_table", "product")]
|
|
152
181
|
extract_options = dict(extract_options or dict(), headers=dict(cookies=cookies))
|
|
153
182
|
options = dict(extract_options=extract_options, transform_options=transform_options)
|
|
154
|
-
return run_with_duckdb(get_module(".sales"), "AggregatedSales", "AggregatedSales", how, table, return_type, args, **options)
|
|
183
|
+
return run_with_duckdb(get_module(".sales"), "AggregatedSales", "AggregatedSales", connection, how, table, return_type, args, **options)
|
|
@@ -6,6 +6,7 @@ if TYPE_CHECKING:
|
|
|
6
6
|
from typing import Any, Literal, Type
|
|
7
7
|
from linkmerce.common.extract import Extractor, JsonObject
|
|
8
8
|
from linkmerce.common.transform import Transformer, DBTransformer, DuckDBTransformer
|
|
9
|
+
from linkmerce.common.load import DuckDBConnection
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
###################################################################
|
|
@@ -109,14 +110,18 @@ def run_with_duckdb(
|
|
|
109
110
|
module: str,
|
|
110
111
|
extractor: str,
|
|
111
112
|
transformer: str | None = None,
|
|
113
|
+
connection: DuckDBConnection | None = None,
|
|
112
114
|
how: Literal["sync","async","async_loop"] = "sync",
|
|
113
115
|
table: str | Sequence[str] = ":default:",
|
|
114
|
-
return_type: Literal["
|
|
116
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
115
117
|
args: tuple = tuple(),
|
|
116
118
|
kwargs: dict = dict(),
|
|
117
119
|
extract_options: dict | None = None,
|
|
118
120
|
transform_options: dict | None = None,
|
|
119
121
|
) -> Any | dict[str,Any]:
|
|
122
|
+
if connection is not None:
|
|
123
|
+
return run_with_duckdb_connection(
|
|
124
|
+
module, connection, extractor, transformer, how, table, return_type, args, kwargs, extract_options, transform_options)
|
|
120
125
|
extract_options = extract_options.copy() if isinstance(extract_options, dict) else dict()
|
|
121
126
|
has_parser = not ((return_type == "raw") or ("parser" in extract_options))
|
|
122
127
|
with import_dbt(module if has_parser else None, transformer)(**(transform_options or dict())) as transformer_:
|
|
@@ -127,6 +132,30 @@ def run_with_duckdb(
|
|
|
127
132
|
return _fetch_all_from_table(transformer_, results, table, return_type)
|
|
128
133
|
|
|
129
134
|
|
|
135
|
+
def run_with_duckdb_connection(
|
|
136
|
+
module: str,
|
|
137
|
+
connection: DuckDBConnection,
|
|
138
|
+
extractor: str,
|
|
139
|
+
transformer: str | None = None,
|
|
140
|
+
how: Literal["sync","async","async_loop"] = "sync",
|
|
141
|
+
table: str | Sequence[str] = ":default:",
|
|
142
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
143
|
+
args: tuple = tuple(),
|
|
144
|
+
kwargs: dict = dict(),
|
|
145
|
+
extract_options: dict | None = None,
|
|
146
|
+
transform_options: dict | None = None,
|
|
147
|
+
) -> Any | dict[str,Any]:
|
|
148
|
+
extract_options = extract_options.copy() if isinstance(extract_options, dict) else dict()
|
|
149
|
+
transform_options = dict(transform_options or dict(), db_info=dict(conn=connection))
|
|
150
|
+
has_parser = not ((return_type == "raw") or ("parser" in extract_options))
|
|
151
|
+
transformer_ = import_dbt(module, transformer)(**transform_options)
|
|
152
|
+
if transformer and has_parser:
|
|
153
|
+
extract_options["parser"] = transformer_.transform
|
|
154
|
+
extractor_ = import_extractor(module, extractor)(**extract_options)
|
|
155
|
+
results = _extract(extractor_, how, args, kwargs)
|
|
156
|
+
return _fetch_all_from_table(transformer_, results, table, return_type)
|
|
157
|
+
|
|
158
|
+
|
|
130
159
|
def get_table_from_options(transform_options: dict | None = None, key: str = "table", default: str = ":default:") -> str:
|
|
131
160
|
if isinstance(transform_options, dict):
|
|
132
161
|
create_options = transform_options.get("create_options")
|
|
@@ -143,14 +172,15 @@ def _fetch_all_from_table(
|
|
|
143
172
|
transformer: DuckDBTransformer,
|
|
144
173
|
results: JsonObject,
|
|
145
174
|
table: str | Sequence[str] = ":default:",
|
|
146
|
-
return_type: Literal["
|
|
175
|
+
return_type: Literal["csv","json","parquet","raw","none"] = "json",
|
|
147
176
|
) -> Any | dict[str,Any]:
|
|
177
|
+
if return_type == "none":
|
|
178
|
+
return
|
|
148
179
|
if (return_type == "raw") or (transformer.conn is None):
|
|
149
180
|
return results
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
return fetch_all(_get_table(transformer, table))
|
|
181
|
+
elif isinstance(table, str):
|
|
182
|
+
return transformer.fetch_all(return_type, _get_table(transformer, table))
|
|
153
183
|
elif isinstance(table, Sequence):
|
|
154
|
-
return {_get_table(transformer, table_): fetch_all(table_) for table_ in table}
|
|
184
|
+
return {_get_table(transformer, table_): transformer.fetch_all(return_type, table_) for table_ in table}
|
|
155
185
|
else:
|
|
156
186
|
raise TypeError("Invalid type for table. A string or sequence type is allowed.")
|
|
@@ -7,6 +7,7 @@ from typing import Any, Callable, Hashable, IO, TypeVar, Union, overload, TYPE_C
|
|
|
7
7
|
|
|
8
8
|
if TYPE_CHECKING:
|
|
9
9
|
from typing import Coroutine, Literal, Sequence
|
|
10
|
+
|
|
10
11
|
from linkmerce.common.tasks import TaskOption, TaskOptions
|
|
11
12
|
from linkmerce.common.tasks import RequestLoop, RequestEach, RequestEachLoop
|
|
12
13
|
from linkmerce.common.tasks import PaginateAll, RequestEachPages
|
|
@@ -18,9 +19,9 @@ if TYPE_CHECKING:
|
|
|
18
19
|
|
|
19
20
|
_KT = TypeVar("_KT", bound=Hashable)
|
|
20
21
|
_VT = TypeVar("_VT", bound=Any)
|
|
21
|
-
|
|
22
22
|
Headers = dict[_KT,_VT]
|
|
23
23
|
Variables = dict[_KT,_VT]
|
|
24
|
+
|
|
24
25
|
JsonObject = Union[dict, list]
|
|
25
26
|
JsonSerialize = Union[dict, list, bytes, IO]
|
|
26
27
|
ResponseComponent = Union[int, bytes, str, JsonObject, dict[str,str]]
|