scraper2-hj3415 2.3.0__py3-none-any.whl → 2.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scraper2/app/composition.py +6 -14
- scraper2/main.py +2 -20
- {scraper2_hj3415-2.3.0.dist-info → scraper2_hj3415-2.4.1.dist-info}/METADATA +1 -1
- {scraper2_hj3415-2.3.0.dist-info → scraper2_hj3415-2.4.1.dist-info}/RECORD +7 -7
- {scraper2_hj3415-2.3.0.dist-info → scraper2_hj3415-2.4.1.dist-info}/WHEEL +0 -0
- {scraper2_hj3415-2.3.0.dist-info → scraper2_hj3415-2.4.1.dist-info}/entry_points.txt +0 -0
- {scraper2_hj3415-2.3.0.dist-info → scraper2_hj3415-2.4.1.dist-info}/licenses/LICENSE +0 -0
scraper2/app/composition.py
CHANGED
|
@@ -112,21 +112,11 @@ class Usecases:
|
|
|
112
112
|
browser_factory: Optional[BrowserFactoryPort] = None
|
|
113
113
|
|
|
114
114
|
async def aclose(self) -> None:
|
|
115
|
-
# 1) playwright 먼저 닫기 (subprocess 정리)
|
|
116
115
|
if self.browser_factory is not None:
|
|
117
|
-
|
|
118
|
-
if callable(close):
|
|
119
|
-
await close()
|
|
116
|
+
await self.browser_factory.aclose()
|
|
120
117
|
|
|
121
|
-
# 2) mongo 닫기
|
|
122
118
|
if self.mongo is not None:
|
|
123
|
-
|
|
124
|
-
if callable(close):
|
|
125
|
-
out = close()
|
|
126
|
-
if hasattr(out, "__await__"):
|
|
127
|
-
await out
|
|
128
|
-
|
|
129
|
-
|
|
119
|
+
await self.mongo.close()
|
|
130
120
|
|
|
131
121
|
# -------------------------
|
|
132
122
|
# builders
|
|
@@ -202,11 +192,13 @@ def build_usecases(
|
|
|
202
192
|
if sink_kind == "memory":
|
|
203
193
|
bundle = build_memory_bundle()
|
|
204
194
|
ingest = build_ingest_usecases(fetch=fetch, sinks=bundle.sinks)
|
|
205
|
-
return Usecases(fetch=fetch, ingest=ingest, sinks=bundle.sinks, store=bundle.store
|
|
195
|
+
return Usecases(fetch=fetch, ingest=ingest, sinks=bundle.sinks, store=bundle.store,
|
|
196
|
+
browser_factory=factory)
|
|
206
197
|
|
|
207
198
|
if sink_kind == "mongo":
|
|
208
199
|
bundle = build_mongo_bundle()
|
|
209
200
|
ingest = build_ingest_usecases(fetch=fetch, sinks=bundle.sinks)
|
|
210
|
-
return Usecases(fetch=fetch, ingest=ingest, sinks=bundle.sinks, mongo=bundle.mongo, db=bundle.db
|
|
201
|
+
return Usecases(fetch=fetch, ingest=ingest, sinks=bundle.sinks, mongo=bundle.mongo, db=bundle.db,
|
|
202
|
+
browser_factory=factory)
|
|
211
203
|
|
|
212
204
|
raise ValueError(f"Unknown sink_kind: {sink_kind}")
|
scraper2/main.py
CHANGED
|
@@ -33,16 +33,6 @@ app.add_typer(mi_app, name="mi")
|
|
|
33
33
|
def _endpoint_list(ep: Endpoint) -> list[str]:
|
|
34
34
|
return ["c101", "c103", "c104", "c106", "c108"] if ep == "all" else [ep]
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
async def _maybe_await_close(obj: Any) -> None:
|
|
38
|
-
close = getattr(obj, "close", None)
|
|
39
|
-
if close is None:
|
|
40
|
-
return
|
|
41
|
-
out = close()
|
|
42
|
-
if asyncio.iscoroutine(out):
|
|
43
|
-
await out
|
|
44
|
-
|
|
45
|
-
|
|
46
36
|
async def _mongo_bootstrap(db) -> None:
|
|
47
37
|
from db2.nfs import ensure_indexes
|
|
48
38
|
from db2.settings import get_settings
|
|
@@ -233,11 +223,7 @@ def nfs_one(
|
|
|
233
223
|
else:
|
|
234
224
|
typer.echo(_dto_to_pretty(dto))
|
|
235
225
|
finally:
|
|
236
|
-
|
|
237
|
-
if callable(close):
|
|
238
|
-
await close()
|
|
239
|
-
elif getattr(ucs, "mongo", None) is not None:
|
|
240
|
-
await _maybe_await_close(ucs.mongo)
|
|
226
|
+
await ucs.aclose()
|
|
241
227
|
|
|
242
228
|
asyncio.run(_run())
|
|
243
229
|
|
|
@@ -284,11 +270,7 @@ def nfs_all(
|
|
|
284
270
|
progress_every=1, # chunk마다
|
|
285
271
|
)
|
|
286
272
|
finally:
|
|
287
|
-
|
|
288
|
-
if callable(close):
|
|
289
|
-
await close()
|
|
290
|
-
elif getattr(ucs, "mongo", None) is not None:
|
|
291
|
-
await _maybe_await_close(ucs.mongo)
|
|
273
|
+
await ucs.aclose()
|
|
292
274
|
|
|
293
275
|
asyncio.run(_run())
|
|
294
276
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
scraper2/.DS_Store,sha256=sEtRhy6uiX4PgYuHnRIsUN_QtI0jR0lSLi4kYurHsso,6148
|
|
2
2
|
scraper2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
scraper2/main.py,sha256=
|
|
3
|
+
scraper2/main.py,sha256=Vwelc73P3WVbjwKFTXXA20P-nA7RoUE6hEq2oD0E2WU,8873
|
|
4
4
|
scraper2/adapters/out/.DS_Store,sha256=nUqwRB5F2DM82P8BALYvDI0YoD1UbmngfSi8ukKkY7E,6148
|
|
5
5
|
scraper2/adapters/out/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
6
|
scraper2/adapters/out/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -22,7 +22,7 @@ scraper2/adapters/out/sinks/mongo/c104_mongo_sink.py,sha256=IicVnc2fyeBXoBbgMasB
|
|
|
22
22
|
scraper2/adapters/out/sinks/mongo/c106_mongo_sink.py,sha256=FMdCp8WVjPwidnh7tIPUoViQWr48O16xtB34O_iCtJI,1203
|
|
23
23
|
scraper2/adapters/out/sinks/mongo/c108_mongo_sink.py,sha256=eSvIRtofWvNKVPchglwL1mOw5hsKDpUfNz5EOum-H3Y,1203
|
|
24
24
|
scraper2/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
25
|
-
scraper2/app/composition.py,sha256=
|
|
25
|
+
scraper2/app/composition.py,sha256=sJuEAPg-mQ51-0ZsXmMOuAN_whKaq1BCMWHy0uAwxMA,6572
|
|
26
26
|
scraper2/app/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
27
27
|
scraper2/app/parsing/_converters.py,sha256=z0kSL4nQnqq5w7QfJZBSbjZOLo1JhoqqDLpqlEAN4bo,1815
|
|
28
28
|
scraper2/app/parsing/_normalize.py,sha256=2qqbXxTbzbuYlu7ttzQjyKgatFnwopme2b_fd0zahio,3738
|
|
@@ -56,8 +56,8 @@ scraper2/app/usecases/ingest/ingest_c103.py,sha256=T_IKs5ikckin_05FnL-dxirAW1PPa
|
|
|
56
56
|
scraper2/app/usecases/ingest/ingest_c104.py,sha256=2rGTcFbsATsn3d2KsSEkL5x4fmkGo7x9MHrysoxiICM,1150
|
|
57
57
|
scraper2/app/usecases/ingest/ingest_c106.py,sha256=mQrbASbKVUQyVcIWiXzIczbX-1mNR5NGk760enVCfvo,1190
|
|
58
58
|
scraper2/app/usecases/ingest/ingest_c108.py,sha256=49ULzdl0dN6z3istAKg29PcD5wHTxYholqAZiIEmUzU,1191
|
|
59
|
-
scraper2_hj3415-2.
|
|
60
|
-
scraper2_hj3415-2.
|
|
61
|
-
scraper2_hj3415-2.
|
|
62
|
-
scraper2_hj3415-2.
|
|
63
|
-
scraper2_hj3415-2.
|
|
59
|
+
scraper2_hj3415-2.4.1.dist-info/entry_points.txt,sha256=jUNx7ZJQedQ3QnsDN1ompQ0PjwdvVmnKdHHFMfQQPlI,46
|
|
60
|
+
scraper2_hj3415-2.4.1.dist-info/licenses/LICENSE,sha256=QBiVGQuKAESeCfQE344Ik2ex6g2zfYdu9WqrRWydxIs,1068
|
|
61
|
+
scraper2_hj3415-2.4.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
|
|
62
|
+
scraper2_hj3415-2.4.1.dist-info/METADATA,sha256=NDBmtiIxLt2b5RsScBZdBiAoNTthZayM0p9zvwV4GO4,3457
|
|
63
|
+
scraper2_hj3415-2.4.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|