scraper2-hj3415 0.1.0__py3-none-any.whl → 0.1.5__py3-none-any.whl

scraper2_hj3415/cli.py

@@ -22,12 +22,16 @@ def nfscraper():
                 run.c101([args.target, ], args.db_path) if args.db_path else run.c101([args.target, ])
         if args.spider == 'c103':
             if args.target == 'all':
-                print(f"The option 'all' isn't allow on {args.spider}.")
+                x = input("It will take a long time. Are you sure? (y/N)")
+                if x == 'y' or x == 'Y':
+                    run.c103(krx.get_codes(), args.db_path) if args.db_path else run.c103(krx.get_codes())
             elif utils.is_6digit(args.target):
                 run.c103([args.target, ], args.db_path) if args.db_path else run.c103([args.target, ])
         if args.spider == 'c104':
             if args.target == 'all':
-                print(f"The option 'all' isn't allow on {args.spider}.")
+                x = input("It will take a long time. Are you sure? (y/N)")
+                if x == 'y' or x == 'Y':
+                    run.c104(krx.get_codes(), args.db_path) if args.db_path else run.c104(krx.get_codes())
             elif utils.is_6digit(args.target):
                 run.c104([args.target, ], args.db_path) if args.db_path else run.c104([args.target, ])
         if args.spider == 'c106':

scraper2_hj3415/nfscrapy/nfs/pipelines.py

@@ -1,5 +1,4 @@
 from itemadapter import ItemAdapter
-from pymongo import errors
 from pprint import pprint
 from util_hj3415 import mongo
 
@@ -9,7 +8,7 @@ formatter = logging.Formatter('%(levelname)s: [%(name)s] %(message)s')
 ch = logging.StreamHandler()
 ch.setFormatter(formatter)
 logger.addHandler(ch)
-logger.setLevel(logging.DEBUG)
+logger.setLevel(logging.WARNING)
 
 
 class ValidationPipeline:
@@ -22,9 +21,9 @@ class ValidationPipeline:
             # eps, bps, per, pbr을 직접 계산해서 바꾸기 위해 c104 page를 찾는다.
             try:
                 logger.debug('Try to get c104 page for calculate values..')
-                c104 = mongo.C104(spider.mongo_client, item['코드'], 'c104q')
-                d, eps = c104.sum_recent_4q('EPS')  # 최근 4분기 eps값을 더한다.
-                d, bps = c104.latest_value('BPS')  # 마지막 분기 bps값을 찾는다.
+                c104q_data = mongo.C104(spider.mongo_client, item['코드'], 'c104q')
+                d, eps = c104q_data.sum_recent_4q('EPS')  # 최근 4분기 eps값을 더한다.
+                d, bps = c104q_data.latest_value('BPS')  # 마지막 분기 bps값을 찾는다.
 
                 # per, pbr을 구하는 람다함수
                 cal_ratio = (lambda eps_bps, pprice:
@@ -41,13 +40,13 @@ class ValidationPipeline:
             item['EPS'], item['BPS'], item['PER'], item['PBR'] = eps, bps, cal_per, cal_pbr
         if 'c103' in spider.name:
             # pprint(item)
-            print(" Nothing special working")
+            print(" Nothing special for working")
         if 'c104' in spider.name:
-            #pprint(item)
-            print(" Nothing special working")
+            # pprint(item)
+            print(" Nothing special for working")
         if spider.name == 'c106':
             pprint(item)
-            print(" Nothing special working")
+            print(" Nothing special for working")
         return item
 
 
@@ -61,19 +60,12 @@ class MongoPipeline:
             if spider.name == 'c101':
                 page = spider.name
                 print(f" code : {item['코드']} / page : {page}")
-                c101_col = spider.mongo_client[item['코드']][page]
-                try:
-                    c101_col.create_index('date', unique=True)
-                    c101_col.insert_one(ItemAdapter(item).asdict())
-                except errors.DuplicateKeyError:
-                    # 스크랩한 데이터가 이미 데이터베이스에 있을경우 지우고 저장
-                    c101_col.delete_many({'date': {"$gte": item['date']}})
-                    c101_col.insert_one(ItemAdapter(item).asdict())
+                mongo.C101(spider.mongo_client, item['코드']).save_dict(ItemAdapter(item).asdict())
             elif 'c103' in spider.name:
                 page = ''.join(['c103', item['title']])
                 print(f" code : {item['코드']} / page : {page}")
-                #logging.debug(item['df'].to_dict('records'))
-                #r = mongo2.C103(self.client, item['코드'], page).save(item['df'])
+                logging.debug(item['df'].to_dict('records'))
+                mongo.C103(spider.mongo_client, item['코드'], page).save_df(item['df'])
             elif 'c104' in spider.name:
                 if item['title'].endswith('y'):
                     page = 'c104y'
@@ -83,7 +75,7 @@ class MongoPipeline:
                     raise ValueError
                 print(f" code : {item['코드']} / page : {page}({item['title']})")
                 logging.debug(item['df'].to_dict('records'))
-                r = mongo.C104(spider.mongo_client, item['코드'], page).save_df(item['df'])
+                mongo.C104(spider.mongo_client, item['코드'], page).save_df(item['df'])
             elif spider.name == 'c106':
                 page = ''.join(['c106', item['title']])
                 print(f" code : {item['코드']} / page : {page}")

scraper2_hj3415/nfscrapy/run.py

@@ -178,7 +178,10 @@ def c103(codes: list, mongo_addr: str = ""):
     :param mongo_addr: 몽고데이터베이스 URI - mongodb://...
     :return:
     """
-    _mp_c1034('c103', codes=codes, mongo_addr=mongo_addr)
+    if utils.get_driver() is None:
+        print("Fail to get chrome driver...Try again", file=sys.stderr)
+    else:
+        _mp_c1034('c103', codes=codes, mongo_addr=mongo_addr)
 
 
 @chcwd
@@ -189,7 +192,10 @@ def c104(codes: list, mongo_addr: str = ""):
     :param mongo_addr: 몽고데이터베이스 URI - mongodb://...
     :return:
     """
-    _mp_c1034('c104', codes=codes, mongo_addr=mongo_addr)
+    if utils.get_driver() is None:
+        print("Fail to get chrome driver...Try again", file=sys.stderr)
+    else:
+        _mp_c1034('c104', codes=codes, mongo_addr=mongo_addr)
 
 
 @chcwd

{scraper2_hj3415-0.1.0.dist-info → scraper2_hj3415-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scraper2_hj3415
-Version: 0.1.0
+Version: 0.1.5
 Summary: Gathering the stock data
 Project-URL: Homepage, https://pypi.org/project/scraper2_hj3415/
 Project-URL: Bug Tracker, https://pypi.org/project/scraper2_hj3415/
@@ -10,7 +10,6 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
-Requires-Dist: pymongo
 Requires-Dist: scrapy
 Requires-Dist: util-hj3415
 Description-Content-Type: text/markdown

{scraper2_hj3415-0.1.0.dist-info → scraper2_hj3415-0.1.5.dist-info}/RECORD

@@ -1,5 +1,5 @@
 scraper2_hj3415/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scraper2_hj3415/cli.py,sha256=1SYXslGOL0lo2tucFTPVnK85DcJT1lXjfeJNUmf5Vk0,1890
+scraper2_hj3415/cli.py,sha256=bPM5XAqT7VcVON_fclHPoBhq6jjJXvN0Ywf0KRoOdM0,2194
 scraper2_hj3415/miscrapy/run.py,sha256=ZBnHJHyUdPVcKdBK7gTyPFY9-9VIJYzhflscrowUE3k,5323
 scraper2_hj3415/miscrapy/scrapy.cfg,sha256=KvMPqIfrkMIydPYMPb0fN9m_IBpr61N7ctwCwudcF2s,247
 scraper2_hj3415/miscrapy/mi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -21,12 +21,12 @@ scraper2_hj3415/miscrapy/mi/spiders/usdidx.py,sha256=TQ0zSxg7xpFiudhsgyjiJ38G3DM
 scraper2_hj3415/miscrapy/mi/spiders/usdkrw.py,sha256=BmLmOEK4HogPFZ1XOFXB_RhDyAO2vjRVC6a1bNnLbxQ,963
 scraper2_hj3415/miscrapy/mi/spiders/wti.py,sha256=Yy49k-uzpUBpAu-zXhFLCWp_ZEqWXiIPXtJK3PvQUGQ,1058
 scraper2_hj3415/nfscrapy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scraper2_hj3415/nfscrapy/run.py,sha256=Esa1yKB4ZEZ_H6Cq7v7lVC0LuzDnCvrkSEzL89n3fKg,6849
+scraper2_hj3415/nfscrapy/run.py,sha256=VpvFzwoWXzhCjA_2ZPY0VTTkbxjipemz2oQniv1SNGI,7097
 scraper2_hj3415/nfscrapy/scrapy.cfg,sha256=yCkEgpzAwc9NWjYGaEUelGdLg3mUuuQF1Zl0k5vITH8,260
 scraper2_hj3415/nfscrapy/nfs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 scraper2_hj3415/nfscrapy/nfs/items.py,sha256=n9xDKoGez8oRmsRLSDjjPhLCXgeJGUsKydjWc_gzgbk,1479
 scraper2_hj3415/nfscrapy/nfs/middlewares.py,sha256=F3kL0OJMsUhiw2mPIxQLGvY3wdMxIsQl1BG2uwo_ZHo,3745
-scraper2_hj3415/nfscrapy/nfs/pipelines.py,sha256=I_co-PfJdUUsvw461s5Kroc1YGCPlEuPOMjptYldzBQ,5264
+scraper2_hj3415/nfscrapy/nfs/pipelines.py,sha256=d1MhW0b3ijAbRkF3aumz7DxZec1qDKaXkC4kOSiWa8w,4844
 scraper2_hj3415/nfscrapy/nfs/settings.py,sha256=BkOy913N9K8Ypwuj0GiZ0zAI_WnBcjJzwjgnZOS_jqw,3472
 scraper2_hj3415/nfscrapy/nfs/spiders/__init__.py,sha256=ULwecZkx3_NTphkz7y_qiazBeUoHFnCCWnKSjoDCZj0,161
 scraper2_hj3415/nfscrapy/nfs/spiders/_c108.py,sha256=gOgWQ7qTOOBEZQTY79K5MWzjmz-ZpFg5wrqBhzLoXjI,4776
@@ -35,8 +35,8 @@ scraper2_hj3415/nfscrapy/nfs/spiders/c103.py,sha256=5E7jZX5UIASCtqGa0eikJVcwlKQs
 scraper2_hj3415/nfscrapy/nfs/spiders/c104.py,sha256=usz29mBeXiQrv_e7uDf2ZihbENVsafEUHUJw8aHRwlw,7645
 scraper2_hj3415/nfscrapy/nfs/spiders/c106.py,sha256=cMbpSLqG4EanKzh6rjpFLAnECt_zLR34MTZo34xu6Go,3783
 scraper2_hj3415/nfscrapy/nfs/spiders/common.py,sha256=QTVWFF386cvoN2czFJQpTeE5jRgnlSE_Z-7y13tPyi8,2876
-scraper2_hj3415-0.1.0.dist-info/METADATA,sha256=kD7GNDR_Qdmcv4qv8PgfqmpJBaW9hg8QrhNzlMdiosw,1154
-scraper2_hj3415-0.1.0.dist-info/WHEEL,sha256=Fd6mP6ydyRguakwUJ05oBE7fh2IPxgtDN9IwHJ9OqJQ,87
-scraper2_hj3415-0.1.0.dist-info/entry_points.txt,sha256=1bGP38AtuY6n2FcP_fLmpqGtFmFf8NLJWsCSWoTjF-0,60
-scraper2_hj3415-0.1.0.dist-info/licenses/LICENSE,sha256=6kbiFSfobTZ7beWiKnHpN902HgBx-Jzgcme0SvKqhKY,1091
-scraper2_hj3415-0.1.0.dist-info/RECORD,,
+scraper2_hj3415-0.1.5.dist-info/METADATA,sha256=Z66j2xuOhAAsDk4g77kTG9B6fWL5JWpiSHZnFF7EYJo,1131
+scraper2_hj3415-0.1.5.dist-info/WHEEL,sha256=Fd6mP6ydyRguakwUJ05oBE7fh2IPxgtDN9IwHJ9OqJQ,87
+scraper2_hj3415-0.1.5.dist-info/entry_points.txt,sha256=1bGP38AtuY6n2FcP_fLmpqGtFmFf8NLJWsCSWoTjF-0,60
+scraper2_hj3415-0.1.5.dist-info/licenses/LICENSE,sha256=6kbiFSfobTZ7beWiKnHpN902HgBx-Jzgcme0SvKqhKY,1091
+scraper2_hj3415-0.1.5.dist-info/RECORD,,