scraper2-hj3415 0.2.2__py3-none-any.whl → 0.2.5__py3-none-any.whl

scraper2_hj3415/nfscrapy/nfs/spiders/c103.py

@@ -1,3 +1,4 @@
+import logging
 import time
 import scrapy
 from util_hj3415 import utils
@@ -46,22 +47,24 @@ class C103Spider(scrapy.Spider, metaclass=ABCMeta):
 
     def parse_c103(self, response, code):
         # 페이지를 먼저 한번 호출하여 버튼을 눌러 세팅한다.
-        self.setting_page(response.url)
-
-        # html에서 table을 추출하여 dataframe생성
-        self.driver.get(response.url)
-        time.sleep(self.WAIT)
-        html = Selector(text=self.driver.page_source)
-        table_xpath = '//table[2]'
-        df = common.get_df_from_html(html, table_xpath, 1)
-        self.logger.debug(df)
-
-        # make item to yield
-        item = items.C103items()
-        item['코드'] = code
-        item['title'] = self.title
-        item['df'] = df
-        yield item
+        if self.setting_page(response.url):
+            # html에서 table을 추출하여 dataframe생성
+            self.driver.get(response.url)
+            time.sleep(self.WAIT)
+            html = Selector(text=self.driver.page_source)
+            table_xpath = '//table[2]'
+            df = common.get_df_from_html(html, table_xpath, 1)
+            self.logger.debug(df)
+
+            # make item to yield
+            item = items.C103items()
+            item['코드'] = code
+            item['title'] = self.title
+            item['df'] = df
+            yield item
+        else:
+            self.logger.warning("Pasing error ... maybe 올바른 종목이 아닙니다.")
+
 
     @abstractmethod
     def setting_page(self, url: str):
@@ -90,13 +93,13 @@ class C103BQ(C103Spider):
     def __init__(self, code, mongo_client):
         super(C103BQ, self).__init__(code, mongo_client, title='재무상태표q')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('재무상태표', '//*[@id="rpt_tab2"]'),
             ('분기', '//*[@id="frqTyp1"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C103CQ(C103Spider):
@@ -105,13 +108,13 @@ class C103CQ(C103Spider):
     def __init__(self, code, mongo_client):
         super().__init__(code, mongo_client, title='현금흐름표q')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('현금흐름표', '//*[@id="rpt_tab3"]'),
             ('분기', '//*[@id="frqTyp1"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C103IQ(C103Spider):
@@ -120,13 +123,13 @@ class C103IQ(C103Spider):
     def __init__(self, code, mongo_client):
         super().__init__(code, mongo_client, title='손익계산서q')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('손익계산서', '//*[@id="rpt_tab1"]'),
             ('분기', '//*[@id="frqTyp1"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C103BY(C103Spider):
@@ -135,13 +138,13 @@ class C103BY(C103Spider):
     def __init__(self, code, mongo_client):
         super().__init__(code, mongo_client, title='재무상태표y')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('재무상태표', '//*[@id="rpt_tab2"]'),
             ('연간', '//*[@id="frqTyp0"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C103CY(C103Spider):
@@ -150,13 +153,13 @@ class C103CY(C103Spider):
     def __init__(self, code, mongo_client):
         super().__init__(code, mongo_client, title='현금흐름표y')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('현금흐름표', '//*[@id="rpt_tab3"]'),
             ('연간', '//*[@id="frqTyp0"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C103IY(C103Spider):
@@ -165,10 +168,10 @@ class C103IY(C103Spider):
     def __init__(self, code, mongo_client):
         super().__init__(code, mongo_client, title='손익계산서y')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('손익계산서', '//*[@id="rpt_tab1"]'),
             ('연간', '//*[@id="frqTyp0"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)

scraper2_hj3415/nfscrapy/nfs/spiders/c104.py

@@ -46,36 +46,37 @@ class C104Spider(scrapy.Spider, metaclass=ABCMeta):
 
     def parse_c104(self, response, code):
         # 페이지를 먼저 한번 호출하여 버튼을 눌러 세팅한다.
-        self.setting_page(response.url)
-
-        # html에서 table을 추출하여 dataframe생성
-        self.driver.get(response.url)
-        time.sleep(self.WAIT)
-        html = Selector(text=self.driver.page_source)
-        table_xpath = '//table[@class="gHead01 all-width data-list"]'
-
-        # 테이블명을 _을 기준으로 나눠 리스트를 만든다.
-        title_list = self.title.split('_')
-        self.logger.debug(title_list)
-
-        # dataframe 리스트를 만든다.
-        df_list = []
-        for i in range(2):
-            # 상위테이블 0, 하위테이블 1
-            df_list.append(common.get_df_from_html(html, table_xpath, i))
-        self.logger.debug(df_list)
-
-        # 테이블명리스트와 df리스트를 매치하여 데이터베이스에 저장하기 위해 yield시킴
-        for title, df in list(zip(title_list, df_list)):
-            # df를 log로 출력한다.
-            self.logger.info(title)
-            self.logger.debug(df)
-            # make item to yield
-            item = items.C104items()
-            item['코드'] = code
-            item['title'] = title
-            item['df'] = df
-            yield item
+        if self.setting_page(response.url):
+            # html에서 table을 추출하여 dataframe생성
+            self.driver.get(response.url)
+            time.sleep(self.WAIT)
+            html = Selector(text=self.driver.page_source)
+            table_xpath = '//table[@class="gHead01 all-width data-list"]'
+
+            # 테이블명을 _을 기준으로 나눠 리스트를 만든다.
+            title_list = self.title.split('_')
+            self.logger.debug(title_list)
+
+            # dataframe 리스트를 만든다.
+            df_list = []
+            for i in range(2):
+                # 상위테이블 0, 하위테이블 1
+                df_list.append(common.get_df_from_html(html, table_xpath, i))
+            self.logger.debug(df_list)
+
+            # 테이블명리스트와 df리스트를 매치하여 데이터베이스에 저장하기 위해 yield시킴
+            for title, df in list(zip(title_list, df_list)):
+                # df를 log로 출력한다.
+                self.logger.info(title)
+                self.logger.debug(df)
+                # make item to yield
+                item = items.C104items()
+                item['코드'] = code
+                item['title'] = title
+                item['df'] = df
+                yield item
+        else:
+            self.logger.warning("Pasing error ... maybe 올바른 종목이 아닙니다.")
 
     @abstractmethod
     def setting_page(self, url: str):
@@ -110,7 +111,7 @@ class C104AQ(C104Spider):
     def __init__(self, code, mongo_client):
         super(C104AQ, self).__init__(code, mongo_client, title='수익성q_가치분석q')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('수익성', '//*[ @id="val_tab1"]'),
             ('분기', '//*[@id="frqTyp1"]'),
@@ -118,7 +119,7 @@ class C104AQ(C104Spider):
             ('가치분석분기', '//*[@id="frqTyp1_2"]'),
             ('가치분석검색', '//*[@id="hfinGubun2"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C104BQ(C104Spider):
@@ -127,13 +128,13 @@ class C104BQ(C104Spider):
     def __init__(self, code, mongo_client):
         super(C104BQ, self).__init__(code, mongo_client, title='성장성q')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('성장성', '//*[ @id="val_tab2"]'),
             ('분기', '//*[@id="frqTyp1"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C104CQ(C104Spider):
@@ -142,13 +143,13 @@ class C104CQ(C104Spider):
     def __init__(self, code, mongo_client):
         super(C104CQ, self).__init__(code, mongo_client, title='안정성q')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('안정성', '//*[ @id="val_tab3"]'),
             ('분기', '//*[@id="frqTyp1"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C104DQ(C104Spider):
@@ -157,13 +158,13 @@ class C104DQ(C104Spider):
     def __init__(self, code, mongo_client):
         super(C104DQ, self).__init__(code, mongo_client, title='활동성q')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('활동성', '//*[ @id="val_tab4"]'),
             ('분기', '//*[@id="frqTyp1"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C104AY(C104Spider):
@@ -172,7 +173,7 @@ class C104AY(C104Spider):
     def __init__(self, code, mongo_client):
         super(C104AY, self).__init__(code, mongo_client, title='수익성y_가치분석y')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('수익성', '//*[ @id="val_tab1"]'),
             ('연간', '//*[@id="frqTyp0"]'),
@@ -180,7 +181,7 @@ class C104AY(C104Spider):
             ('가치분석연간', '//*[@id="frqTyp0_2"]'),
             ('가치분석검색', '//*[@id="hfinGubun2"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C104BY(C104Spider):
@@ -189,13 +190,13 @@ class C104BY(C104Spider):
     def __init__(self, code, mongo_client):
         super(C104BY, self).__init__(code, mongo_client, title='성장성y')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('성장성', '//*[ @id="val_tab2"]'),
             ('연간', '//*[@id="frqTyp0"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C104CY(C104Spider):
@@ -204,13 +205,13 @@ class C104CY(C104Spider):
     def __init__(self, code, mongo_client):
         super(C104CY, self).__init__(code, mongo_client, title='안정성y')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('안정성', '//*[ @id="val_tab3"]'),
             ('연간', '//*[@id="frqTyp0"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)
 
 
 class C104DY(C104Spider):
@@ -219,10 +220,10 @@ class C104DY(C104Spider):
     def __init__(self, code, mongo_client):
         super(C104DY, self).__init__(code, mongo_client, title='활동성y')
 
-    def setting_page(self, url: str):
+    def setting_page(self, url: str) -> bool:
         buttons = [
             ('활동성', '//*[ @id="val_tab4"]'),
             ('연간', '//*[@id="frqTyp0"]'),
             ('검색', '//*[@id="hfinGubun"]'),
         ]
-        common.click_buttons(self.driver, url, buttons, self.WAIT)
+        return common.click_buttons(self.driver, url, buttons, self.WAIT)

scraper2_hj3415/nfscrapy/nfs/spiders/c106.py

@@ -1,6 +1,7 @@
 import time
 import scrapy
 import pandas as pd
+import selenium.common.exceptions
 from util_hj3415 import utils
 from scrapy.selector import Selector
 
@@ -37,28 +38,31 @@ class C106Spider(scrapy.Spider):
         self.driver.get(response.url)
         # 경험상 1초는 데이터 수집에러 가능성 있었음.
         time.sleep(self.WAIT)
-        html = Selector(text=self.driver.page_source)
+        try:
+            html = Selector(text=self.driver.page_source)
+        except selenium.common.exceptions.UnexpectedAlertPresentException:
+            self.logger.warning("Pasing error ... maybe 올바른 종목이 아닙니다.")
+        else:
+            # 컬럼명을 얻어 다음 request에 실어 보낸다.
+            cols = []
+            for i in range(1, 7):
+                pretitle = html.xpath(f'//*[@id="cTB611_h"]/thead/tr/th[{i}]/text()[1]').getall()[0].strip().replace('.','')
+                # 인덱스에 공칸일 경우 데이터베이스 저장시 에러가 발생하기 때문에 추가한 코드
+                if pretitle == '':
+                    pretitle = 'Unnamed'
+                cols.append(pretitle)
+            self.logger.info(f'Parsing column names - {code} >>>> {cols}')
 
-        # 컬럼명을 얻어 다음 request에 실어 보낸다.
-        cols = []
-        for i in range(1, 7):
-            pretitle = html.xpath(f'//*[@id="cTB611_h"]/thead/tr/th[{i}]/text()[1]').getall()[0].strip().replace('.','')
-            # 인덱스에 공칸일 경우 데이터베이스 저장시 에러가 발생하기 때문에 추가한 코드
-            if pretitle == '':
-                pretitle = 'Unnamed'
-            cols.append(pretitle)
-        self.logger.info(f'Parsing column names - {code} >>>> {cols}')
-
-        titles = ['y', 'q'] # pipeline에서 테이블명으로 됨
-        for title in titles:
-            # C106의 내부의 iframe주소, 분기와 연간 2개임
-            # reference from https://docs.scrapy.org/en/latest/topics/request-response.html (request 연쇄보내기)
-            yield scrapy.Request(
-                url=f'https://navercomp.wisereport.co.kr/company/cF6002.aspx?cmp_cd={code}'
-                    f'&finGubun=MAIN&cmp_cd1=&cmp_cd2=&cmp_cd3=&cmp_cd4=&sec_cd=G453010&frq={title.upper()}',
-                callback=self.parse_c106,
-                cb_kwargs=dict(code=code, cols=cols, title=title)
-            )
+            titles = ['y', 'q'] # pipeline에서 테이블명으로 됨
+            for title in titles:
+                # C106의 내부의 iframe주소, 분기와 연간 2개임
+                # reference from https://docs.scrapy.org/en/latest/topics/request-response.html (request 연쇄보내기)
+                yield scrapy.Request(
+                    url=f'https://navercomp.wisereport.co.kr/company/cF6002.aspx?cmp_cd={code}'
+                        f'&finGubun=MAIN&cmp_cd1=&cmp_cd2=&cmp_cd3=&cmp_cd4=&sec_cd=G453010&frq={title.upper()}',
+                    callback=self.parse_c106,
+                    cb_kwargs=dict(code=code, cols=cols, title=title)
+                )
 
     def parse_c106(self, response, code, cols, title):
         df = C106Spider.get_df_from_html(response.text, cols)

scraper2_hj3415/nfscrapy/nfs/spiders/common.py

@@ -1,6 +1,8 @@
 import time
 import pandas as pd
 from typing import Tuple, List
+
+import selenium.common.exceptions
 from selenium.webdriver.chrome.webdriver import WebDriver
 from selenium.webdriver.common.by import By
 
@@ -25,15 +27,28 @@ def adjust_arg_type(code) -> list:
         raise TypeError
 
 
-def click_buttons(driver: WebDriver, url: str, buttons: List[Tuple[str, str]], wait: float):
-    # 하부 클래스에서 buttons 리스트를 입력받아 실제 버튼을 클릭하는 함수
+def click_buttons(driver: WebDriver, url: str, buttons: List[Tuple[str, str]], wait: float) -> bool:
+    """
+    하부 클래스에서 buttons 리스트를 입력받아 실제 버튼을 클릭하는 함수
+
+    :param driver:
+    :param url:
+    :param buttons:
+    :param wait:
+    :return: 함수 작업이 무사히 완료되면 True
+    """
+    #
     logger.debug(f'*** Setting {url} page by clicking buttons ***')
     driver.get(url)
     for name, xpath in buttons:
         logger.debug(f'- Click the {name} button')
-        driver.find_element(By.XPATH, xpath).click()
+        try:
+            driver.find_element(By.XPATH, xpath).click()
+        except selenium.common.exceptions.UnexpectedAlertPresentException:
+            return False
         time.sleep(wait)
     logger.debug('*** Buttons click done ***')
+    return True
 
 
 def get_df_from_html(selector, xpath, table_num):

{scraper2_hj3415-0.2.2.dist-info → scraper2_hj3415-0.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scraper2_hj3415
-Version: 0.2.2
+Version: 0.2.5
 Summary: Gathering the stock data
 Project-URL: Homepage, https://pypi.org/project/scraper2_hj3415/
 Project-URL: Bug Tracker, https://pypi.org/project/scraper2_hj3415/

{scraper2_hj3415-0.2.2.dist-info → scraper2_hj3415-0.2.5.dist-info}/RECORD

@@ -38,12 +38,12 @@ scraper2_hj3415/nfscrapy/nfs/spiders/.DS_Store,sha256=4bCeSNMYj0-03QEqosRz4reLOv
 scraper2_hj3415/nfscrapy/nfs/spiders/__init__.py,sha256=ULwecZkx3_NTphkz7y_qiazBeUoHFnCCWnKSjoDCZj0,161
 scraper2_hj3415/nfscrapy/nfs/spiders/_c108.py,sha256=gOgWQ7qTOOBEZQTY79K5MWzjmz-ZpFg5wrqBhzLoXjI,4776
 scraper2_hj3415/nfscrapy/nfs/spiders/c101.py,sha256=-b_bC0jS_MgDP9h9n2MFHRq-wDx1CGWsgWj6-KQKnpc,3931
-scraper2_hj3415/nfscrapy/nfs/spiders/c103.py,sha256=5E7jZX5UIASCtqGa0eikJVcwlKQsveT-4nyTN6TXGkQ,5614
-scraper2_hj3415/nfscrapy/nfs/spiders/c104.py,sha256=usz29mBeXiQrv_e7uDf2ZihbENVsafEUHUJw8aHRwlw,7645
-scraper2_hj3415/nfscrapy/nfs/spiders/c106.py,sha256=cMbpSLqG4EanKzh6rjpFLAnECt_zLR34MTZo34xu6Go,3783
-scraper2_hj3415/nfscrapy/nfs/spiders/common.py,sha256=QTVWFF386cvoN2czFJQpTeE5jRgnlSE_Z-7y13tPyi8,2876
-scraper2_hj3415-0.2.2.dist-info/METADATA,sha256=kPI700MqeXZAiZiusI6KidxmKo4rDXh882zoh6gmYs0,1303
-scraper2_hj3415-0.2.2.dist-info/WHEEL,sha256=Fd6mP6ydyRguakwUJ05oBE7fh2IPxgtDN9IwHJ9OqJQ,87
-scraper2_hj3415-0.2.2.dist-info/entry_points.txt,sha256=1bGP38AtuY6n2FcP_fLmpqGtFmFf8NLJWsCSWoTjF-0,60
-scraper2_hj3415-0.2.2.dist-info/licenses/LICENSE,sha256=6kbiFSfobTZ7beWiKnHpN902HgBx-Jzgcme0SvKqhKY,1091
-scraper2_hj3415-0.2.2.dist-info/RECORD,,
+scraper2_hj3415/nfscrapy/nfs/spiders/c103.py,sha256=OUcieEOQhD4ttWwDNEFX2F65KtCHe6TOhh3slXuYqhE,5884
+scraper2_hj3415/nfscrapy/nfs/spiders/c104.py,sha256=ugF9yEKtAfZgLDqmX7Y-y1aoIfUR81Txouda27x1Mro,7975
+scraper2_hj3415/nfscrapy/nfs/spiders/c106.py,sha256=SRm6tJiWOCZrxvHF5LnaAXzK2Ui1U80D65JGRaGRNEM,4094
+scraper2_hj3415/nfscrapy/nfs/spiders/common.py,sha256=Y4ENlLTuQAQjPX3z1Xco7HvFdcIUByHohs4lYMjw69Q,3218
+scraper2_hj3415-0.2.5.dist-info/METADATA,sha256=3Am9xsWTglARO3YFF7KsVao33FieXHoJ812UIXNsrQ0,1303
+scraper2_hj3415-0.2.5.dist-info/WHEEL,sha256=Fd6mP6ydyRguakwUJ05oBE7fh2IPxgtDN9IwHJ9OqJQ,87
+scraper2_hj3415-0.2.5.dist-info/entry_points.txt,sha256=1bGP38AtuY6n2FcP_fLmpqGtFmFf8NLJWsCSWoTjF-0,60
+scraper2_hj3415-0.2.5.dist-info/licenses/LICENSE,sha256=6kbiFSfobTZ7beWiKnHpN902HgBx-Jzgcme0SvKqhKY,1091
+scraper2_hj3415-0.2.5.dist-info/RECORD,,