scraper2-hj3415 0.0.2__py3-none-any.whl → 0.1.0__py3-none-any.whl

scraper2_hj3415/miscrapy/mi/spiders/mihistory.py

@@ -0,0 +1,333 @@
+import time
+import scrapy
+from util_hj3415 import utils
+from scrapy.selector import Selector
+from selenium.common.exceptions import NoSuchElementException
+from mi import items
+
+# cmd usage : scrapy crawl mihistory -a year=1
+
+
+WAIT = 1
+
+
+class MIHistory(scrapy.Spider):
+    name = 'mihistory'
+    allowed_domains = ['finance.naver.com']
+
+    def __init__(self, year=1):
+        self.year = int(year)
+        self.driver = utils.get_driver()
+        if self.driver is None:
+            raise
+        # 대략1년전 kospi, kosdaq -> 42, gbond3y -> 38, s&p -> 27, usdkrw -> 26, wti -> 38, gold -> 38, audchf -> 46
+        self.last_page_kospi_kosdaq = 42 * self.year
+        self.last_page_3bond3y = 38 * self.year
+        self.last_page_sp500 = 27 * self.year
+        self.last_page_usdkrw = 26 * self.year
+        self.last_page_wti = 38 * self.year
+        self.last_page_gold = 38 * self.year
+        self.last_page_silver = 38 * self.year
+        self.last_page_audchf = 46 * self.year
+        self.item_list = []
+        self.aud_dict = {}
+        self.audchf_dict = {}
+
+    def start_requests(self):
+        # reference from https://docs.scrapy.org/en/latest/topics/request-response.html
+        print(f'Parsing Market Index history...{self.year} year..')
+        yield scrapy.Request(
+            url=f'https://finance.naver.com/sise/sise_index_day.nhn?code=KOSPI&page={self.last_page_kospi_kosdaq}',
+            callback=self.parse_kospi,
+            cb_kwargs=dict(page=self.last_page_kospi_kosdaq),
+        )
+
+    def parse_kospi(self, response, page):
+        print(f"Parsing ...kospi {page} page", flush=True)
+        item = items.MIitems()
+        # KOSPI를 스크랩하는코드
+        time.sleep(WAIT)
+        self.logger.info(response.url)
+        for i in [12, 11, 10, 5, 4, 3]:
+            item['title'] = 'kospi'
+            item['date'] = response.xpath(f'/html/body/div/table[1]/tr[{i}]/td[1]/text()').get()
+            item['value'] = response.xpath(f'/html/body/div/table[1]/tr[{i}]/td[2]/text()').get().replace(',', '')
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item
+        if page > 1:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/sise/sise_index_day.nhn?code=KOSPI&page={page - 1}',
+                callback=self.parse_kospi,
+                cb_kwargs=dict(page=page - 1),
+            )
+        else:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/sise/sise_index_day.nhn?code=KOSDAQ&page={self.last_page_kospi_kosdaq}',
+                callback=self.parse_kosdaq,
+                cb_kwargs=dict(page=self.last_page_kospi_kosdaq),
+            )
+
+    def parse_kosdaq(self, response, page):
+        print(f"Parsing ...kosdaq {page} page", flush=True)
+        item = items.MIitems()
+        # KOSDAQ를 스크랩하는코드
+        time.sleep(WAIT)
+        self.logger.info(response.url)
+        for i in [12, 11, 10, 5, 4, 3]:
+            item['title'] = 'kosdaq'
+            item['date'] = response.xpath(f'/html/body/div/table[1]/tr[{i}]/td[1]/text()').get()
+            item['value'] = response.xpath(f'/html/body/div/table[1]/tr[{i}]/td[2]/text()').get().replace(',', '')
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item
+        if page > 1:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/sise/sise_index_day.nhn?code=KOSDAQ&page={page - 1}',
+                callback=self.parse_kosdaq,
+                cb_kwargs=dict(page=page - 1),
+            )
+        else:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_GOVT03Y&page={self.last_page_3bond3y}',
+                callback=self.parse_gbond3y,
+                cb_kwargs=dict(page=self.last_page_3bond3y),
+            )
+
+    def parse_gbond3y(self, response, page):
+        print(f"Parsing ...gbond3y {page} page", flush=True)
+        item = items.MIitems()
+        # 국고채 3년금리를 스크랩하는코드
+        time.sleep(WAIT)
+        self.logger.info(response.url)
+        for i in range(7, 0, -1):
+            item['title'] = 'gbond3y'
+            item['date'] = (response.css(f'body > div > table > tbody > tr:nth-child({i}) > td.date::text')
+                            .extract()[0].replace('\n', '').replace('\t', ''))
+            item['value'] = (response.css(f'body > div > table > tbody > tr:nth-child({i}) > td:nth-child(2)::text')
+                .extract()[0])
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item
+        if page > 1:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_GOVT03Y&page={page - 1}',
+                callback=self.parse_gbond3y,
+                cb_kwargs=dict(page=page - 1),
+                )
+        else:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/worldDailyQuote.nhn?fdtc=4&marketindexCd=FX_USDAUD',
+                callback=self.parse_aud,
+                cb_kwargs=dict(page=self.last_page_audchf),
+            )
+
+    def parse_aud(self, response, page):
+        print(f"Parsing ...aud {page} page", flush=True)
+        item = items.MIitems()
+        # AUD를 스크랩하는코드
+        time.sleep(WAIT)
+        self.logger.info(response.url)
+        for i in range(7, 0, -1):
+            item['title'] = 'aud'
+            item['date'] = (response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[1]/text()')
+                            .get().replace('\t', '').replace('\n', ''))
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[2]/text()')
+                             .get().replace(',', '').replace('\t', '').replace('\n', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item
+        if page > 1:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/worldDailyQuote.nhn?fdtc=4&marketindexCd=FX_USDAUD&page={page - 1}',
+                callback=self.parse_aud,
+                cb_kwargs=dict(page=page - 1),
+                )
+        else:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/worldDailyQuote.nhn?fdtc=4&marketindexCd=FX_USDCHF',
+                callback=self.parse_chf,
+                cb_kwargs=dict(page=self.last_page_audchf),
+            )
+
+    def parse_chf(self, response, page):
+        print(f"Parsing ...chf {page} page", flush=True)
+        item = items.MIitems()
+        # CHF를 스크랩하는코드
+        time.sleep(WAIT)
+        self.logger.info(response.url)
+        for i in range(7, 0, -1):
+            item['title'] = 'chf'
+            item['date'] = (response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[1]/text()')
+                            .get().replace('\t', '').replace('\n', ''))
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[2]/text()')
+                             .get().replace(',', '').replace('\t', '').replace('\n', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item
+        if page > 1:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/worldDailyQuote.nhn?fdtc=4&marketindexCd=FX_USDCHF&page={page - 1}',
+                callback=self.parse_chf,
+                cb_kwargs=dict(page=page - 1),
+                )
+        else:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_USDKRW&page={self.last_page_usdkrw}',
+                callback=self.parse_usdkrw,
+                cb_kwargs=dict(page=self.last_page_usdkrw),
+            )
+
+    def parse_usdkrw(self, response, page):
+        print(f"Parsing ...usdkrw {page} page", flush=True)
+        item = items.MIitems()
+        # 달러 원화 환율 스크랩
+        time.sleep(WAIT)
+        self.logger.info(response.url)
+        for i in range(10, 0, -1):
+            item['title'] = 'usdkrw'
+            item['date'] = response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[1]/text()').get()
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[2]/text()')
+                             .get().replace(',', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item
+        if page > 1:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_USDKRW&page={page - 1}',
+                callback=self.parse_usdkrw,
+                cb_kwargs=dict(page=page - 1),
+            )
+        else:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2&page={self.last_page_wti}',
+                callback=self.parse_wti,
+                cb_kwargs=dict(page=self.last_page_wti),
+            )
+
+    def parse_wti(self, response, page):
+        print(f"Parsing ...wti {page} page", flush=True)
+        item = items.MIitems()
+        # 원유 스크랩
+        time.sleep(WAIT)
+        self.logger.info(response.url)
+        for i in range(7, 0, -1):
+            item['title'] = 'wti'
+            item['date'] = (response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[1]/text()')
+                            .get().replace('\t', '').replace('\n', ''))
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[2]/text()')
+                             .get().replace(',', '').replace('\t', '').replace('\n', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item
+        if page > 1:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2&page={page - 1}',
+                callback=self.parse_wti,
+                cb_kwargs=dict(page=page - 1),
+            )
+        else:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2&page={self.last_page_gold}',
+                callback=self.parse_gold,
+                cb_kwargs=dict(page=self.last_page_gold),
+            )
+
+    def parse_gold(self, response, page):
+        print(f"Parsing ...gold {page} page", flush=True)
+        item = items.MIitems()
+        # 금 스크랩
+        time.sleep(WAIT)
+        self.logger.info(response.url)
+        for i in range(7, 0, -1):
+            item['title'] = 'gold'
+            item['date'] = (response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[1]/text()')
+                            .get().replace('\t', '').replace('\n', ''))
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[2]/text()')
+                             .get().replace(',', '').replace('\t', '').replace('\n', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item
+        if page > 1:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2&page={page - 1}',
+                callback=self.parse_gold,
+                cb_kwargs=dict(page=page - 1),
+            )
+        else:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_SI&fdtc=2&page={self.last_page_silver}',
+                callback=self.parse_silver,
+                cb_kwargs=dict(page=self.last_page_silver),
+            )
+
+    def parse_silver(self, response, page):
+        print(f"Parsing ...silver {page} page", flush=True)
+        item = items.MIitems()
+        # 은 스크랩
+        time.sleep(WAIT)
+        self.logger.info(response.url)
+        for i in range(7, 0, -1):
+            item['title'] = 'silver'
+            item['date'] = (response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[1]/text()')
+                            .get().replace('\t', '').replace('\n', ''))
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{i}]/td[2]/text()')
+                             .get().replace(',', '').replace('\t', '').replace('\n', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item
+        if page > 1:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_SI&fdtc=2&page={page - 1}',
+                callback=self.parse_silver,
+                cb_kwargs=dict(page=page - 1),
+            )
+        else:
+            yield scrapy.Request(
+                url=f'https://finance.naver.com/world/sise.nhn?symbol=SPI@SPX',
+                callback=self.parse_sp500,
+                cb_kwargs=dict(page=self.last_page_sp500),
+            )
+
+    def parse_sp500(self, response, page):
+        print(f"Parsing ...sp500 {page} page", flush=True)
+        item = items.MIitems()
+        # S&P500을 스크랩하는코드
+        time.sleep(WAIT)
+        self.logger.info(response.url)
+        self.driver.get(response.url)
+        time.sleep(WAIT)
+        next1 = '//*[@id="dayPaging"]/a[11]'    # 첫페이지의 '다음' 버튼
+        next2 = '//*[@id="dayPaging"]/a[12]'    # 첫페이지 이후의 '다음' 버튼
+        self.driver.find_element_by_xpath(next1).click()
+        time.sleep(WAIT)
+        '''
+        self.driver.find_element_by_xpath(next2).click()
+        time.sleep(1)
+        self.driver.find_element_by_xpath(next2).click()
+        time.sleep(1)
+        self.driver.find_element_by_xpath(next2).click()
+        time.sleep(1)
+        '''
+        for j in range(page, 0, -1):
+            if j % 10 == 0:
+                prev = '//*[@id="dayPaging"]/a[1]'
+                self.driver.find_element_by_xpath(prev).click()
+                self.logger.info('click prev...')
+                time.sleep(1)
+            link = f'//*[@id="dayLink{j}"]'
+            while True:
+                try:
+                    self.driver.find_element_by_xpath(link).click()
+                    self.logger.info(f'click {j} button..')
+                    break
+                except NoSuchElementException as e:
+                    self.logger.error(f'Error : {e}')
+                    self.driver.find_element_by_xpath(next2).click()
+                    time.sleep(1)
+            time.sleep(1)
+            sel = Selector(text=self.driver.page_source)
+
+            for i in range(10, 0, -1):
+                item['title'] = 'sp500'
+                item['date'] = sel.xpath(f'//*[@id="dayTable"]/tbody/tr[{i}]/td[1]/text()').get()
+                item['value'] = (sel.xpath(f'//*[@id="dayTable"]/tbody/tr[{i}]/td[2]/span/text()')
+                                 .get().replace(',', ''))
+                self.logger.info(f"date : {item['date']}, value : {item['value']}")
+                yield item
+
+    def __del__(self):
+        if self.driver is not None:
+            print('Retrieve chrome driver...')
+            self.driver.quit()

scraper2_hj3415/miscrapy/mi/spiders/silver.py

@@ -0,0 +1,25 @@
+import scrapy
+from mi import items
+
+# cmd usage : scrapy crawl silver
+
+
+class SilverSpider(scrapy.Spider):
+    name = 'silver'
+    allowed_domains = ['finance.naver.com']
+    start_urls = ['https://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_SI&fdtc=2']
+
+    def parse(self, response):
+        print(f"Parsing ...'{self.name}' page", flush=True)
+        item = items.MIitems()
+        # Silver를 스크랩하는코드
+        self.logger.info(response.url)
+        # 최근 3개일의 데이터를 스크랩한다.
+        for r in range(3, 0, -1):
+            item['title'] = self.name
+            item['date'] = (response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[1]/text()')
+                            .get().replace('\t', '').replace('\n', ''))
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[2]/text()')
+                             .get().replace(',', '').replace('\t', '').replace('\n', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item

scraper2_hj3415/miscrapy/mi/spiders/sp500.py

@@ -0,0 +1,24 @@
+import scrapy
+from mi import items
+
+# cmd usage : scrapy crawl sp500
+
+
+class Sp500Spider(scrapy.Spider):
+    name = 'sp500'
+    allowed_domains = ['finance.naver.com']
+    start_urls = ['https://finance.naver.com/world/sise.nhn?symbol=SPI@SPX']
+
+    def parse(self, response):
+        print(f"Parsing ...'{self.name}' page", flush=True)
+        item = items.MIitems()
+        # S&P500를 스크랩하는코드
+        self.logger.info(response.url)
+        # 최근 3개일의 데이터를 스크랩한다.
+        for r in range(3, 0, -1):
+            item['title'] = self.name
+            item['date'] = response.xpath(f'//*[@id="dayTable"]/tbody/tr[{r}]/td[1]/text()').get()
+            item['value'] = (response.xpath(f'//*[@id="dayTable"]/tbody/tr[{r}]/td[2]/span/text()')
+                             .get().replace(',', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item

scraper2_hj3415/miscrapy/mi/spiders/usdidx.py

@@ -0,0 +1,30 @@
+import scrapy
+from mi import items
+
+# cmd usage : scrapy crawl usdkrw
+
+
+class UsdidxSpider(scrapy.Spider):
+    name = 'usdidx'
+    allowed_domains = ['finance.naver.com']
+    start_urls = ['https://finance.naver.com/marketindex/worldExchangeDetail.nhn?marketindexCd=FX_USDX']
+
+    def parse(self, response):
+        print(f"Parsing ...'{self.name}' page", flush=True)
+        item = items.MIitems()
+        # Dollar Index를 스크랩하는코드
+        self.logger.info(response.url)
+
+        # 최근 데이터를 스크랩한다.
+        # date - //*[@id="content"]/div[1]/div[2]/span[1]
+        # value - //*[@id="content"]/div[1]/div[1]/p[1]/em
+        value = []
+        for span in response.xpath(f'//*[@id="content"]/div[1]/div[1]/p[1]/em/span'):
+            value.append(span.xpath('text()').get())
+
+        item['title'] = self.name
+        item['date'] = response.xpath('//*[@id="content"]/div[1]/div[2]/span[1]/text()').get()
+        item['value'] = ''.join(value)
+
+        self.logger.info(f"date : {item['date']}, value : {item['value']}")
+        yield item

scraper2_hj3415/miscrapy/mi/spiders/usdkrw.py

@@ -0,0 +1,24 @@
+import scrapy
+from mi import items
+
+# cmd usage : scrapy crawl usdkrw
+
+
+class UsdkrwSpider(scrapy.Spider):
+    name = 'usdkrw'
+    allowed_domains = ['finance.naver.com']
+    start_urls = ['https://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_USDKRW']
+
+    def parse(self, response):
+        print(f"Parsing ...'{self.name}' page", flush=True)
+        item = items.MIitems()
+        # S&P500를 스크랩하는코드
+        self.logger.info(response.url)
+        # 최근 3개일의 데이터를 스크랩한다.
+        for r in range(3, 0, -1):
+            item['title'] = self.name
+            item['date'] = response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[1]/text()').get()
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[2]/text()')
+                             .get().replace(',', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item

scraper2_hj3415/miscrapy/mi/spiders/wti.py

@@ -0,0 +1,25 @@
+import scrapy
+from mi import items
+
+# cmd usage : scrapy crawl wti
+
+
+class WtiSpider(scrapy.Spider):
+    name = 'wti'
+    allowed_domains = ['finance.naver.com']
+    start_urls = ['https://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2']
+
+    def parse(self, response):
+        print(f"Parsing ...'{self.name}' page", flush=True)
+        item = items.MIitems()
+        # S&P500를 스크랩하는코드
+        self.logger.info(response.url)
+        # 최근 3개일의 데이터를 스크랩한다.
+        for r in range(3, 0, -1):
+            item['title'] = self.name
+            item['date'] = (response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[1]/text()')
+                            .get().replace('\t', '').replace('\n', ''))
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[2]/text()')
+                             .get().replace(',', '').replace('\t', '').replace('\n', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item

scraper2_hj3415/miscrapy/run.py

@@ -0,0 +1,156 @@
+import os
+import time
+
+from scrapy.crawler import CrawlerProcess
+from scrapy.utils.project import get_project_settings
+from multiprocessing import Process
+
+from .mi import calc
+from db_hj3415 import mongo2, dbpath
+import datetime
+
+import logging
+
+logger = logging.getLogger(__name__)
+formatter = logging.Formatter('%(levelname)s: [%(name)s] %(message)s')
+ch = logging.StreamHandler()
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+logger.setLevel(logging.ERROR)
+
+
+def chcwd(func):
+    """
+    scrapy는 항상 프로젝트 내부에서 실행해야 하기 때문에 일시적으로 현재 실행 경로를 변경해주는 목적의 데코레이션 함수
+    :param func:
+    :return:
+    """
+
+    def wrapper(*args, **kwargs):
+        before_cwd = os.getcwd()
+        logger.info(f'current path : {before_cwd}')
+        after_cwd = os.path.dirname(os.path.realpath(__file__))
+        logger.info(f'change path to {after_cwd}')
+        os.chdir(after_cwd)
+        func(*args, **kwargs)
+        logger.info(f'restore path to {before_cwd}')
+        os.chdir(before_cwd)
+
+    return wrapper
+
+
+def _use_single(spider):
+    # reference from https://docs.scrapy.org/en/latest/topics/practices.html(코드로 스파이더 실행하기)
+    process = CrawlerProcess(get_project_settings())
+    process.crawl(spider)
+    process.start()
+
+
+@chcwd
+def mi():
+    spider_list = ('aud', 'chf', 'gbond3y', 'gold', 'kosdaq', 'kospi', 'silver', 'sp500', 'usdidx', 'usdkrw', 'wti',)
+    print('*' * 25, f"Scrape multiprocess mi", '*' * 25)
+    logger.info(spider_list)
+
+    start_time = time.time()
+    ths = []
+    error = False
+    for spider in spider_list:
+        ths.append(Process(target=_use_single, args=(spider,)))
+    for i in range(len(ths)):
+        ths[i].start()
+    for i in range(len(ths)):
+        ths[i].join()
+        if ths[i].exitcode != 0:
+            error = True
+
+    # calc 모듈을 이용해서 avg_per 과 yield_gap 을 계산하여 저장한다.
+    print('*' * 25, f"Calculate and save avgper and yieldgap", '*' * 25)
+    client = mongo2.connect_mongo(dbpath.load())
+    mi_mongo2 = mongo2.MI(client, 'avgper')
+    # mi_sqlite = sqlite.MI()
+    today_str = datetime.datetime.today().strftime('%Y.%m.%d')
+
+    avgper = calc.avg_per()
+    avgper_dict = {'date': today_str, 'value': str(avgper)}
+    logger.info(avgper_dict)
+    mi_mongo2.save(mi_dict=avgper_dict, index='avgper')
+    print(f'\tSave to mongo... date : {today_str} / title : avgper / value : {avgper}')
+    #mi_sqlite.save(mi_dict=avgper_dict, index='avgper')
+    #print(f'\tSave to sqlite... date : {today_str} / title : avgper / value : {avgper}')
+
+    yieldgap = calc.yield_gap(client, avgper)
+    yieldgap_dict = {'date': today_str, 'value': str(yieldgap)}
+    logger.info(yieldgap_dict)
+    mi_mongo2.save(mi_dict=yieldgap_dict, index='yieldgap')
+    print(f'\tSave to mongo... date : {today_str} / title : yieldgap / value : {yieldgap}')
+    #mi_sqlite.save(mi_dict=yieldgap_dict, index='yieldgap')
+    #print(f'\tSave to sqlite... date : {today_str} / title : yieldgap / value : {yieldgap}')
+
+    print(f'Total spent time : {round(time.time() - start_time, 2)} sec')
+    print('done.')
+    return error
+
+
+@chcwd
+def _mi_test(spider: str):
+    _use_single(spider=spider)
+
+
+@chcwd
+def mihistory(year: int):
+    process = CrawlerProcess(get_project_settings())
+    process.crawl('mihistory', year=year)
+    process.start()
+
+
+"""avgper 과 yieldgap 계산
+"""
+import math
+from db_hj3415 import mongo2, dbpath
+from eval_hj3415 import eval
+from util_hj3415 import utils
+
+import logging
+logger = logging.getLogger(__name__)
+formatter = logging.Formatter('%(levelname)s: [%(name)s] %(message)s')
+ch = logging.StreamHandler()
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+logger.setLevel(logging.WARNING)
+
+
+def avg_per() -> float:
+    # 가중조화평균으로 평균 per 산출 mi db에 저장
+    per_r_cap_all = []
+    cap_all = []
+    eval_list = eval.make_today_eval_df(dbpath.load()).to_dict('records')
+    for data in eval_list:
+        # eval data: {'code': '111870', '종목명': 'KH 일렉트론', '주가': 1070, 'PER': -2.28, 'PBR': 0.96,
+        # '시가총액': 103300000000, 'RED': -11055.0, '주주수익률': -7.13, '이익지표': -0.30426, 'ROIC': -40.31,
+        # 'ROE': 0.0, 'PFCF': -7.7, 'PCR': nan}
+        logger.debug(f'eval data: {data}')
+        if math.isnan(data['PER']) or data['PER'] == 0:
+            continue
+        if math.isnan(data['시가총액']):
+            continue
+        cap_all.append(data['시가총액'])
+        per_r_cap_all.append((1 / data['PER']) * data['시가총액'])
+    logger.debug(f'Count cap_all :{len(cap_all)}')
+    logger.debug(f'Count per_r_cap_all : {len(per_r_cap_all)}')
+    try:
+        return round(sum(cap_all) / sum(per_r_cap_all), 2)
+    except ZeroDivisionError:
+        return float('nan')
+
+
+def yield_gap(client, avg_per: float) -> float:
+    # 장고에서 사용할 yield gap, mi db에 저장
+    date, gbond3y = mongo2.MI(client, index='gbond3y').get_recent()
+    if math.isnan(avg_per) or avg_per == 0:
+        return float('nan')
+    else:
+        yield_share = (1 / avg_per) * 100
+        yield_gap = round(yield_share - utils.to_float(gbond3y), 2)
+        logger.debug(f"Date - {date}, gbond3y - {gbond3y}, yield_gap - {yield_gap}")
+        return yield_gap

scraper2_hj3415/miscrapy/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = mi.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = mi

scraper2_hj3415/nfscrapy/nfs/items.py

@@ -32,3 +32,27 @@ class C101items(scrapy.Item):
     intro1 = scrapy.Field()
     intro2 = scrapy.Field()
     intro3 = scrapy.Field()
+
+
+class C103items(scrapy.Item):
+    코드 = scrapy.Field()
+    title = scrapy.Field()
+    df = scrapy.Field()
+
+
+class C104items(scrapy.Item):
+    코드 = scrapy.Field()
+    title = scrapy.Field()
+    df = scrapy.Field()
+
+
+class C106items(scrapy.Item):
+    코드 = scrapy.Field()
+    title = scrapy.Field()      # y or q
+    df = scrapy.Field()
+
+"""
+class C108items(scrapy.Item):
+    코드 = scrapy.Field()
+    df = scrapy.Field()
+"""