scraper2-hj3415 0.0.3__py3-none-any.whl → 0.1.1__py3-none-any.whl

scraper2_hj3415/cli.py

@@ -1,23 +1,47 @@
 from .nfscrapy import run
 from krx_hj3415 import krx
+from util_hj3415 import utils
 import argparse
 
 
-def c101_one(code: str):
-    run.c101([code, ], "mongodb://192.168.0.173:27017")
+def nfscraper():
+    spiders = ['c101', 'c106', 'c103', 'c104']
 
-
-def c101_all():
-    run.c101(krx.get_codes(), "mongodb://192.168.0.173:27017")
-
-
-def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('-c', '--code', action="store", help='code')
+    parser.add_argument('spider', help=f"Spiders - {spiders}")
+    parser.add_argument('target', help="Target for scraping (type 6digit code or 'all')")
+
+    parser.add_argument('-d', '--db_path', help="Set mongo database path")
     args = parser.parse_args()
 
-    c101_one(args.code)
+    if args.spider in spiders:
+        if args.spider == 'c101':
+            if args.target == 'all':
+                run.c101(krx.get_codes(), args.db_path) if args.db_path else run.c101(krx.get_codes())
+            elif utils.is_6digit(args.target):
+                run.c101([args.target, ], args.db_path) if args.db_path else run.c101([args.target, ])
+        if args.spider == 'c103':
+            if args.target == 'all':
+                x = input("It will take a long time. Are you sure? (y/N)")
+                if x == 'y' or x == 'Y':
+                    run.c103(krx.get_codes(), args.db_path) if args.db_path else run.c103(krx.get_codes())
+            elif utils.is_6digit(args.target):
+                run.c103([args.target, ], args.db_path) if args.db_path else run.c103([args.target, ])
+        if args.spider == 'c104':
+            if args.target == 'all':
+                x = input("It will take a long time. Are you sure? (y/N)")
+                if x == 'y' or x == 'Y':
+                    run.c104(krx.get_codes(), args.db_path) if args.db_path else run.c104(krx.get_codes())
+            elif utils.is_6digit(args.target):
+                run.c104([args.target, ], args.db_path) if args.db_path else run.c104([args.target, ])
+        if args.spider == 'c106':
+            if args.target == 'all':
+                run.c106(krx.get_codes(), args.db_path) if args.db_path else run.c106(krx.get_codes())
+            elif utils.is_6digit(args.target):
+                run.c106([args.target, ], args.db_path) if args.db_path else run.c106([args.target, ])
+    else:
+        print(f"The spider option should be in {spiders}")
 
 
 if __name__ == '__main__':
-    main()
+    nfscraper()

scraper2_hj3415/miscrapy/mi/items.py

@@ -0,0 +1,7 @@
+import scrapy
+
+
+class MIitems(scrapy.Item):
+    title = scrapy.Field()
+    date = scrapy.Field()
+    value = scrapy.Field()

scraper2_hj3415/miscrapy/mi/middlewares.py

@@ -0,0 +1,103 @@
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+# useful for handling different item types with a single interface
+from itemadapter import is_item, ItemAdapter
+
+
+class MiSpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)
+
+
+class MiDownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)

scraper2_hj3415/miscrapy/mi/pipelines.py

@@ -0,0 +1,39 @@
+
+import logging
+logger = logging.getLogger(__name__)
+formatter = logging.Formatter('%(levelname)s: [%(name)s] %(message)s')
+ch = logging.StreamHandler()
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+logger.setLevel(logging.INFO)
+
+# 한개의 스파이더에서 연속 3일분량의 데이터가 전달된다.
+
+
+class ValidationPipeline:
+    def process_item(self, item, spider):
+        pass
+
+
+class MongoPipeline:
+    def open_spider(self, spider):
+        self.client = spider.mongo_client
+
+
+
+
+    from db_hj3415 import mongo2, dbpath
+
+    client = mongo2.connect_mongo(dbpath.load())
+
+    # 몽고 데이터 베이스에 저장하는 파이프라인
+    def process_item(self, item, spider):
+        """
+        아이템 구조
+            title = scrapy.Field()
+            date = scrapy.Field()
+            value = scrapy.Field()
+        """
+        print(f"In the {self.__class__.__name__}...date : {item['date']} / title : {item['title']} / value : {item['value']}")
+        mongo2.MI(self.client, item['title']).save(mi_dict={"date": item['date'], "value": item['value']})
+        return item

scraper2_hj3415/miscrapy/mi/settings.py

@@ -0,0 +1,103 @@
+# Scrapy settings for mi project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = "mi"
+
+SPIDER_MODULES = ["mi.spiders"]
+NEWSPIDER_MODULE = "mi.spiders"
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = "mi (+http://www.yourdomain.com)"
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+DOWNLOAD_DELAY = 1
+RANDOMIZE_DOWNLOAD_DELAY = True
+
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+#    "Accept-Language": "en",
+#}
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    "mi.middlewares.MiSpiderMiddleware": 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    "mi.middlewares.MiDownloaderMiddleware": 543,
+#}
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    "scrapy.extensions.telnet.TelnetConsole": None,
+#}
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+ITEM_PIPELINES = {
+    "mi.pipelines.ValidationPipeline": 300,
+    "mi.pipelines.MongoPipeline": 400,
+}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = "httpcache"
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+FEED_EXPORT_ENCODING = "utf-8"
+
+LOG_ENABLED = False
+#LOG_LEVEL = 'DEBUG'
+LOG_LEVEL = 'WARNING'
+
+# The port range to use for the telnet console. If set to None or 0, a dynamically assigned port is used.
+TELNETCONSOLE_PORT = None

scraper2_hj3415/miscrapy/mi/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

scraper2_hj3415/miscrapy/mi/spiders/aud.py

@@ -0,0 +1,23 @@
+import scrapy
+from mi import items
+
+
+class AudSpider(scrapy.Spider):
+    name = 'aud'
+    allowed_domains = ['finance.naver.com']
+    start_urls = ['https://finance.naver.com/marketindex/worldDailyQuote.nhn?fdtc=4&marketindexCd=FX_USDAUD']
+
+    def parse(self, response):
+        print(f"Parsing ...'{self.name}' page", flush=True)
+        item = items.MIitems()
+        # S&P500를 스크랩하는코드
+        self.logger.info(response.url)
+        # 최근 3개일의 데이터를 스크랩한다.
+        for r in range(3, 0, -1):
+            item['title'] = self.name
+            item['date'] = (response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[1]/text()')
+                            .get().replace('\t', '').replace('\n', ''))
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[2]/text()')
+                             .get().replace(',', '').replace('\t', '').replace('\n', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item

scraper2_hj3415/miscrapy/mi/spiders/chf.py

@@ -0,0 +1,25 @@
+import scrapy
+from mi import items
+
+# cmd usage : scrapy crawl chf
+
+
+class ChfSpider(scrapy.Spider):
+    name = 'chf'
+    allowed_domains = ['finance.naver.com']
+    start_urls = ['https://finance.naver.com/marketindex/worldDailyQuote.nhn?fdtc=4&marketindexCd=FX_USDCHF']
+
+    def parse(self, response):
+        print(f"Parsing ...'{self.name}' page", flush=True)
+        item = items.MIitems()
+        # S&P500를 스크랩하는코드
+        self.logger.info(response.url)
+        # 최근 3개일의 데이터를 스크랩한다.
+        for r in range(3, 0, -1):
+            item['title'] = self.name
+            item['date'] = (response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[1]/text()')
+                            .get().replace('\t', '').replace('\n', ''))
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[2]/text()')
+                             .get().replace(',', '').replace('\t', '').replace('\n', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item

scraper2_hj3415/miscrapy/mi/spiders/gbond3y.py

@@ -0,0 +1,24 @@
+import scrapy
+from mi import items
+
+# cmd usage : scrapy crawl gbond3y
+
+
+class Gbond3ySpider(scrapy.Spider):
+    name = 'gbond3y'
+    allowed_domains = ['finance.naver.com']
+    start_urls = ['https://finance.naver.com/marketindex/interestDailyQuote.nhn?marketindexCd=IRR_GOVT03Y']
+
+    def parse(self, response):
+        print(f"Parsing ...'{self.name}' page", flush=True)
+        item = items.MIitems()
+        # 국고채 3년금리를 스크랩하는코드
+        self.logger.info(response.url)
+        for r in range(3, 0, -1):
+            item['title'] = self.name
+            item['date'] = (response.css(f'body > div > table > tbody > tr:nth-child({r}) > td.date::text')
+                            .extract()[0].replace('\n', '').replace('\t', ''))
+            item['value'] = (response.css(f'body > div > table > tbody > tr:nth-child({r}) > td:nth-child(2)::text')
+                .extract()[0])
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item

scraper2_hj3415/miscrapy/mi/spiders/gold.py

@@ -0,0 +1,25 @@
+import scrapy
+from mi import items
+
+# cmd usage : scrapy crawl gold
+
+
+class GoldSpider(scrapy.Spider):
+    name = 'gold'
+    allowed_domains = ['finance.naver.com']
+    start_urls = ['https://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=CMDT_GC&fdtc=2']
+
+    def parse(self, response):
+        print(f"Parsing ...'{self.name}' page", flush=True)
+        item = items.MIitems()
+        # S&P500를 스크랩하는코드
+        self.logger.info(response.url)
+        # 최근 3개일의 데이터를 스크랩한다.
+        for r in range(3, 0, -1):
+            item['title'] = self.name
+            item['date'] = (response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[1]/text()')
+                            .get().replace('\t', '').replace('\n', ''))
+            item['value'] = (response.xpath(f'/html/body/div/table/tbody/tr[{r}]/td[2]/text()')
+                             .get().replace(',', '').replace('\t', '').replace('\n', ''))
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item

scraper2_hj3415/miscrapy/mi/spiders/kosdaq.py

@@ -0,0 +1,23 @@
+import scrapy
+from mi import items
+
+# cmd usage : scrapy crawl kosdaq
+
+
+class KosdaqSpider(scrapy.Spider):
+    name = 'kosdaq'
+    allowed_domains = ['finance.naver.com']
+    start_urls = ['https://finance.naver.com/sise/sise_index_day.nhn?code=KOSDAQ']
+
+    def parse(self, response):
+        print(f"Parsing ...'{self.name}' page", flush=True)
+        item = items.MIitems()
+        # KOSDAQ를 스크랩하는코드
+        self.logger.info(response.url)
+        # 최근 3개일의 데이터를 스크랩한다.
+        for r in range(3, 6):
+            item['title'] = self.name
+            item['date'] = response.xpath(f'/html/body/div/table[1]/tr[{r}]/td[1]/text()').get()
+            item['value'] = response.xpath(f'/html/body/div/table[1]/tr[{r}]/td[2]/text()').get().replace(',', '')
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item

scraper2_hj3415/miscrapy/mi/spiders/kospi.py

@@ -0,0 +1,23 @@
+import scrapy
+from mi import items
+
+# cmd usage : scrapy crawl kospi
+
+
+class KospiSpider(scrapy.Spider):
+    name = 'kospi'
+    allowed_domains = ['finance.naver.com']
+    start_urls = ['https://finance.naver.com/sise/sise_index_day.nhn?code=KOSPI']
+
+    def parse(self, response):
+        print(f"Parsing ...'{self.name}' page", flush=True)
+        item = items.MIitems()
+        # KOSPI를 스크랩하는코드
+        self.logger.info(response.url)
+        # 최근 3개일의 데이터를 스크랩한다.
+        for i in range(3, 6):
+            item['title'] = self.name
+            item['date'] = response.xpath(f'/html/body/div/table[1]/tr[{i}]/td[1]/text()').get()
+            item['value'] = response.xpath(f'/html/body/div/table[1]/tr[{i}]/td[2]/text()').get().replace(',', '')
+            self.logger.info(f"date : {item['date']}, value : {item['value']}")
+            yield item