nytimes-scraper-fork 1.1.2.dev1__tar.gz

nytimes_scraper_fork-1.1.2.dev1/PKG-INFO

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: nytimes-scraper-fork
+Version: 1.1.2.dev1
+Summary: Scrape article metadata and comments from NYTimes
+Home-page: https://github.com/ietz/nytimes-scraper
+Author: Tim Pietz
+Author-email: tim@pietz.me
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Description-Content-Type: text/markdown
+Requires-Dist: cssselect
+Requires-Dist: fire
+Requires-Dist: lxml
+Requires-Dist: pandas
+Requires-Dist: requests
+Requires-Dist: tqdm
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary
+
+# nytimes-scraper
+
+[![PyPI](https://img.shields.io/pypi/v/nytimes-scraper)](https://pypi.org/project/nytimes-scraper/)
+
+Scrape article metadata and comments from NYTimes
+
+## Setup
+```bash
+pip install nytimes-scraper
+```
+
+## CLI usage
+The scraper will automatically fetch every article and all the user comments published on
+[nytimes.com](https://www.nytimes.com/).
+Articles are processed month by month, starting with the current month.
+For each month, a `{year}-{month}-articles.pickle` and `{year}-{month}-comments.pickle` will be
+generated in the current directory.
+If the process is restarted, existing outputs will not be overridden and the scraper will continue
+at the month where it left off.
+To use it, run
+```bash
+python -m nytimes_scraper <API_KEY>
+```
+
+## Programmatic usage
+The scraper can also be started programmatically
+```python
+import datetime as dt
+from nytimes_scraper import run_scraper, scrape_month
+
+# scrape february of 2020
+article_df, comment_df = scrape_month('<your_api_key>', date=dt.date(2020, 2, 1))
+
+# scrape all articles month by month
+run_scraper('<your_api_key>')
+```
+
+Alternatively, the `nytimes_scraper.articles` and `nytimes_scraper.comments` modules can be used for more
+fine-grained access:
+```python
+import datetime as dt
+from nytimes_scraper.nyt_api import NytApi
+from nytimes_scraper.articles import fetch_articles_by_month, articles_to_df
+from nytimes_scraper.comments import fetch_comments, fetch_comments_by_article, comments_to_df
+
+api = NytApi('<your_api_key>')
+
+# Fetch articles of a specific month
+articles = fetch_articles_by_month(api, dt.date(2020, 2, 1))
+article_df = articles_to_df(articles)
+
+# Fetch comments from multiple articles
+# a) using the results of a previous article query
+article_ids_and_urls = list(article_df['web_url'].iteritems())
+comments_a = fetch_comments(api, article_ids_and_urls)
+comment_df = comments_to_df(comments_a)
+
+# b) using a custom list of articles
+comments_b = fetch_comments(api, article_ids_and_urls=[
+    ('nyt://article/316ef65c-7021-5755-885c-a9e1ef2cfdf2', 'https://www.nytimes.com/2020/01/03/world/middleeast/trump-iran-suleimani.html'),
+    ('nyt://article/b2d1b802-412e-51f7-8864-efc931e87bb3', 'https://www.nytimes.com/2020/01/04/opinion/impeachment-witnesses.html'),
+])
+
+# Fetch comment for one specific article by its URL
+comments_c = fetch_comments_by_article(api, 'https://www.nytimes.com/2019/11/30/opinion/sunday/bernie-sanders.html')
+```

nytimes_scraper_fork-1.1.2.dev1/README.md

@@ -0,0 +1,66 @@
+# nytimes-scraper
+
+[![PyPI](https://img.shields.io/pypi/v/nytimes-scraper)](https://pypi.org/project/nytimes-scraper/)
+
+Scrape article metadata and comments from NYTimes
+
+## Setup
+```bash
+pip install nytimes-scraper
+```
+
+## CLI usage
+The scraper will automatically fetch every article and all the user comments published on
+[nytimes.com](https://www.nytimes.com/).
+Articles are processed month by month, starting with the current month.
+For each month, a `{year}-{month}-articles.pickle` and `{year}-{month}-comments.pickle` will be
+generated in the current directory.
+If the process is restarted, existing outputs will not be overridden and the scraper will continue
+at the month where it left off.
+To use it, run
+```bash
+python -m nytimes_scraper <API_KEY>
+```
+
+## Programmatic usage
+The scraper can also be started programmatically
+```python
+import datetime as dt
+from nytimes_scraper import run_scraper, scrape_month
+
+# scrape february of 2020
+article_df, comment_df = scrape_month('<your_api_key>', date=dt.date(2020, 2, 1))
+
+# scrape all articles month by month
+run_scraper('<your_api_key>')
+```
+
+Alternatively, the `nytimes_scraper.articles` and `nytimes_scraper.comments` modules can be used for more
+fine-grained access:
+```python
+import datetime as dt
+from nytimes_scraper.nyt_api import NytApi
+from nytimes_scraper.articles import fetch_articles_by_month, articles_to_df
+from nytimes_scraper.comments import fetch_comments, fetch_comments_by_article, comments_to_df
+
+api = NytApi('<your_api_key>')
+
+# Fetch articles of a specific month
+articles = fetch_articles_by_month(api, dt.date(2020, 2, 1))
+article_df = articles_to_df(articles)
+
+# Fetch comments from multiple articles
+# a) using the results of a previous article query
+article_ids_and_urls = list(article_df['web_url'].iteritems())
+comments_a = fetch_comments(api, article_ids_and_urls)
+comment_df = comments_to_df(comments_a)
+
+# b) using a custom list of articles
+comments_b = fetch_comments(api, article_ids_and_urls=[
+    ('nyt://article/316ef65c-7021-5755-885c-a9e1ef2cfdf2', 'https://www.nytimes.com/2020/01/03/world/middleeast/trump-iran-suleimani.html'),
+    ('nyt://article/b2d1b802-412e-51f7-8864-efc931e87bb3', 'https://www.nytimes.com/2020/01/04/opinion/impeachment-witnesses.html'),
+])
+
+# Fetch comment for one specific article by its URL
+comments_c = fetch_comments_by_article(api, 'https://www.nytimes.com/2019/11/30/opinion/sunday/bernie-sanders.html')
+```

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/__init__.py

@@ -0,0 +1,3 @@
+from nytimes_scraper.scraper import run_scraper, scrape_month
+
+__version__ = '1.1.2'

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/__main__.py

@@ -0,0 +1,6 @@
+import fire
+
+from nytimes_scraper import run_scraper
+
+if __name__ == '__main__':
+    fire.Fire(run_scraper)

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/articles/__init__.py

@@ -0,0 +1,2 @@
+from nytimes_scraper.articles.scraper import fetch_articles_by_month
+from nytimes_scraper.articles.postprocessing import articles_to_df

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/articles/postprocessing.py

@@ -0,0 +1,16 @@
+from typing import List, Dict
+
+import pandas as pd
+
+
+def articles_to_df(articles: List[Dict]) -> pd.DataFrame:
+    df = pd.json_normalize(articles).set_index('_id')
+
+    if 'pub_date' in df:
+        df['pub_date'] = pd.to_datetime(df['pub_date'])
+
+    for col in ['document_type', 'news_desk', 'section_name', 'type_of_material', 'byline.organization', 'subsectoinName']:
+        if col in df:
+            df[col] = df[col].astype(pd.CategoricalDtype())
+
+    return df[~df.index.duplicated()]

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/articles/scraper.py

@@ -0,0 +1,41 @@
+import datetime as dt
+import re
+from typing import List, Dict
+
+import lxml.html
+import requests
+from tqdm import tqdm
+
+from nytimes_scraper.nyt_api.api import NytApi
+
+
+def fetch_articles_by_month(api: NytApi, date: dt.date, show_progress: bool = True) -> List[Dict]:
+    """Fetch the article metadata for the year-month of `date`"""
+    print(f'Fetching articles for {date.year}-{date.month:02d}')
+
+    articles = api.archive.archive(date.year, date.month)['response']['docs']
+    for article in tqdm(articles, unit='Article', disable=not show_progress):
+        try:
+            article['html'] = fetch_article_html(article['web_url'])
+            article['text'] = scrape_article_text(article['html'])
+        except ValueError:
+            pass
+
+    return articles
+
+
+def fetch_article_html(article_url: str) -> str:
+    response = requests.get(article_url)
+    response.raise_for_status()
+    return response.text
+
+
+def scrape_article_text(article_html: str) -> str:
+    doc = lxml.html.fromstring(article_html)
+    text_nodes = doc.cssselect('section[name="articleBody"] > .StoryBodyCompanionColumn > :first-child > *')
+    text_node_contents = [re.sub(r'[\n\s]+', ' ', node.text_content()).strip() for node in text_nodes]
+    return '\n'.join(text for text in text_node_contents if text != '')
+
+
+def fetch_recent_articles(api: NytApi):
+    return api.newswire.newswire()['results']

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/comments/__init__.py

@@ -0,0 +1,2 @@
+from nytimes_scraper.comments.scraper import fetch_comments_by_article, fetch_comments
+from nytimes_scraper.comments.postprocessing import comments_to_df

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/comments/postprocessing.py

@@ -0,0 +1,23 @@
+from typing import List, Dict
+
+import pandas as pd
+
+from nytimes_scraper.comments.util import flatten_replies, remove_reply_references
+
+
+def comments_to_df(comments: List[Dict]) -> pd.DataFrame:
+    all_comments = remove_reply_references(flatten_replies(comments))
+    df = pd.json_normalize(all_comments).set_index('commentID')
+
+    if 'parentID' in df:
+        df['parentID'] = df['parentID'].astype(pd.Int64Dtype())
+
+    for col in ['status', 'commentType']:
+        if col in df:
+            df[col] = df[col].astype(pd.CategoricalDtype())
+
+    for col in ['createDate', 'updateDate', 'approveDate']:
+        if col in df:
+            df[col] = pd.to_datetime(df[col], unit='s')
+
+    return df

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/comments/scraper.py

@@ -0,0 +1,77 @@
+from typing import List, Dict, Optional, Tuple
+
+from tqdm import tqdm
+
+from nytimes_scraper.comments.util import flatten_replies
+from nytimes_scraper.nyt_api.api import NytApi
+
+
+def fetch_comments(api: NytApi, article_ids_and_urls: List[Tuple[str, str]], show_progess: bool = True, pagination_size: int = 100) -> List[Dict]:
+    """Fetch all comments from multiple articles, given a list of article IDs and URLs
+    `[(id_1, url_1), (id_2, url_2), …]`
+
+    The IDs are not used for processing but are added to the comment objects as an attribute for
+    later reference."""
+
+    comments = []
+    for article_id, article_url in tqdm(article_ids_and_urls, unit='Article', disable=not show_progess):
+        comments.extend(fetch_comments_by_article(api, article_url, article_id=article_id, pagination_size=pagination_size))
+
+    return comments
+
+
+def fetch_comments_by_article(api: NytApi, article_url: str, article_id: Optional[str] = None, pagination_size: int = 100) -> List[Dict]:
+    """Fetch all comments from one specific article"""
+
+    comments = fetch_top_level_comments(api, article_url, pagination_size=pagination_size)
+    fetch_replies(api, article_url, comments, pagination_size=pagination_size)
+
+    if article_id is not None:
+        for comment in flatten_replies(comments):
+            comment['articleID'] = article_id
+
+    return comments
+
+
+def fetch_top_level_comments(api: NytApi, article_url: str, pagination_size: int) -> List[Dict]:
+    """Fetch all top level comments by paginating through the comment list.
+    Might also include some replies."""
+
+    comments = []
+    while True:
+        response = api.community.get_comments(article_url, offset=len(comments), limit=pagination_size)
+        if response['status'] != 'OK':
+            # some multimedia articles dont allow comments and instead throw an error here
+            return []
+
+        results = response['results']
+        new_comments = results['comments']
+        comments.extend(new_comments)
+
+        if len(new_comments) < pagination_size or len(comments) >= results['totalParentCommentsFound']:
+            return comments
+
+
+def fetch_replies(api: NytApi, article_url: str, comments: List[Dict], pagination_size: int):
+    """Fetch all replies for every comment.
+    Modifies the comment objects by extending the reply lists."""
+
+    comment_reply_queue = flatten_replies(comments)
+    while len(comment_reply_queue) > 0:
+        comment = comment_reply_queue.pop()
+
+        while len(comment['replies']) < comment['replyCount']:
+            response = api.community.get_replies(
+                article_url=article_url,
+                comment_sequence=comment['commentSequence'],
+                offset=len(comment['replies']),
+            )
+            results = response['results']
+
+            replies = results['comments'][0]['replies']
+            comment['replies'].extend(replies)
+
+            comment_reply_queue.extend(flatten_replies(replies))
+
+            if len(replies) < pagination_size:
+                break

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/comments/util.py

@@ -0,0 +1,24 @@
+from typing import List, Dict
+
+
+def flatten_replies(comments: List[Dict]) -> List[Dict]:
+    """Flattens all comments and replies into one list without copying or modifying the objects"""
+
+    result = []
+    for comment in comments:
+        result.append(comment)
+        result.extend(flatten_replies(comment['replies']))
+
+    return result
+
+
+def remove_reply_references(comments: List[Dict]) -> List[Dict]:
+    """Removes the `replies` list from every comments"""
+
+    result = []
+    for comment in comments:
+        copy = comment.copy()
+        copy.pop('replies', None)
+        result.append(copy)
+
+    return result

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/nyt_api/__init__.py

@@ -0,0 +1 @@
+from nytimes_scraper.nyt_api.api import NytApi

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/nyt_api/api.py

@@ -0,0 +1,10 @@
+from nytimes_scraper.nyt_api.archive import ArchiveApi
+from nytimes_scraper.nyt_api.community import CommunityApi
+from nytimes_scraper.nyt_api.newswire import NewswireApi
+
+
+class NytApi:
+    def __init__(self, api_key: str):
+        self.archive = ArchiveApi(api_key)
+        self.community = CommunityApi(api_key)
+        self.newswire = NewswireApi(api_key)

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/nyt_api/archive.py

@@ -0,0 +1,19 @@
+from time import sleep
+
+import requests
+
+
+class ArchiveApi:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+
+    def archive(self, year: int, month: int):
+        response = requests.get(
+            url=f'https://api.nytimes.com/svc/archive/v1/{year}/{month}.json',
+            params={
+                'api-key': self.api_key,
+            },
+        )
+
+        sleep(1)
+        return response.json()

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/nyt_api/community.py

@@ -0,0 +1,38 @@
+from time import sleep
+
+import requests
+
+
+class CommunityApi:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+
+    def get_comments(self, article_url: str, limit: int = 100, offset: int = 0, sort: str = 'oldest'):
+        response = requests.get(
+            url='https://www.nytimes.com/svc/community/V3/requestHandler',
+            params={
+                'cmd': 'GetCommentsAll',
+                'url': article_url,
+                'sort': sort,
+                'limit': limit,
+                'offset': offset,
+            },
+        )
+
+        sleep(1)
+        return response.json()
+
+    def get_replies(self, article_url: str, comment_sequence: int, limit: int = 100, offset: int = 0):
+        response = requests.get(
+            url='https://www.nytimes.com/svc/community/V3/requestHandler',
+            params={
+                'cmd': 'GetRepliesBySequence',
+                'url': article_url,
+                'commentSequence': comment_sequence,
+                'limit': limit,
+                'offset': offset,
+            },
+        )
+
+        sleep(1)
+        return response.json()

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/nyt_api/newswire.py

@@ -0,0 +1,19 @@
+from time import sleep
+
+import requests
+
+
+class NewswireApi:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+
+    def newswire(self, source: str = "nyt", section: str = "world"):
+        response = requests.get(
+            url=f'https://api.nytimes.com/svc/news/v3/content/{source}/{section}.json',
+            params={
+                'api-key': self.api_key,
+            },
+        )
+
+        sleep(1)
+        return response.json()

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper/scraper.py

@@ -0,0 +1,65 @@
+import datetime as dt
+from pathlib import Path
+from typing import Tuple, Callable
+
+import pandas as pd
+
+from nytimes_scraper.articles import fetch_articles_by_month, articles_to_df
+from nytimes_scraper.comments import fetch_comments, comments_to_df
+from nytimes_scraper.nyt_api import NytApi
+
+
+out_dir = Path.cwd()
+
+
+def run_scraper(api_key: str):
+    """Scrape articles and comments month by month, starting from the current month"""
+
+    # the year and month to be fetched
+    date = dt.datetime.now().date().replace(day=1)
+    while True:
+        scrape_month(api_key, date)
+
+        # go one month back
+        date = (date - dt.timedelta(days=1)).replace(day=1)
+
+
+def scrape_month(api_key: str, date: dt.date, force_fetch: bool = False, store: bool = True) -> Tuple[pd.DataFrame, pd.DataFrame]:
+    """Scrape articles and comments for a given month. The `date.day` is ignored."""
+
+    api = NytApi(api_key)
+
+    article_df = cached(
+        fetch=lambda: articles_to_df(fetch_articles_by_month(api, date)),
+        file=out_file(date, 'articles'),
+        force_fetch=force_fetch,
+        store=store,
+    )
+
+    article_ids_and_urls = list(article_df['web_url'].iteritems())
+    comment_df = cached(
+        fetch=lambda: comments_to_df(fetch_comments(api, article_ids_and_urls)),
+        file=out_file(date, 'comments'),
+        force_fetch=force_fetch,
+        store=store,
+    )
+
+    return article_df, comment_df
+
+
+def cached(file: Path, fetch: Callable[[], pd.DataFrame], force_fetch: bool, store: bool) -> pd.DataFrame:
+    if not file.exists() or force_fetch:
+        df = fetch()
+
+        if store:
+            file.parent.mkdir(exist_ok=True)
+            df.to_pickle(str(file))
+
+        return df
+    else:
+        return pd.read_pickle(str(file))
+
+
+def out_file(date: dt.date, kind: str) -> Path:
+    return out_dir / f'{date.year}-{date.month:02d}-{kind}.pickle'
+

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper_fork.egg-info/PKG-INFO

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: nytimes-scraper-fork
+Version: 1.1.2.dev1
+Summary: Scrape article metadata and comments from NYTimes
+Home-page: https://github.com/ietz/nytimes-scraper
+Author: Tim Pietz
+Author-email: tim@pietz.me
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Description-Content-Type: text/markdown
+Requires-Dist: cssselect
+Requires-Dist: fire
+Requires-Dist: lxml
+Requires-Dist: pandas
+Requires-Dist: requests
+Requires-Dist: tqdm
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary
+
+# nytimes-scraper
+
+[![PyPI](https://img.shields.io/pypi/v/nytimes-scraper)](https://pypi.org/project/nytimes-scraper/)
+
+Scrape article metadata and comments from NYTimes
+
+## Setup
+```bash
+pip install nytimes-scraper
+```
+
+## CLI usage
+The scraper will automatically fetch every article and all the user comments published on
+[nytimes.com](https://www.nytimes.com/).
+Articles are processed month by month, starting with the current month.
+For each month, a `{year}-{month}-articles.pickle` and `{year}-{month}-comments.pickle` will be
+generated in the current directory.
+If the process is restarted, existing outputs will not be overridden and the scraper will continue
+at the month where it left off.
+To use it, run
+```bash
+python -m nytimes_scraper <API_KEY>
+```
+
+## Programmatic usage
+The scraper can also be started programmatically
+```python
+import datetime as dt
+from nytimes_scraper import run_scraper, scrape_month
+
+# scrape february of 2020
+article_df, comment_df = scrape_month('<your_api_key>', date=dt.date(2020, 2, 1))
+
+# scrape all articles month by month
+run_scraper('<your_api_key>')
+```
+
+Alternatively, the `nytimes_scraper.articles` and `nytimes_scraper.comments` modules can be used for more
+fine-grained access:
+```python
+import datetime as dt
+from nytimes_scraper.nyt_api import NytApi
+from nytimes_scraper.articles import fetch_articles_by_month, articles_to_df
+from nytimes_scraper.comments import fetch_comments, fetch_comments_by_article, comments_to_df
+
+api = NytApi('<your_api_key>')
+
+# Fetch articles of a specific month
+articles = fetch_articles_by_month(api, dt.date(2020, 2, 1))
+article_df = articles_to_df(articles)
+
+# Fetch comments from multiple articles
+# a) using the results of a previous article query
+article_ids_and_urls = list(article_df['web_url'].iteritems())
+comments_a = fetch_comments(api, article_ids_and_urls)
+comment_df = comments_to_df(comments_a)
+
+# b) using a custom list of articles
+comments_b = fetch_comments(api, article_ids_and_urls=[
+    ('nyt://article/316ef65c-7021-5755-885c-a9e1ef2cfdf2', 'https://www.nytimes.com/2020/01/03/world/middleeast/trump-iran-suleimani.html'),
+    ('nyt://article/b2d1b802-412e-51f7-8864-efc931e87bb3', 'https://www.nytimes.com/2020/01/04/opinion/impeachment-witnesses.html'),
+])
+
+# Fetch comment for one specific article by its URL
+comments_c = fetch_comments_by_article(api, 'https://www.nytimes.com/2019/11/30/opinion/sunday/bernie-sanders.html')
+```

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper_fork.egg-info/SOURCES.txt

@@ -0,0 +1,22 @@
+README.md
+setup.py
+nytimes_scraper/__init__.py
+nytimes_scraper/__main__.py
+nytimes_scraper/scraper.py
+nytimes_scraper/articles/__init__.py
+nytimes_scraper/articles/postprocessing.py
+nytimes_scraper/articles/scraper.py
+nytimes_scraper/comments/__init__.py
+nytimes_scraper/comments/postprocessing.py
+nytimes_scraper/comments/scraper.py
+nytimes_scraper/comments/util.py
+nytimes_scraper/nyt_api/__init__.py
+nytimes_scraper/nyt_api/api.py
+nytimes_scraper/nyt_api/archive.py
+nytimes_scraper/nyt_api/community.py
+nytimes_scraper/nyt_api/newswire.py
+nytimes_scraper_fork.egg-info/PKG-INFO
+nytimes_scraper_fork.egg-info/SOURCES.txt
+nytimes_scraper_fork.egg-info/dependency_links.txt
+nytimes_scraper_fork.egg-info/requires.txt
+nytimes_scraper_fork.egg-info/top_level.txt

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper_fork.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper_fork.egg-info/requires.txt

@@ -0,0 +1,6 @@
+cssselect
+fire
+lxml
+pandas
+requests
+tqdm

nytimes_scraper_fork-1.1.2.dev1/nytimes_scraper_fork.egg-info/top_level.txt

@@ -0,0 +1 @@
+nytimes_scraper

nytimes_scraper_fork-1.1.2.dev1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

nytimes_scraper_fork-1.1.2.dev1/setup.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+from setuptools import setup, find_packages
+
+# The directory containing this file
+here = Path(__file__).parent
+
+# The text of the README file
+readme = (here / 'README.md').read_text()
+
+# This call to setup() does all the work
+setup(
+    name='nytimes-scraper-fork',
+    version='1.1.2.dev1',
+    description='Scrape article metadata and comments from NYTimes',
+    long_description=readme,
+    long_description_content_type='text/markdown',
+    url='https://github.com/ietz/nytimes-scraper',
+    author='Tim Pietz',
+    author_email='tim@pietz.me',
+    license='MIT',
+    classifiers=[
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.8',
+    ],
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=[
+        'cssselect',
+        'fire',
+        'lxml',
+        'pandas',
+        'requests',
+        'tqdm',
+    ],
+)