ohmyscrapper 0.2.3__py3-none-any.whl → 0.7.0__py3-none-any.whl

ohmyscrapper/modules/sniff_url.py

@@ -3,31 +3,101 @@ from bs4 import BeautifulSoup
 import json
 
 
-def sniff_url(url="https://www.linkedin.com/in/cesardesouzacardoso/", silent=False):
+def sniff_url(
+    url="https://www.linkedin.com/in/cesardesouzacardoso/",
+    silent=False,
+    sniffing_config={},
+):
+    if "metatags" in sniffing_config:
+        metatags_to_search = sniffing_config["metatags"]
+    else:
+        metatags_to_search = [
+            "description",
+            "og:url",
+            "og:title",
+            "og:description",
+            "og:type",
+            "lnkd:url",
+        ]
+
+    if "bodytags" in sniffing_config:
+        body_tags_to_search = sniffing_config["bodytags"]
+    else:
+        body_tags_to_search = {
+            "h1": "",
+            "h2": "",
+        }
+
+    if type(metatags_to_search) is dict:
+        metatags_to_search = list(metatags_to_search.keys())
+
+    # force clean concatenate without any separator
+    if type(body_tags_to_search) is dict:
+        body_tags_to_search = list(body_tags_to_search.keys())
+
+    if type(body_tags_to_search) is list:
+        body_tags_to_search = dict.fromkeys(body_tags_to_search, " ")
+
     if not silent:
         print("checking url:", url)
-    report_meta_tags = []
-    tags_to_search = [
-        "description",
-        "og:url",
-        "og:title",
-        "og:description",
-        "og:type",
-        "lnkd:url",
-    ]
 
     r = requests.get(url=url)
     soup = BeautifulSoup(r.text, "html.parser")
 
+    final_report = {}
+    final_report["scrapped-url"] = url
+    if len(metatags_to_search) > 0:
+        final_report.update(
+            _extract_meta_tags(
+                soup=soup, silent=silent, metatags_to_search=metatags_to_search
+            )
+        )
+
+    if len(body_tags_to_search) > 0:
+        final_report.update(
+            _extract_text_tags(
+                soup=soup, silent=silent, body_tags_to_search=body_tags_to_search
+            )
+        )
+    final_report["a_links"] = _extract_a_tags(soup=soup, silent=silent)
+    final_report = _complementary_report(final_report, soup, silent).copy()
+    final_report["json"] = json.dumps(final_report)
+
+    return final_report
+
+
+def _extract_a_tags(soup, silent):
+    a_links = []
+    if not silent:
+        print("\n\n\n\n---- all <a> links ---")
+
+    i = 0
+    for a_tag in soup.find_all("a"):
+        i = i + 1
+        a_links.append({"text": a_tag.text, "href": a_tag.get("href")})
+        if not silent:
+            print("\n-- <a> link", i, "-- ")
+            print("target:", a_tag.get("target"))
+            print("text:", str(a_tag.text).strip())
+            print("href:", a_tag.get("href"))
+            print("-------------- ")
+    return a_links
+
+
+def _extract_meta_tags(soup, silent, metatags_to_search):
+    valid_meta_tags = {}
     if not silent:
         print("\n\n\n\n---- all <meta> tags ---\n")
     i = 0
     for meta_tag in soup.find_all("meta"):
         if (
-            meta_tag.get("name") in tags_to_search
-            or meta_tag.get("property") in tags_to_search
+            meta_tag.get("name") in metatags_to_search
+            or meta_tag.get("property") in metatags_to_search
         ):
-            report_meta_tags.append(meta_tag)
+            if meta_tag.get("name") is not None:
+                valid_meta_tags[meta_tag.get("name")] = meta_tag.get("content")
+            elif meta_tag.get("property") is not None:
+                valid_meta_tags[meta_tag.get("property")] = meta_tag.get("content")
         i = i + 1
         if not silent:
             print("-- meta tag", i, "--")
@@ -35,54 +105,51 @@ def sniff_url(url="https://www.linkedin.com/in/cesardesouzacardoso/", silent=Fal
             print("property:", meta_tag.get("property"))
             print("content:", meta_tag.get("content"))
             print("---------------- \n")
+    return valid_meta_tags
 
+
+def _extract_text_tags(soup, silent, body_tags_to_search):
+    valid_text_tags = {}
     if not silent:
-        print("\n\n\n\n---- all <a> links ---")
-        i = 0
-        for a_tag in soup.find_all("a"):
-            i = i + 1
-            print("\n-- a link", i, "-- ")
-            print("target:", a_tag.get("target"))
-            print("text:", a_tag.text)
-            print("href:", a_tag.get("href"))
-            print("-------------- ")
+        print("\n\n\n\n---- all <text> tags ---\n")
+    i = 0
+    for text_tag, separator in body_tags_to_search.items():
+        if len(soup.find_all(text_tag)) > 0:
+            valid_text_tags[text_tag] = []
+            for obj_tag in soup.find_all(text_tag):
+                valid_text_tags[text_tag].append(obj_tag.text.strip())
+            valid_text_tags[text_tag] = separator.join(valid_text_tags[text_tag])
+        i = i + 1
+        if not silent:
+            print("-- text tag", i, "--")
+            print("name:", text_tag)
+            print("separator:", separator)
+            print("texts:", valid_text_tags[text_tag])
+            print("---------------- \n")
+    return valid_text_tags
 
-    final_report = {}
-    final_report["scrapped-url"] = url
-    if len(soup.find_all("h1")) > 0:
-        final_report["h1"] = soup.find("h1").text
-
-    for report_meta_tag in report_meta_tags:
-        if report_meta_tag.get("name") is not None:
-            final_report[report_meta_tag.get("name")] = report_meta_tag.get("content")
-        elif report_meta_tag.get("property") is not None:
-            final_report[report_meta_tag.get("property")] = report_meta_tag.get(
-                "content"
-            )
 
-    if len(soup.find_all("a")) > 0:
-        final_report["first-a-link"] = soup.find("a").get("href")
-        final_report["total-a-links"] = len(soup.find_all("a"))
+def _complementary_report(final_report, soup, silent):
+
+    if len(final_report["a_links"]) > 0:
+        final_report["first-a-link"] = final_report["a_links"][0]["href"]
+        final_report["total-a-links"] = len(final_report["a_links"])
     else:
         final_report["first-a-link"] = ""
         final_report["total-a-links"] = 0
 
-    if len(soup.find_all("h2")) > 0:
-        final_report["h2"] = soup.find("h2").text
-
     if len(soup.find_all("meta")) > 0:
         final_report["total-meta-tags"] = len(soup.find_all("meta"))
     else:
         final_report["total-meta-tags"] = 0
-
-    final_report["json"] = json.dumps(final_report)
     if not silent:
         print("\n\n\n----report---\n")
         for key in final_report:
-            print("* ", key, ":", final_report[key])
+            if key != "a_links":
+                print("* ", key, ":", final_report[key])
 
     return final_report
 
 
-def get_tags(url):
-    return sniff_url(url=url, silent=True)
+def get_tags(url, sniffing_config={}):
+    return sniff_url(url=url, silent=True, sniffing_config=sniffing_config)

ohmyscrapper/modules/untouch_all.py

@@ -3,5 +3,5 @@ import ohmyscrapper.models.urls_manager as urls_manager
 
 def untouch_all():
     urls_manager.untouch_all_urls()
-    print("urls have been untouched")
+    print("🙌 urls have been untouched")
     return

{ohmyscrapper-0.2.3.dist-info → ohmyscrapper-0.7.0.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.3
 Name: ohmyscrapper
-Version: 0.2.3
-Summary: This project aims to create a text-based scraper containing links to create a final PDF with general information about job openings.
+Version: 0.7.0
+Summary: OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a final report with general information about job positions.
 Author: Cesar Cardoso
 Author-email: Cesar Cardoso <hello@cesarcardoso.cc>
 Requires-Dist: beautifulsoup4>=4.14.3
@@ -16,16 +16,17 @@ Requires-Dist: urlextract>=1.9.0
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 
-# OhMyScrapper - v0.2.3
+# 🐶 OhMyScrapper - v0.7.0
 
-This project aims to create a text-based scraper containing links to create a
-final PDF with general information about job openings.
+OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a
+final report with general information about job positions.
 
 ## Scope
 
 - Read texts;
-- Extract links;
-- Use meta og:tags to extract information;
+- Extract and load urls;
+- Scrapes the urls looking for og:tags and titles;
+- Export a list of links with relevant information;
 
 ## Installation
 
@@ -50,7 +51,7 @@ uvx ohmyscrapper --version
 
 OhMyScrapper works in 3 stages:
 
-1. It collects and loads urls from a text (by default `input/_chat.txt`) in a database;
+1. It collects and loads urls from a text in a database;
 2. It scraps/access the collected urls and read what is relevant. If it finds new urls, they are collected as well;
 3. Export a list of urls in CSV files;
 
@@ -58,7 +59,7 @@ You can do 3 stages with the command:
 ```shell
 ohmyscrapper start
 ```
-> Remember to add your text file in the folder `/input` with the name `_chat.txt`!
+> Remember to add your text file in the folder `/input` with the name that finishes with `.txt`!
 
 You will find the exported files in the folder `/output` like this:
 - `/output/report.csv`
@@ -70,18 +71,23 @@ You will find the exported files in the folder `/output` like this:
 
 ### BUT: if you want to do step by step, here it is:
 
-First we load a text file you would like to look for urls, the idea here is to
-use the whatsapp history, but it works with any txt file.
+First we load a text file you would like to look for urls. It it works with any txt file.
 
-The default file is `input/_chat.txt`. If you have the default file you just use
-the command `load`:
+The default folder is `/input`. Put one or more text (finished with `.txt`) files
+in this folder and use the command `load`:
 ```shell
 ohmyscrapper load
 ```
-or, if you have another file, just use the argument `-file` like this:
+or, if you have another file in a different folder, just use the argument `-input` like this:
 ```shell
-ohmyscrapper load -file=my-text-file.txt
+ohmyscrapper load -input=my-text-file.txt
 ```
+In this case, you can add an url directly to the database, like this:
+```shell
+ohmyscrapper load -input=https://cesarcardoso.cc/
+```
+That will append the last url in the database to be scraped.
+
 That will create a database if it doesn't exist and store every url the oh-my-scrapper
 find. After that, let's scrap the urls with the command `scrap-urls`:
 

ohmyscrapper-0.7.0.dist-info/RECORD

@@ -0,0 +1,21 @@
+ohmyscrapper/__init__.py,sha256=w5Ty9eszf8tEv72IQrFov0YbZWMqsraq448xhX3YGQs,6493
+ohmyscrapper/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
+ohmyscrapper/core/config.py,sha256=i_RA-zReNQIWWmsFar85qzRUqdqvTFMPeCP7Hya7ltU,2996
+ohmyscrapper/core/config_files.py,sha256=KC3yChTnlclclU9EKTqFBoAu9p6XdOKuegub5NPYDDY,2434
+ohmyscrapper/core/default_files/config.yaml,sha256=bgPBVlze2tOCbyrA47h_5BJ35UsXnqsjQszzy0vn-Pw,248
+ohmyscrapper/core/default_files/url_sniffing.yaml,sha256=MKdVR5HQ1i2yTRw2ijzxPSmIyhUno_R4L2k17r3EBBc,417
+ohmyscrapper/core/default_files/url_types.yaml,sha256=20kvv8_iWRT-pLa014RXYpAmPSonn6tDnG302rx7l-o,228
+ohmyscrapper/models/urls_manager.py,sha256=FC1j72M1gzNwC_PzPqnew986b-BI6s7zUv8Z7HiM1M0,11849
+ohmyscrapper/modules/classify_urls.py,sha256=GhiosAQUITy1DQe_PksYV9QRKVTgpkSE28dkutzbWVA,1038
+ohmyscrapper/modules/load_txt.py,sha256=dNkUZ2ehBiPx-q4fPczRiHFvnpzCrjeycFtexhWGmEE,3967
+ohmyscrapper/modules/merge_dbs.py,sha256=0pK3PPUGSbnaDkdpQUGCHemOVaKO37bfHwnsy_EVpWQ,115
+ohmyscrapper/modules/process_with_ai.py,sha256=kl39Jzl-PUwh6AfmTZ9SLFUYs9Sk4biqgt8rNz3X1FA,7255
+ohmyscrapper/modules/scrap_urls.py,sha256=CNoEC-d1r-u4qxnEVimm4ctP6MJGdU8y8VI2Nx0bBdM,6033
+ohmyscrapper/modules/seed.py,sha256=qDUE7TWx9iNQEzqThK4p7g8pTZjdpkmoqI8kOo_zdtk,983
+ohmyscrapper/modules/show.py,sha256=jsAs4g8ouA9wymkBfkDCbpVWKD-m_20uKG-m1cZAUGA,3877
+ohmyscrapper/modules/sniff_url.py,sha256=zJ2Uox2aUdQibL4UFLxg3t7GqJ7WwWEl0q3QSUbMEbc,4960
+ohmyscrapper/modules/untouch_all.py,sha256=DAwWYfqMFifHPtFCxSamu0AxHCgk6aJbTnBy6wLucXM,167
+ohmyscrapper-0.7.0.dist-info/WHEEL,sha256=xDCZ-UyfvkGuEHPeI7BcJzYKIZzdqN8A8o1M5Om8IyA,79
+ohmyscrapper-0.7.0.dist-info/entry_points.txt,sha256=BZud6D16XkfjelDa4Z33mji-KJbbZXgq2FoLrzjru5I,52
+ohmyscrapper-0.7.0.dist-info/METADATA,sha256=Doakf4oDT6oskPGdSlEoRJHBxUmm9FhWaHfDlNIfNuM,4096
+ohmyscrapper-0.7.0.dist-info/RECORD,,

ohmyscrapper-0.2.3.dist-info/RECORD

@@ -1,16 +0,0 @@
-ohmyscrapper/__init__.py,sha256=6k-fyuKkTefy5lJiRJFDP7wfkGhYRC8qgdfYlNB_3sk,5841
-ohmyscrapper/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
-ohmyscrapper/models/urls_manager.py,sha256=xKql_xdwfRwgpMyriuIrZ0Srz4gYQGMfWClEWpGRtNE,11183
-ohmyscrapper/modules/classify_urls.py,sha256=eyHtTHDZp2pGmYw_X-7LrbeVOgDPcRQdhu0oEuwQtKA,743
-ohmyscrapper/modules/load_txt.py,sha256=mL60OGsh-R80P88vxyqvfBEFag9yhSFFbg5pwtu1f90,889
-ohmyscrapper/modules/merge_dbs.py,sha256=0pK3PPUGSbnaDkdpQUGCHemOVaKO37bfHwnsy_EVpWQ,115
-ohmyscrapper/modules/process_with_ai.py,sha256=TpumucIVNZulKOw2idy4hD3vG5IhG5pbhyJImYFP8g0,6844
-ohmyscrapper/modules/scrap_urls.py,sha256=KQVs3R03X80hmvvJAU1SqnNhwXEeVV99WlN8TxSKqA8,6097
-ohmyscrapper/modules/seed.py,sha256=KeTSbmTdNTkVCtzk9iQmeuEqB0kG-rTZJb2a1WdROL4,129
-ohmyscrapper/modules/show.py,sha256=u0L9uxgU8Xt_-myA3r7byuOmnX_-2gkpTtXWkXon1ns,3572
-ohmyscrapper/modules/sniff_url.py,sha256=jQDc7aSimuOOedw2fSXZlf6_o0OqQHOr6NsWb4n0XgI,2720
-ohmyscrapper/modules/untouch_all.py,sha256=E1U9e3sOG7suzc8ZTWcYiQQo9mPmLJ0piXdXUjFLEd4,162
-ohmyscrapper-0.2.3.dist-info/WHEEL,sha256=xDCZ-UyfvkGuEHPeI7BcJzYKIZzdqN8A8o1M5Om8IyA,79
-ohmyscrapper-0.2.3.dist-info/entry_points.txt,sha256=BZud6D16XkfjelDa4Z33mji-KJbbZXgq2FoLrzjru5I,52
-ohmyscrapper-0.2.3.dist-info/METADATA,sha256=uwthvf7vwhb6H14KdbPLibzp1c3PaVmcJIYePmV8cRc,3832
-ohmyscrapper-0.2.3.dist-info/RECORD,,