ohmyscrapper 0.7.4__py3-none-any.whl → 0.8.2__py3-none-any.whl

ohmyscrapper/__init__.py

@@ -20,7 +20,7 @@ from ohmyscrapper.core.config import update
 
 def main():
     parser = argparse.ArgumentParser(prog="ohmyscrapper")
-    parser.add_argument("--version", action="version", version="%(prog)s v0.7.4")
+    parser.add_argument("--version", action="version", version="%(prog)s v0.8.2")
 
     update()
     subparsers = parser.add_subparsers(dest="command", help="Available commands")
@@ -104,6 +104,16 @@ def main():
     sniff_url_parser.add_argument(
         "url", default="https://cesarcardoso.cc/", help="Url to sniff"
     )
+    sniff_url_parser.add_argument(
+        "--metatags",
+        default="mt",
+        help="Meta tags you want to watch separated by comma ','",
+    )
+    sniff_url_parser.add_argument(
+        "--bodytags",
+        default="bd",
+        help="Body tags you want to watch separated by comma ','",
+    )
 
     show_urls_parser = subparsers.add_parser("show", help="Show urls and prefixes")
     show_urls_parser.add_argument(
@@ -153,7 +163,15 @@ def main():
         return
 
     if args.command == "sniff-url":
-        sniff_url(args.url)
+        sniffing_config = {}
+        if len(args.metatags) > 0:
+            sniffing_config["metatags"] = str(args.metatags).split(",")
+
+        if len(args.bodytags) > 0:
+            sniffing_config["bodytags"] = str(args.bodytags).split(",")
+
+        sniff_url(args.url, sniffing_config=sniffing_config)
+
         return
 
     if args.command == "scrap-urls":

ohmyscrapper/core/config.py

@@ -69,14 +69,14 @@ def url_types_file_exists():
 def get_url_types():
     url_types_file = get_files("url_types")
     return config_files.create_and_read_config_file(
-        url_types_file, default_app_dir=default_app_dir
+        url_types_file, default_app_dir=default_app_dir, complete_file=False
     )
 
 
 def get_url_sniffing():
     file = get_files("url_sniffing")
     return config_files.create_and_read_config_file(
-        file, default_app_dir=default_app_dir
+        file, default_app_dir=default_app_dir, complete_file=False
     )
 
 

ohmyscrapper/core/config_files.py

@@ -2,7 +2,9 @@ import os
 import yaml
 
 
-def create_and_read_config_file(file_name, default_app_dir, force_default=False):
+def create_and_read_config_file(
+    file_name, default_app_dir, force_default=False, complete_file=True
+):
     config_file = config_file_path(file_name, default_app_dir)
     default_config_params = _get_default_file(default_file=file_name)
     if force_default or not os.path.exists(config_file):
@@ -15,17 +17,18 @@ def create_and_read_config_file(file_name, default_app_dir, force_default=False)
     else:
         with open(config_file, "r") as f:
             config_params = yaml.safe_load(f.read())
-        if complete_config_file(
-            config_params=config_params,
-            default_config_params=default_config_params,
-            file_name=file_name,
-            default_app_dir=default_app_dir,
-        ):
-            config_params = create_and_read_config_file(
+        if complete_file:
+            if complete_config_file(
+                config_params=config_params,
+                default_config_params=default_config_params,
                 file_name=file_name,
                 default_app_dir=default_app_dir,
-                force_default=force_default,
-            )
+            ):
+                config_params = create_and_read_config_file(
+                    file_name=file_name,
+                    default_app_dir=default_app_dir,
+                    force_default=force_default,
+                )
 
     if config_params is None:
         config_params = create_and_read_config_file(

ohmyscrapper/core/default_files/config.yaml

@@ -17,3 +17,6 @@ ai:
 
 sniffing:
   timeout: 10
+  use-browser: false
+  browser-waiting-time: 5
+  round-sleeping: 10

ohmyscrapper/core/default_files/url_sniffing.yaml

@@ -26,4 +26,4 @@ linkedin_redirect:
 
 read_all_a_tags:
   atags:
-    load_atags: True
+    load_links: True

ohmyscrapper/models/urls_manager.py

@@ -71,7 +71,7 @@ def seeds(seeds={}):
 
 @use_connection
 def reset_seeds():
-    sql = "DELETE FROM urls_valid_prefix"
+    sql = "DELETE FROM urls_valid_prefix WHERE 1 = 1"
     c = conn.cursor()
     c.execute(sql)
     conn.commit()

ohmyscrapper/modules/browser.py

@@ -0,0 +1,27 @@
+from selenium import webdriver
+from ohmyscrapper.core.config import get_sniffing
+
+
+def get_driver():
+    if get_sniffing("use-browser") == "safari":
+        from selenium.webdriver.safari.options import Options
+
+        options = Options()
+        driver = webdriver.Safari(options=options)
+    elif get_sniffing("use-browser") == "firefox":
+        from selenium.webdriver.firefox.options import Options
+
+        options = Options()
+        driver = webdriver.Firefox(options=options)
+    elif get_sniffing("use-browser") == "ie":
+        from selenium.webdriver.ie.options import Options
+
+        options = Options()
+        driver = webdriver.Ie(options=options)
+    else:  # default: chrome
+        from selenium.webdriver.chrome.options import Options
+
+        options = Options()
+        driver = webdriver.Chrome(options=options)
+
+    return driver

ohmyscrapper/modules/scrap_urls.py

@@ -2,13 +2,14 @@ import ohmyscrapper.models.urls_manager as urls_manager
 import ohmyscrapper.modules.sniff_url as sniff_url
 import ohmyscrapper.modules.load_txt as load_txt
 import ohmyscrapper.modules.classify_urls as classify_urls
+import ohmyscrapper.modules.browser as browser
 from ohmyscrapper.core import config
 
 import time
 import random
 
 
-def scrap_url(url, verbose=False):
+def scrap_url(url, verbose=False, driver=None):
     if url["url_type"] is None:
         url["url_type"] = "generic"
 
@@ -32,7 +33,7 @@ def scrap_url(url, verbose=False):
             sniffing_config = config.get_url_sniffing()
 
         url_report = sniff_url.get_tags(
-            url=url["url"], sniffing_config=sniffing_config[url_type]
+            url=url["url"], sniffing_config=sniffing_config[url_type], driver=driver
         )
     except Exception as e:
         urls_manager.set_url_error(url=url["url"], value="error on scrapping")
@@ -147,6 +148,7 @@ def scrap_urls(
     only_parents=True,
     verbose=False,
     n_urls=0,
+    driver=None,
 ):
     limit = 10
     classify_urls.classify_urls()
@@ -170,13 +172,19 @@ def scrap_urls(
         time.sleep(wait)
 
         print("🐕 Scrapper is sniffing the url...")
-        scrap_url(url=url, verbose=verbose)
+
+        if driver is None and config.get_sniffing("use-browser"):
+            driver = browser.get_driver()
+        scrap_url(url=url, verbose=verbose, driver=driver)
 
     n_urls = n_urls + len(urls)
     print(f"-- 🗃️ {n_urls} scraped urls...")
     classify_urls.classify_urls()
     if recursive:
-        wait = random.randint(5, 10)
+        wait = random.randint(
+            int(config.get_sniffing("round-sleeping") / 2),
+            int(config.get_sniffing("round-sleeping")),
+        )
         print(
             f"🐶 Scrapper is sleeping for {wait} seconds before next round of {limit} urls"
         )
@@ -188,6 +196,7 @@ def scrap_urls(
             only_parents=only_parents,
             verbose=verbose,
             n_urls=n_urls,
+            driver=driver,
         )
     else:
         print("scrapping is over...")

ohmyscrapper/modules/sniff_url.py

@@ -2,15 +2,17 @@ import requests
 from bs4 import BeautifulSoup
 import json
 from ohmyscrapper.core import config
+import ohmyscrapper.modules.browser as browser
+import time
 
 
 def sniff_url(
     url="https://www.linkedin.com/in/cesardesouzacardoso/",
     silent=False,
     sniffing_config={},
+    driver=None,
 ):
     final_report = {}
-    final_report["error"] = None
     if "metatags" in sniffing_config:
         metatags_to_search = sniffing_config["metatags"]
     else:
@@ -45,13 +47,14 @@ def sniff_url(
         print("checking url:", url)
 
     try:
-        r = requests.get(url=url, timeout=config.get_sniffing("timeout"))
-        soup = BeautifulSoup(r.text, "html.parser")
+        r = get_url(url=url, driver=driver)
+        soup = BeautifulSoup(r, "html.parser")
     except requests.exceptions.ReadTimeout:
         url_domain = url.split("/")[2]
         final_report["error"] = (
             f"!!! timeout (10 seconds) while checking the url with domain: `{url_domain}` !!!"
         )
+
         print(f"\n\n{final_report['error']}\n\n")
         soup = BeautifulSoup("", "html.parser")
 
@@ -69,14 +72,14 @@ def sniff_url(
                 soup=soup, silent=silent, body_tags_to_search=body_tags_to_search
             )
         )
-    final_report["a_links"] = _extract_a_tags(soup=soup, silent=silent)
+    final_report["a_links"] = _extract_a_tags(soup=soup, silent=silent, url=url)
     final_report = _complementary_report(final_report, soup, silent).copy()
     final_report["json"] = json.dumps(final_report)
 
     return final_report
 
 
-def _extract_a_tags(soup, silent):
+def _extract_a_tags(soup, silent, url=None):
     a_links = []
     if not silent:
         print("\n\n\n\n---- all <a> links ---")
@@ -84,12 +87,18 @@ def _extract_a_tags(soup, silent):
     i = 0
     for a_tag in soup.find_all("a"):
         i = i + 1
-        a_links.append({"text": a_tag.text, "href": a_tag.get("href")})
+
+        href = a_tag.get("href")
+        if url is not None and href[:1] == "/":
+            domain = url.split("//")[0] + "//" + url.split("//")[1].split("/")[0]
+            href = domain + href
+
+        a_links.append({"text": a_tag.text, "href": href})
         if not silent:
             print("\n-- <a> link", i, "-- ")
             print("target:", a_tag.get("target"))
             print("text:", str(a_tag.text).strip())
-            print("href:", a_tag.get("href"))
+            print("href:", href)
             print("-------------- ")
     return a_links
 
@@ -124,9 +133,21 @@ def _extract_text_tags(soup, silent, body_tags_to_search):
         print("\n\n\n\n---- all <text> tags ---\n")
     i = 0
     for text_tag, separator in body_tags_to_search.items():
-        if len(soup.find_all(text_tag)) > 0:
+        tag = text_tag
+        tag_class = None
+        tag_id = None
+
+        if len(text_tag.split(".")) > 1:
+            tag = text_tag.split(".")[0]
+            tag_class = text_tag.split(".")[1]
+
+        if len(text_tag.split("#")) > 1:
+            tag = text_tag.split("#")[0]
+            tag_id = text_tag.split("#")[1]
+
+        if len(soup.find_all(tag, class_=tag_class, id=tag_id)) > 0:
             valid_text_tags[text_tag] = []
-            for obj_tag in soup.find_all(text_tag):
+            for obj_tag in soup.find_all(tag, class_=tag_class, id=tag_id):
                 valid_text_tags[text_tag].append(obj_tag.text.strip())
             valid_text_tags[text_tag] = separator.join(valid_text_tags[text_tag])
             i = i + 1
@@ -161,5 +182,23 @@ def _complementary_report(final_report, soup, silent):
     return final_report
 
 
-def get_tags(url, sniffing_config={}):
-    return sniff_url(url=url, silent=True, sniffing_config=sniffing_config)
+def get_tags(url, sniffing_config={}, driver=None):
+    return sniff_url(
+        url=url, silent=True, sniffing_config=sniffing_config, driver=driver
+    )
+
+
+def get_url(url, driver=None):
+    if driver is None and config.get_sniffing("use-browser"):
+        driver = browser.get_driver()
+
+    if driver is not None:
+        try:
+            driver.get(url)
+            time.sleep(config.get_sniffing("browser-waiting-time"))
+            driver.implicitly_wait(config.get_sniffing("browser-waiting-time"))
+            return driver.page_source
+        except:
+            print("error")
+            pass
+    return requests.get(url=url, timeout=config.get_sniffing("timeout")).text

{ohmyscrapper-0.7.4.dist-info → ohmyscrapper-0.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ohmyscrapper
-Version: 0.7.4
+Version: 0.8.2
 Summary: OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a final report with general information about job positions.
 Author: Cesar Cardoso
 Author-email: Cesar Cardoso <hello@cesarcardoso.cc>
@@ -13,13 +13,14 @@ Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: requests>=2.32.5
 Requires-Dist: rich>=14.2.0
+Requires-Dist: selenium>=4.39.0
 Requires-Dist: urlextract>=1.9.0
 Requires-Python: >=3.11
 Project-URL: Changelog, https://github.com/bouli/ohmyscrapper/releases/latest
 Project-URL: Repository, https://github.com/bouli/ohmyscrapper
 Description-Content-Type: text/markdown
 
-# 🐶 OhMyScrapper - v0.7.4
+# 🐶 OhMyScrapper - v0.8.2
 
 OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a
 final report with general information about job positions.

ohmyscrapper-0.8.2.dist-info/RECORD

@@ -0,0 +1,22 @@
+ohmyscrapper/__init__.py,sha256=WzXXhhlVkyAPbqeo7NgezLb6TbEJcuFf8JYAdcf3zBE,7678
+ohmyscrapper/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
+ohmyscrapper/core/config.py,sha256=bfmoTr1j4SnIhKFZC_F9fh4Y90VqSPFf_g6Rm-aNui4,3184
+ohmyscrapper/core/config_files.py,sha256=3mIXVxurmyXCpKueyyGsZ6lUnV8VJ2gnLU2QaqhWhhI,3410
+ohmyscrapper/core/default_files/config.yaml,sha256=y54QAjOnogpl8LEzhmn89tAfRzle4ZWWtIbYRjxX8Rk,341
+ohmyscrapper/core/default_files/url_sniffing.yaml,sha256=HUwmGUwuJy7t97bJHgNiZOl1thvD9bLaelPgbEr5bMY,465
+ohmyscrapper/core/default_files/url_types.yaml,sha256=20kvv8_iWRT-pLa014RXYpAmPSonn6tDnG302rx7l-o,228
+ohmyscrapper/models/urls_manager.py,sha256=sP2T4k1HOj8ccaVGWbuhfw1BDfOUSVL4_WR9vRyHjOA,12115
+ohmyscrapper/modules/browser.py,sha256=6AaNFQ7jV91DvHqbsBT6It_-tNbVN2qJC_c1vXTweJY,856
+ohmyscrapper/modules/classify_urls.py,sha256=GhiosAQUITy1DQe_PksYV9QRKVTgpkSE28dkutzbWVA,1038
+ohmyscrapper/modules/load_txt.py,sha256=pkWBIdh6vORPfENDZ6wGM89vswnOnc1flqKfkLs9RD8,4138
+ohmyscrapper/modules/merge_dbs.py,sha256=0pK3PPUGSbnaDkdpQUGCHemOVaKO37bfHwnsy_EVpWQ,115
+ohmyscrapper/modules/process_with_ai.py,sha256=kl39Jzl-PUwh6AfmTZ9SLFUYs9Sk4biqgt8rNz3X1FA,7255
+ohmyscrapper/modules/scrap_urls.py,sha256=_e4jT7eBWGP6cqI6RaD0xzNX1vCFBx96JIBPGW3mAV4,6627
+ohmyscrapper/modules/seed.py,sha256=hHEGSoPXsmclTaRPeIcK2oC1Xpg3_JqBv_YFMD0m5Jw,1044
+ohmyscrapper/modules/show.py,sha256=jsAs4g8ouA9wymkBfkDCbpVWKD-m_20uKG-m1cZAUGA,3877
+ohmyscrapper/modules/sniff_url.py,sha256=BZphbr2V7MDyFPW7APlh7_CLTtc_u3kcB7DY2QjVVQo,6579
+ohmyscrapper/modules/untouch_all.py,sha256=DAwWYfqMFifHPtFCxSamu0AxHCgk6aJbTnBy6wLucXM,167
+ohmyscrapper-0.8.2.dist-info/WHEEL,sha256=xDCZ-UyfvkGuEHPeI7BcJzYKIZzdqN8A8o1M5Om8IyA,79
+ohmyscrapper-0.8.2.dist-info/entry_points.txt,sha256=BZud6D16XkfjelDa4Z33mji-KJbbZXgq2FoLrzjru5I,52
+ohmyscrapper-0.8.2.dist-info/METADATA,sha256=2OnXXefFcRT_ChEVdL6LKZoe6iNKyFqjoN10LBUBB34,4293
+ohmyscrapper-0.8.2.dist-info/RECORD,,

ohmyscrapper-0.7.4.dist-info/RECORD

@@ -1,21 +0,0 @@
-ohmyscrapper/__init__.py,sha256=x3wLMhIU744W9DRtXoTrPpWghb7UdC3UJSYZh_gpzlw,7095
-ohmyscrapper/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
-ohmyscrapper/core/config.py,sha256=aaSLxk6Fuzp88EMax6MAOX3WszH4OfYLz_dJoXlu0ME,3142
-ohmyscrapper/core/config_files.py,sha256=C79-Vgz1E5_jUWtob-yrCyBxsqWEXxqPI_r6TL7D1_Q,3314
-ohmyscrapper/core/default_files/config.yaml,sha256=gi8tqhSumQYJIl8QDisJ6eaib2tdcBNT-GFU-e6Dtns,273
-ohmyscrapper/core/default_files/url_sniffing.yaml,sha256=RU5GYWmC1PdBl4nn7HUfRBwuXz8Rlap75d4W3zWDzPM,465
-ohmyscrapper/core/default_files/url_types.yaml,sha256=20kvv8_iWRT-pLa014RXYpAmPSonn6tDnG302rx7l-o,228
-ohmyscrapper/models/urls_manager.py,sha256=k0N1If4YoRUWHX80OyBNEeJNIzDROc2ur6j8q2OBlqo,12103
-ohmyscrapper/modules/classify_urls.py,sha256=GhiosAQUITy1DQe_PksYV9QRKVTgpkSE28dkutzbWVA,1038
-ohmyscrapper/modules/load_txt.py,sha256=pkWBIdh6vORPfENDZ6wGM89vswnOnc1flqKfkLs9RD8,4138
-ohmyscrapper/modules/merge_dbs.py,sha256=0pK3PPUGSbnaDkdpQUGCHemOVaKO37bfHwnsy_EVpWQ,115
-ohmyscrapper/modules/process_with_ai.py,sha256=kl39Jzl-PUwh6AfmTZ9SLFUYs9Sk4biqgt8rNz3X1FA,7255
-ohmyscrapper/modules/scrap_urls.py,sha256=uN5j0dychVMGu7n1rcpYdba4sqc47ssyCn0tVaiz-Ic,6264
-ohmyscrapper/modules/seed.py,sha256=hHEGSoPXsmclTaRPeIcK2oC1Xpg3_JqBv_YFMD0m5Jw,1044
-ohmyscrapper/modules/show.py,sha256=jsAs4g8ouA9wymkBfkDCbpVWKD-m_20uKG-m1cZAUGA,3877
-ohmyscrapper/modules/sniff_url.py,sha256=1QnxEdCWLjLh0uM72dlPzst64qglqg2MHA_xYlNcLSA,5435
-ohmyscrapper/modules/untouch_all.py,sha256=DAwWYfqMFifHPtFCxSamu0AxHCgk6aJbTnBy6wLucXM,167
-ohmyscrapper-0.7.4.dist-info/WHEEL,sha256=xDCZ-UyfvkGuEHPeI7BcJzYKIZzdqN8A8o1M5Om8IyA,79
-ohmyscrapper-0.7.4.dist-info/entry_points.txt,sha256=BZud6D16XkfjelDa4Z33mji-KJbbZXgq2FoLrzjru5I,52
-ohmyscrapper-0.7.4.dist-info/METADATA,sha256=CVE8WUcraUtONy9UVIU0y8Y7wjsk4zEmMVfpA_al1CU,4261
-ohmyscrapper-0.7.4.dist-info/RECORD,,