ohmyscrapper 0.7.1__tar.gz → 0.7.4__tar.gz

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/PKG-INFO

@@ -1,9 +1,10 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: ohmyscrapper
-Version: 0.7.1
+Version: 0.7.4
 Summary: OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a final report with general information about job positions.
 Author: Cesar Cardoso
 Author-email: Cesar Cardoso <hello@cesarcardoso.cc>
+License-Expression: MIT
 Requires-Dist: beautifulsoup4>=4.14.3
 Requires-Dist: google-genai>=1.55.0
 Requires-Dist: markdown>=3.10
@@ -14,9 +15,11 @@ Requires-Dist: requests>=2.32.5
 Requires-Dist: rich>=14.2.0
 Requires-Dist: urlextract>=1.9.0
 Requires-Python: >=3.11
+Project-URL: Changelog, https://github.com/bouli/ohmyscrapper/releases/latest
+Project-URL: Repository, https://github.com/bouli/ohmyscrapper
 Description-Content-Type: text/markdown
 
-# 🐶 OhMyScrapper - v0.7.1
+# 🐶 OhMyScrapper - v0.7.4
 
 OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a
 final report with general information about job positions.

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/README.md

@@ -1,4 +1,4 @@
-# 🐶 OhMyScrapper - v0.7.1
+# 🐶 OhMyScrapper - v0.7.4
 
 OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a
 final report with general information about job positions.

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/pyproject.toml

@@ -1,6 +1,7 @@
 [project]
 name = "ohmyscrapper"
-version = "0.7.1"
+version = "0.7.4"
+license = "MIT"
 description = "OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a final report with general information about job positions."
 readme = "README.md"
 authors = [
@@ -19,6 +20,10 @@ dependencies = [
     "urlextract>=1.9.0",
 ]
 
+[project.urls]
+Repository = "https://github.com/bouli/ohmyscrapper"
+Changelog = "https://github.com/bouli/ohmyscrapper/releases/latest"
+
 [project.scripts]
 ohmyscrapper = "ohmyscrapper:main"
 

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/src/ohmyscrapper/__init__.py

@@ -20,7 +20,7 @@ from ohmyscrapper.core.config import update
 
 def main():
     parser = argparse.ArgumentParser(prog="ohmyscrapper")
-    parser.add_argument("--version", action="version", version="%(prog)s v0.7.1")
+    parser.add_argument("--version", action="version", version="%(prog)s v0.7.4")
 
     update()
     subparsers = parser.add_subparsers(dest="command", help="Available commands")
@@ -53,6 +53,14 @@ def main():
         help="Add all `url_types` from the bank to the `/ohmyscrapper/url_types.yaml` file.",
         action="store_true",
     )
+
+    seed_parser.add_argument(
+        "--reset",
+        default=False,
+        help="Reset all `url_types`.",
+        action="store_true",
+    )
+
     untouch_parser = subparsers.add_parser(
         "untouch-all", help="Untouch all urls. That resets classification"
     )
@@ -137,7 +145,7 @@ def main():
         if args.export:
             export_url_types_to_file()
         else:
-            seed()
+            seed(args.reset)
         return
 
     if args.command == "untouch-all":
@@ -191,6 +199,7 @@ def main():
         return
 
     if args.command == "start":
+        seed()
         if args.input != None:
             load_txt(file_name=args.input)
         else:

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/src/ohmyscrapper/core/config.py

@@ -39,6 +39,12 @@ def get_ai(param):
     )
 
 
+def get_sniffing(param):
+    return config_files.get_param(
+        parent_param="sniffing", param=param, default_app_dir=default_app_dir
+    )
+
+
 def load_config(force_default=False):
     config_file_name = "config.yaml"
     config_params = config_files.create_and_read_config_file(

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/src/ohmyscrapper/core/config_files.py

@@ -4,14 +4,29 @@ import yaml
 
 def create_and_read_config_file(file_name, default_app_dir, force_default=False):
     config_file = config_file_path(file_name, default_app_dir)
+    default_config_params = _get_default_file(default_file=file_name)
     if force_default or not os.path.exists(config_file):
-        config_params = _get_default_file(default_file=file_name)
         overwrite_config_file(
-            data=config_params, file_name=file_name, default_app_dir=default_app_dir
+            data=default_config_params,
+            file_name=file_name,
+            default_app_dir=default_app_dir,
         )
+        config_params = default_config_params
     else:
         with open(config_file, "r") as f:
             config_params = yaml.safe_load(f.read())
+        if complete_config_file(
+            config_params=config_params,
+            default_config_params=default_config_params,
+            file_name=file_name,
+            default_app_dir=default_app_dir,
+        ):
+            config_params = create_and_read_config_file(
+                file_name=file_name,
+                default_app_dir=default_app_dir,
+                force_default=force_default,
+            )
+
     if config_params is None:
         config_params = create_and_read_config_file(
             file_name=file_name, default_app_dir=default_app_dir, force_default=True
@@ -19,6 +34,18 @@ def create_and_read_config_file(file_name, default_app_dir, force_default=False)
     return config_params
 
 
+def complete_config_file(
+    config_params, default_config_params, file_name, default_app_dir
+):
+    has_updated = False
+    for key, values in default_config_params.items():
+        if key not in config_params.keys():
+            has_updated = True
+            data = {key: values}
+            append_config_file(data, file_name, default_app_dir)
+    return has_updated
+
+
 def overwrite_config_file(data, file_name, default_app_dir):
     config_file = config_file_path(file_name, default_app_dir)
     with open(config_file, "+w") as f:

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/src/ohmyscrapper/core/default_files/config.yaml

@@ -14,3 +14,6 @@ default_files:
 
 ai:
   default_prompt_file: prompt.md
+
+sniffing:
+  timeout: 10

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/src/ohmyscrapper/core/default_files/url_sniffing.yaml

@@ -23,3 +23,7 @@ linkedin_redirect:
     og:url: url_destiny
   atags:
     first-tag-as-url_destiny: 5
+
+read_all_a_tags:
+  atags:
+    load_atags: True

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/src/ohmyscrapper/models/urls_manager.py

@@ -69,6 +69,14 @@ def seeds(seeds={}):
     return True
 
 
+@use_connection
+def reset_seeds():
+    sql = "DELETE FROM urls_valid_prefix"
+    c = conn.cursor()
+    c.execute(sql)
+    conn.commit()
+
+
 @use_connection
 def add_urls_valid_prefix(url_prefix, url_type):
 
@@ -198,6 +206,8 @@ def get_url_like_unclassified(like_condition):
 
 @use_connection
 def add_url(url, title=None, parent_url=None):
+    if url[:1] == "/":
+        return
     url = clean_url(url)
     c = conn.cursor()
 
@@ -340,7 +350,9 @@ def set_url_error(url, value):
 @use_connection
 def set_url_type_by_id(url_id, url_type):
     c = conn.cursor()
-    c.execute(f"UPDATE urls SET url_type = '{url_type}' WHERE id = {url_id}")
+    c.execute(
+        f"UPDATE urls SET url_type = '{url_type}', last_touch = NULL WHERE id = {url_id}"
+    )
     conn.commit()
 
 
@@ -392,8 +404,10 @@ def touch_url(url):
 @use_connection
 def untouch_url(url):
     url = clean_url(url)
+    url = str(url.strip())
+
     c = conn.cursor()
-    c.execute("UPDATE urls SET last_touch = NULL WHERE url = ?", (url))
+    c.execute(f"UPDATE urls SET last_touch = NULL, url_type = NULL WHERE url = '{url}'")
     conn.commit()
 
 

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/src/ohmyscrapper/modules/load_txt.py

@@ -19,14 +19,16 @@ def load_txt(file_name="input", verbose=False):
 
     text_file_content = ""
     if file_name is not None and not os.path.isdir(file_name):
-        print(f"📖 reading file `{file_name}`... ")
         if not os.path.exists(file_name):
             if file_name.startswith("https://") or file_name.startswith("http://"):
+                print(f"📖 reading url `{file_name}`... ")
                 text_file_content = " " + file_name + " "
+                urls_manager.untouch_url(url=file_name)
             else:
                 print(f"\n file `{file_name}` not found.")
                 return
         else:
+            print(f"📖 reading file `{file_name}`... ")
             text_file_content = _increment_file_name(
                 text_file_content=text_file_content, file_name=file_name
             )
@@ -51,13 +53,15 @@ def load_txt(file_name="input", verbose=False):
                 file_name=os.path.join(dir_files, text_files[0]),
             )
         else:
-            print("\nChoose a text file. Use `*` for process all and `q` to quit:")
+            print("\nFiles list:")
             for index, file in enumerate(text_files):
                 print(f"[{index}]:", os.path.join(dir_files, file))
 
             text_file_option = -1
             while text_file_option < 0 or text_file_option >= len(text_files):
-                text_file_option = input("Enter the file number: ")
+                text_file_option = input(
+                    "Choose a text file. Use `*` for process all and `q` to quit. Enter the file number: "
+                )
                 if text_file_option == "*":
                     for file in text_files:
                         text_file_content = _increment_file_name(

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/src/ohmyscrapper/modules/scrap_urls.py

@@ -104,6 +104,12 @@ def process_sniffed_url(url_report, url, sniffing_config, verbose=False):
         ):
             if "first-a-link" in url_report.keys():
                 db_fields["url_destiny"] = url_report["first-a-link"]
+    if (
+        "atags" in sniffing_config.keys()
+        and "load_links" in sniffing_config["atags"].keys()
+    ):
+        for a_link in url_report["a_links"]:
+            urls_manager.add_url(url=a_link["href"], parent_url=url["url"])
 
     if db_fields["title"] is not None:
         urls_manager.set_url_title(url=url["url"], value=db_fields["title"])

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/src/ohmyscrapper/modules/seed.py

@@ -2,7 +2,10 @@ import ohmyscrapper.models.urls_manager as urls_manager
 from ohmyscrapper.core import config
 
 
-def seed():
+def seed(reset=False):
+    if reset:
+        urls_manager.reset_seeds()
+
     if not config.url_types_file_exists():
         db_url_types = urls_manager.get_urls_valid_prefix()
         if len(db_url_types) > 0:

{ohmyscrapper-0.7.1 → ohmyscrapper-0.7.4}/src/ohmyscrapper/modules/sniff_url.py

@@ -1,6 +1,7 @@
 import requests
 from bs4 import BeautifulSoup
 import json
+from ohmyscrapper.core import config
 
 
 def sniff_url(
@@ -8,6 +9,8 @@ def sniff_url(
     silent=False,
     sniffing_config={},
 ):
+    final_report = {}
+    final_report["error"] = None
     if "metatags" in sniffing_config:
         metatags_to_search = sniffing_config["metatags"]
     else:
@@ -41,10 +44,17 @@ def sniff_url(
     if not silent:
         print("checking url:", url)
 
-    r = requests.get(url=url)
-    soup = BeautifulSoup(r.text, "html.parser")
+    try:
+        r = requests.get(url=url, timeout=config.get_sniffing("timeout"))
+        soup = BeautifulSoup(r.text, "html.parser")
+    except requests.exceptions.ReadTimeout:
+        url_domain = url.split("/")[2]
+        final_report["error"] = (
+            f"!!! timeout (10 seconds) while checking the url with domain: `{url_domain}` !!!"
+        )
+        print(f"\n\n{final_report['error']}\n\n")
+        soup = BeautifulSoup("", "html.parser")
 
-    final_report = {}
     final_report["scrapped-url"] = url
     if len(metatags_to_search) > 0:
         final_report.update(