ohmyscrapper 0.4.0__tar.gz → 0.7.1__tar.gz

{ohmyscrapper-0.4.0 → ohmyscrapper-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ohmyscrapper
-Version: 0.4.0
+Version: 0.7.1
 Summary: OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a final report with general information about job positions.
 Author: Cesar Cardoso
 Author-email: Cesar Cardoso <hello@cesarcardoso.cc>
@@ -16,7 +16,7 @@ Requires-Dist: urlextract>=1.9.0
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 
-# 🐶 OhMyScrapper - v0.4.0
+# 🐶 OhMyScrapper - v0.7.1
 
 OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a
 final report with general information about job positions.

{ohmyscrapper-0.4.0 → ohmyscrapper-0.7.1}/README.md

@@ -1,4 +1,4 @@
-# 🐶 OhMyScrapper - v0.4.0
+# 🐶 OhMyScrapper - v0.7.1
 
 OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a
 final report with general information about job positions.

{ohmyscrapper-0.4.0 → ohmyscrapper-0.7.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ohmyscrapper"
-version = "0.4.0"
+version = "0.7.1"
 description = "OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a final report with general information about job positions."
 readme = "README.md"
 authors = [
@@ -29,11 +29,13 @@ build-backend = "uv_build"
 [tool.bumpversion]
 tag = true
 tag_name = "v{new_version}"
-pre_commit_hooks = ["uv sync --upgrade", "git add uv.lock"]
+pre_commit_hooks = ["uvx black ./src", "git add src", "git commit -m 'chore: Beautify with black'", "uv sync --upgrade", "git add uv.lock"]
 commit = true
 
 [[tool.bumpversion.files]]
 filename = "pyproject.toml"
+search = 'version = "{current_version}"'
+replace = 'version = "{new_version}"'
 
 [[tool.bumpversion.files]]
 filename = "README.md"

{ohmyscrapper-0.4.0 → ohmyscrapper-0.7.1}/src/ohmyscrapper/__init__.py

@@ -3,7 +3,7 @@ import argparse
 from ohmyscrapper.modules.classify_urls import classify_urls
 from ohmyscrapper.modules.sniff_url import sniff_url
 from ohmyscrapper.modules.load_txt import load_txt
-from ohmyscrapper.modules.seed import seed
+from ohmyscrapper.modules.seed import seed, export_url_types_to_file
 from ohmyscrapper.modules.scrap_urls import scrap_urls
 from ohmyscrapper.modules.show import (
     show_url,
@@ -15,17 +15,22 @@ from ohmyscrapper.modules.show import (
 from ohmyscrapper.modules.untouch_all import untouch_all
 from ohmyscrapper.modules.process_with_ai import process_with_ai, reprocess_ai_history
 from ohmyscrapper.modules.merge_dbs import merge_dbs
+from ohmyscrapper.core.config import update
 
 
 def main():
     parser = argparse.ArgumentParser(prog="ohmyscrapper")
-    parser.add_argument("--version", action="version", version="%(prog)s v0.4.0")
+    parser.add_argument("--version", action="version", version="%(prog)s v0.7.1")
 
+    update()
     subparsers = parser.add_subparsers(dest="command", help="Available commands")
     start_parser = subparsers.add_parser(
         "start",
         help="Make the entire process of 📦 loading, 🐶 scraping and 📜🖋️ exporting with the default configuration.",
     )
+    start_parser.add_argument(
+        "-input", default=None, help="File/Folder path or url for pre-loading."
+    )
 
     start_parser.add_argument(
         "--ai",
@@ -40,7 +45,13 @@ def main():
     )
 
     seed_parser = subparsers.add_parser(
-        "seed", help="Seed database. Necessary to classify urls."
+        "seed", help="Seed database with `url_types` to classify the `urls`."
+    )
+    seed_parser.add_argument(
+        "--export",
+        default=False,
+        help="Add all `url_types` from the bank to the `/ohmyscrapper/url_types.yaml` file.",
+        action="store_true",
     )
     untouch_parser = subparsers.add_parser(
         "untouch-all", help="Untouch all urls. That resets classification"
@@ -54,7 +65,9 @@ def main():
     )
 
     load_txt_parser = subparsers.add_parser("load", help="📦 Load txt file")
-    load_txt_parser.add_argument("-input", default=None, help="File path or url.")
+    load_txt_parser.add_argument(
+        "-input", default=None, help="File/Folder path or url."
+    )
     load_txt_parser.add_argument(
         "--verbose", default=False, help="Run in verbose mode", action="store_true"
     )
@@ -75,6 +88,9 @@ def main():
     scrap_urls_parser.add_argument(
         "--verbose", default=False, help="Run in verbose mode", action="store_true"
     )
+    scrap_urls_parser.add_argument(
+        "-input", default=None, help="File/Folder path or url for pre-loading."
+    )
 
     sniff_url_parser = subparsers.add_parser("sniff-url", help="🐕 Sniff/Check url")
     sniff_url_parser.add_argument(
@@ -118,7 +134,10 @@ def main():
         return
 
     if args.command == "seed":
-        seed()
+        if args.export:
+            export_url_types_to_file()
+        else:
+            seed()
         return
 
     if args.command == "untouch-all":
@@ -130,6 +149,9 @@ def main():
         return
 
     if args.command == "scrap-urls":
+        if args.input != None:
+            load_txt(file_name=args.input, verbose=args.verbose)
+
         scrap_urls(
             recursive=args.recursive,
             ignore_valid_prefix=args.ignore_type,
@@ -169,7 +191,11 @@ def main():
         return
 
     if args.command == "start":
-        load_txt()
+        if args.input != None:
+            load_txt(file_name=args.input)
+        else:
+            load_txt()
+
         scrap_urls(
             recursive=True,
             ignore_valid_prefix=True,

ohmyscrapper-0.7.1/src/ohmyscrapper/core/config.py

@@ -0,0 +1,107 @@
+import os
+from ohmyscrapper.core import config_files
+
+default_app_dir = "ohmyscrapper"
+
+
+def get_dir(param="ohmyscrapper"):
+    parent_param = "default_dirs"
+
+    if param == default_app_dir:
+        folder = "./" + param
+    else:
+        folder = config_files.get_param(
+            parent_param=parent_param, param=param, default_app_dir=default_app_dir
+        )
+    if not os.path.exists(folder):
+        os.mkdir(folder)
+    return folder
+
+
+def get_files(param):
+    parent_param = "default_files"
+    return config_files.get_param(
+        parent_param=parent_param, param=param, default_app_dir=default_app_dir
+    )
+
+
+def get_db(param="db_file"):
+    if param == "folder":
+        return get_dir(param="db")
+    return config_files.get_param(
+        parent_param="db", param=param, default_app_dir=default_app_dir
+    )
+
+
+def get_ai(param):
+    return config_files.get_param(
+        parent_param="ai", param=param, default_app_dir=default_app_dir
+    )
+
+
+def load_config(force_default=False):
+    config_file_name = "config.yaml"
+    config_params = config_files.create_and_read_config_file(
+        file_name=config_file_name,
+        default_app_dir=default_app_dir,
+        force_default=force_default,
+    )
+
+    if config_params is None or "default_dirs" not in config_params:
+        config_params = load_config(force_default=True)
+
+    return config_params
+
+
+def url_types_file_exists():
+    url_types_file = get_files("url_types")
+    return config_files.config_file_exists(
+        url_types_file, default_app_dir=default_app_dir
+    )
+
+
+def get_url_types():
+    url_types_file = get_files("url_types")
+    return config_files.create_and_read_config_file(
+        url_types_file, default_app_dir=default_app_dir
+    )
+
+
+def get_url_sniffing():
+    file = get_files("url_sniffing")
+    return config_files.create_and_read_config_file(
+        file, default_app_dir=default_app_dir
+    )
+
+
+def append_url_sniffing(data):
+    file = get_files("url_sniffing")
+    _append_config_file(data, file)
+
+
+def append_url_types(url_types):
+    url_types_file = get_files("url_types")
+    _append_config_file(url_types, url_types_file)
+
+
+def overwrite_config_file(data, file_name):
+    config_files.overwrite_config_file(data, file_name, default_app_dir=default_app_dir)
+
+
+def _append_config_file(data, file_name):
+    config_files.append_config_file(data, file_name, default_app_dir=default_app_dir)
+
+
+def update():
+    legacy_folder = "./customize"
+    new_folder = "./ohmyscrapper"
+    if os.path.exists(legacy_folder) and not os.path.exists(new_folder):
+        yes_no = input(
+            "We detected a legacy folder system for your OhMyScrapper, would you like to update? \n"
+            "If you don't update, a new version will be used and your legacy folder will be ignored. \n"
+            "[Y] for yes or  any other thing to ignore: "
+        )
+        if yes_no == "Y":
+            os.rename(legacy_folder, new_folder)
+        print(" You are up-to-date! =)")
+        print("")

ohmyscrapper-0.7.1/src/ohmyscrapper/core/config_files.py

@@ -0,0 +1,73 @@
+import os
+import yaml
+
+
+def create_and_read_config_file(file_name, default_app_dir, force_default=False):
+    config_file = config_file_path(file_name, default_app_dir)
+    if force_default or not os.path.exists(config_file):
+        config_params = _get_default_file(default_file=file_name)
+        overwrite_config_file(
+            data=config_params, file_name=file_name, default_app_dir=default_app_dir
+        )
+    else:
+        with open(config_file, "r") as f:
+            config_params = yaml.safe_load(f.read())
+    if config_params is None:
+        config_params = create_and_read_config_file(
+            file_name=file_name, default_app_dir=default_app_dir, force_default=True
+        )
+    return config_params
+
+
+def overwrite_config_file(data, file_name, default_app_dir):
+    config_file = config_file_path(file_name, default_app_dir)
+    with open(config_file, "+w") as f:
+        f.write(yaml.safe_dump(data))
+
+
+def append_config_file(data, file_name, default_app_dir):
+    config_file = config_file_path(file_name, default_app_dir)
+    # append
+    with open(config_file, "+a") as f:
+        yaml.dump(data, f, allow_unicode=True)
+    # read
+    with open(config_file, "r") as f:
+        data = yaml.safe_load(f.read())
+    # overwrite preventing repetition
+    with open(config_file, "w") as f:
+        yaml.dump(data, f, allow_unicode=True)
+
+
+def get_param(parent_param, param, default_app_dir):
+    default_dirs = create_and_read_config_file(
+        file_name="config.yaml", default_app_dir=default_app_dir
+    )[parent_param]
+
+    if param in default_dirs:
+        return default_dirs[param]
+    else:
+        raise Exception(f"{param} do not exist in your params {parent_param}.")
+
+
+def config_file_exists(file_name, default_app_dir):
+    return os.path.exists(config_file_path(file_name, default_app_dir))
+
+
+def config_file_path(file_name, default_app_dir):
+    _ensure_default_app_dir(default_app_dir)
+    config_file = os.path.join(default_app_dir, file_name)
+    return config_file
+
+
+def _ensure_default_app_dir(default_app_dir):
+    if not os.path.exists(default_app_dir):
+        os.mkdir(default_app_dir)
+
+
+def _get_default_file(default_file):
+    default_files_dir = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "default_files"
+    )
+    default_file = os.path.join(default_files_dir, default_file)
+    with open(default_file, "r") as f:
+        return yaml.safe_load(f.read())

ohmyscrapper-0.7.1/src/ohmyscrapper/core/default_files/config.yaml

@@ -0,0 +1,16 @@
+db:
+  db_file: local.db
+
+default_dirs:
+  db: ./db
+  input: ./input
+  output: ./output
+  prompts: ./prompts
+  templates: ./templates
+
+default_files:
+  url_types: url_types.yaml
+  url_sniffing: url_sniffing.yaml
+
+ai:
+  default_prompt_file: prompt.md

ohmyscrapper-0.7.1/src/ohmyscrapper/core/default_files/url_sniffing.yaml

@@ -0,0 +1,25 @@
+linkedin_feed:
+  metatags:
+    og:url: url_destiny
+
+linkedin_job:
+  bodytags:
+    h1: title
+  metatags:
+    og:title: title
+    og:description: description
+    description: description
+
+linkedin_post:
+  bodytags:
+    h1: title
+  metatags:
+    og:title: title
+    og:description: description
+    description: description
+
+linkedin_redirect:
+  metatags:
+    og:url: url_destiny
+  atags:
+    first-tag-as-url_destiny: 5

ohmyscrapper-0.7.1/src/ohmyscrapper/core/default_files/url_types.yaml

@@ -0,0 +1,5 @@
+linkedin_company: https://%.linkedin.com/company/%
+linkedin_feed: https://%.linkedin.com/feed/%
+linkedin_job: https://%.linkedin.com/jobs/view/%
+linkedin_post: https://%.linkedin.com/posts/%
+linkedin_redirect: https://lnkd.in/%

{ohmyscrapper-0.4.0 → ohmyscrapper-0.7.1}/src/ohmyscrapper/models/urls_manager.py

@@ -4,16 +4,19 @@ import time
 import glob
 import pandas as pd
 from urllib.parse import urlparse, urlunparse
+from ohmyscrapper.core import config
 
 
 def get_db_dir():
-    if not os.path.exists("db"):
-        os.mkdir("db")
-    return "db"
+    db_folder = config.get_dir("db")
+    if not os.path.exists(db_folder):
+        os.mkdir(db_folder)
+    return db_folder
 
 
 def get_db_path():
-    return get_db_dir() + "/local.db"
+    db_file = config.get_db()
+    return os.path.join(get_db_dir(), db_file)
 
 
 def get_db_connection():
@@ -26,7 +29,11 @@ def use_connection(func):
     def provide_connection(*args, **kwargs):
         global conn
         with get_db_connection() as conn:
-            return func(*args, **kwargs)
+            try:
+                return func(*args, **kwargs)
+            except:
+                update_db()
+                return func(*args, **kwargs)
 
     return provide_connection
 
@@ -35,7 +42,7 @@ def create_tables(conn):
 
     c = conn.cursor()
     c.execute(
-        "CREATE TABLE IF NOT EXISTS urls (id INTEGER PRIMARY KEY, url_type STRING, parent_url TEXT, url TEXT UNIQUE, url_destiny TEXT, h1 TEXT, error TEXT, description TEXT, description_links INTEGER DEFAULT 0, json TEXT, json_ai TEXT, ai_processed INTEGER DEFAULT 0, history INTEGER DEFAULT 0, last_touch DATETIME, created_at DATETIME)"
+        "CREATE TABLE IF NOT EXISTS urls (id INTEGER PRIMARY KEY, url_type STRING, parent_url TEXT, url TEXT UNIQUE, url_destiny TEXT, title TEXT, error TEXT, description TEXT, description_links INTEGER DEFAULT 0, json TEXT, json_ai TEXT, ai_processed INTEGER DEFAULT 0, history INTEGER DEFAULT 0, last_touch DATETIME, created_at DATETIME)"
     )
     c.execute(
         "CREATE TABLE IF NOT EXISTS ai_log (id INTEGER PRIMARY KEY, instructions STRING, response STRING, model STRING, prompt_file STRING, prompt_name STRING, created_at DATETIME)"
@@ -46,12 +53,18 @@ def create_tables(conn):
     )
 
 
-def seeds():
-    add_urls_valid_prefix("https://%.linkedin.com/posts/%", "linkedin_post")
-    add_urls_valid_prefix("https://lnkd.in/%", "linkedin_redirect")
-    add_urls_valid_prefix("https://%.linkedin.com/jobs/view/%", "linkedin_job")
-    add_urls_valid_prefix("https://%.linkedin.com/feed/%", "linkedin_feed")
-    add_urls_valid_prefix("https://%.linkedin.com/company/%", "linkedin_company")
+def update_db():
+    try:
+        c = conn.cursor()
+        c.execute("ALTER TABLE urls RENAME COLUMN h1 TO title")
+    except:
+        pass
+
+
+def seeds(seeds={}):
+
+    for url_type, url_prefix in seeds.items():
+        add_urls_valid_prefix(url_prefix, url_type)
 
     return True
 
@@ -117,7 +130,7 @@ def get_urls_report():
         SELECT
             u.id,
             u.url,
-            u.h1
+            u.title
             FROM urls u
                 INNER JOIN parent_url p
                     ON u.url = p.parent_url
@@ -126,9 +139,9 @@ def get_urls_report():
         u.id,
         u.url_type,
         u.url,
-        COALESCE(u.h1, p.h1) as h1,
+        COALESCE(u.title, p.title) as title,
         p.url as parent_url,
-        p.h1 as parent_h1
+        p.title as parent_title
         FROM urls u
         LEFT JOIN parents p
             ON u.parent_url = p.url
@@ -184,12 +197,12 @@ def get_url_like_unclassified(like_condition):
 
 
 @use_connection
-def add_url(url, h1=None, parent_url=None):
+def add_url(url, title=None, parent_url=None):
     url = clean_url(url)
     c = conn.cursor()
 
-    if h1 is not None:
-        h1 = h1.strip()
+    if title is not None:
+        title = title.strip()
 
     if parent_url is None:
         parent_url = None
@@ -198,8 +211,8 @@ def add_url(url, h1=None, parent_url=None):
 
     if len(get_url_by_url(url)) == 0:
         c.execute(
-            "INSERT INTO urls (url, h1, parent_url, created_at, ai_processed, description_links, history) VALUES (?, ?, ?, ?, 0, 0, 0)",
-            (url, h1, parent_url, int(time.time())),
+            "INSERT INTO urls (url, title, parent_url, created_at, ai_processed, description_links, history) VALUES (?, ?, ?, ?, 0, 0, 0)",
+            (url, title, parent_url, int(time.time())),
         )
         conn.commit()
 
@@ -238,20 +251,20 @@ def set_url_destiny(url, destiny):
 
 
 @use_connection
-def set_url_h1(url, value):
+def set_url_title(url, value):
     value = str(value).strip()
     url = clean_url(url)
     c = conn.cursor()
-    c.execute("UPDATE urls SET h1 = ? WHERE url = ?", (value, url))
+    c.execute("UPDATE urls SET title = ? WHERE url = ?", (value, url))
     conn.commit()
 
 
 @use_connection
-def set_url_h1_by_id(id, value):
+def set_url_title_by_id(id, value):
     value = str(value).strip()
 
     c = conn.cursor()
-    c.execute("UPDATE urls SET h1 = ? WHERE id = ?", (value, id))
+    c.execute("UPDATE urls SET title = ? WHERE id = ?", (value, id))
     conn.commit()
 
 
@@ -426,16 +439,16 @@ def merge_dbs() -> None:
 
 
 @use_connection
-def merge_url(url, h1, last_touch, created_at, description, json):
+def merge_url(url, title, last_touch, created_at, description, json):
     url = clean_url(url)
     c = conn.cursor()
 
-    if h1 is not None:
-        h1 = h1.strip()
+    if title is not None:
+        title = title.strip()
 
     if len(get_url_by_url(url)) == 0:
         c.execute(
-            "INSERT INTO urls (url, h1, last_touch , created_at, history, ai_processed, description_links, description, json) VALUES (?, ?, ?, ?, 1, 0, 0, ? , ?)",
-            (url, h1, last_touch, created_at, description, json),
+            "INSERT INTO urls (url, title, last_touch , created_at, history, ai_processed, description_links, description, json) VALUES (?, ?, ?, ?, 1, 0, 0, ? , ?)",
+            (url, title, last_touch, created_at, description, json),
         )
         conn.commit()

{ohmyscrapper-0.4.0 → ohmyscrapper-0.7.1}/src/ohmyscrapper/modules/classify_urls.py

@@ -1,11 +1,15 @@
 import ohmyscrapper.models.urls_manager as urls_manager
+from ohmyscrapper.modules import seed
 import pandas as pd
 import time
 
 
 def classify_urls(recursive=False):
-    urls_manager.seeds()
     df = urls_manager.get_urls_valid_prefix()
+    if len(df) == 0:
+        seed.seed()
+        classify_urls(recursive=recursive)
+        return
 
     keep_alive = True
     while keep_alive:

{ohmyscrapper-0.4.0 → ohmyscrapper-0.7.1}/src/ohmyscrapper/modules/load_txt.py

@@ -1,6 +1,7 @@
 import os
 from urlextract import URLExtract
 import ohmyscrapper.models.urls_manager as urls_manager
+from ohmyscrapper.core import config
 
 
 def _increment_file_name(text_file_content, file_name):
@@ -9,17 +10,15 @@ def _increment_file_name(text_file_content, file_name):
         return text_file_content + f.read()
 
 
-def load_txt(file_name=None, verbose=False):
-    if not os.path.exists("db"):
-        os.mkdir("db")
-
-    if not os.path.exists("input"):
-        os.mkdir("input")
+def load_txt(file_name="input", verbose=False):
+    input_folder = config.get_dir("input")
+    if not os.path.exists(input_folder):
+        os.mkdir(input_folder)
 
     urls_manager.seeds()
 
     text_file_content = ""
-    if file_name is not None:
+    if file_name is not None and not os.path.isdir(file_name):
         print(f"📖 reading file `{file_name}`... ")
         if not os.path.exists(file_name):
             if file_name.startswith("https://") or file_name.startswith("http://"):
@@ -32,27 +31,30 @@ def load_txt(file_name=None, verbose=False):
                 text_file_content=text_file_content, file_name=file_name
             )
     else:
-        print("📂 reading /input directory... ")
-        dir_files = "input"
+        input_folder = config.get_dir("input")
+        print(f"📂 reading {input_folder} directory... ")
+        if file_name is None:
+            dir_files = input_folder
+        else:
+            dir_files = file_name
         text_files = os.listdir(dir_files)
         for file in text_files:
             if not file.endswith(".txt"):
                 text_files.remove(file)
         if len(text_files) == 0:
-            print("No text files found in /input directory!")
+            print(f"No text files found in {input_folder} directory!")
             return
         elif len(text_files) == 1:
             print(f"📖 reading file `{dir_files}/{text_files[0]}`... ")
             text_file_content = _increment_file_name(
                 text_file_content=text_file_content,
-                file_name=dir_files + "/" + text_files[0],
+                file_name=os.path.join(dir_files, text_files[0]),
             )
         else:
             print("\nChoose a text file. Use `*` for process all and `q` to quit:")
             for index, file in enumerate(text_files):
-                print(f"[{index}]:", dir_files + "/" + file)
+                print(f"[{index}]:", os.path.join(dir_files, file))
 
-            # TODO: there is a better way for sure!
             text_file_option = -1
             while text_file_option < 0 or text_file_option >= len(text_files):
                 text_file_option = input("Enter the file number: ")
@@ -60,7 +62,7 @@ def load_txt(file_name=None, verbose=False):
                     for file in text_files:
                         text_file_content = _increment_file_name(
                             text_file_content=text_file_content,
-                            file_name=dir_files + "/" + file,
+                            file_name=os.path.join(dir_files, file),
                         )
                         text_file_option = 0
                 elif text_file_option == "q":
@@ -70,9 +72,9 @@ def load_txt(file_name=None, verbose=False):
                     if text_file_option >= 0 and text_file_option < len(text_files):
                         text_file_content = _increment_file_name(
                             text_file_content=text_file_content,
-                            file_name=dir_files
-                            + "/"
-                            + text_files[int(text_file_option)],
+                            file_name=os.path.join(
+                                dir_files, text_files[int(text_file_option)]
+                            ),
                         )
 
     print("🔎 looking for urls...")