ohmyscrapper 0.1.1__py3-none-any.whl

ohmyscrapper/__init__.py

@@ -0,0 +1,155 @@
+import argparse
+
+from ohmyscrapper.modules.classify_urls import classify_urls
+from ohmyscrapper.modules.sniff_url import sniff_url
+from ohmyscrapper.modules.load_txt import load_txt
+from ohmyscrapper.modules.seed import seed
+from ohmyscrapper.modules.scrap_urls import scrap_urls
+from ohmyscrapper.modules.show import (
+    show_url,
+    show_urls,
+    show_urls_valid_prefix,
+    export_urls,
+    export_report,
+)
+from ohmyscrapper.modules.untouch_all import untouch_all
+from ohmyscrapper.modules.process_with_ai import process_with_ai
+from ohmyscrapper.modules.merge_dbs import merge_dbs
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="ohmyscrapper")
+    parser.add_argument("--version", action="version", version="%(prog)s v0.1.1")
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    ai_process_parser = subparsers.add_parser(
+        "process-with-ai", help="Process with AI."
+    )
+    seed_parser = subparsers.add_parser(
+        "seed", help="Seed database. Necessary to classify urls."
+    )
+    untouch_parser = subparsers.add_parser(
+        "untouch-all", help="Untouch all urls. That resets classification"
+    )
+
+    classify_urls_parser = subparsers.add_parser(
+        "classify-urls", help="Classify loaded urls"
+    )
+    classify_urls_parser.add_argument(
+        "--recursive", default=False, help="Run in recursive mode", action="store_true"
+    )
+
+    load_txt_parser = subparsers.add_parser("load", help="Load txt file")
+    load_txt_parser.add_argument(
+        "-file", default="input/_chat.txt", help="File path. Default is input/_chat.txt"
+    )
+
+    scrap_urls_parser = subparsers.add_parser("scrap-urls", help="Scrap urls")
+    scrap_urls_parser.add_argument(
+        "--recursive", default=False, help="Run in recursive mode", action="store_true"
+    )
+    scrap_urls_parser.add_argument(
+        "--ignore-type", default=False, help="Ignore urls types", action="store_true"
+    )
+    scrap_urls_parser.add_argument(
+        "--randomize", default=False, help="Random order", action="store_true"
+    )
+    scrap_urls_parser.add_argument(
+        "--only-parents", default=False, help="Only parents urls", action="store_true"
+    )
+
+    sniff_url_parser = subparsers.add_parser("sniff-url", help="Check url")
+    sniff_url_parser.add_argument(
+        "url", default="https://cesarcardoso.cc/", help="Url to sniff"
+    )
+
+    show_urls_parser = subparsers.add_parser("show", help="Show urls and prefixes")
+    show_urls_parser.add_argument(
+        "--prefixes", default=False, help="Show urls valid prefix", action="store_true"
+    )
+    show_urls_parser.add_argument("--limit", default=0, help="Limit of lines to show")
+    show_urls_parser.add_argument("-url", default="", help="Url to show")
+
+    export_parser = subparsers.add_parser("export", help="Export urls to csv.")
+    export_parser.add_argument("--limit", default=0, help="Limit of lines to export")
+    export_parser.add_argument(
+        "--file",
+        default="output/urls.csv",
+        help="File path. Default is output/urls.csv",
+    )
+    export_parser.add_argument(
+        "--simplify",
+        default=False,
+        help="Ignore json and descriptions",
+        action="store_true",
+    )
+
+    report_parser = subparsers.add_parser("report", help="Export urls report to csv.")
+    merge_parser = subparsers.add_parser("merge_dbs", help="Merge databases.")
+
+    # TODO: What is that?
+    # seed_parser.set_defaults(func=seed)
+    # classify_urls_parser.set_defaults(func=classify_urls)
+    # load_txt_parser.set_defaults(func=load_txt)
+
+    args = parser.parse_args()
+
+    if args.command == "classify-urls":
+        classify_urls(args.recursive)
+        return
+
+    if args.command == "load":
+        load_txt(args.file)
+        return
+
+    if args.command == "seed":
+        seed()
+        return
+
+    if args.command == "untouch-all":
+        untouch_all()
+        return
+
+    if args.command == "sniff-url":
+        sniff_url(args.url)
+        return
+
+    if args.command == "scrap-urls":
+        scrap_urls(
+            recursive=args.recursive,
+            ignore_valid_prefix=args.ignore_type,
+            randomize=args.randomize,
+            only_parents=args.only_parents,
+        )
+        return
+
+    if args.command == "show":
+        if args.prefixes:
+            show_urls_valid_prefix(int(args.limit))
+            return
+        if args.url != "":
+            show_url(args.url)
+            return
+        show_urls(int(args.limit))
+        return
+
+    if args.command == "export":
+        export_urls(limit=int(args.limit), csv_file=args.file, simplify=args.simplify)
+        return
+
+    if args.command == "process-with-ai":
+        process_with_ai()
+        return
+
+    if args.command == "report":
+        export_report()
+        return
+
+    if args.command == "merge_dbs":
+        merge_dbs()
+        return
+
+
+if __name__ == "__main__":
+    main()

ohmyscrapper/__main__.py

@@ -0,0 +1,4 @@
+from . import main
+
+if __name__ == "__main__":
+    main()

ohmyscrapper/models/urls_manager.py

@@ -0,0 +1,388 @@
+import os
+import sqlite3
+import time
+import glob
+import pandas as pd
+from urllib.parse import urlparse, urlunparse
+
+
+def get_db_dir():
+    if not os.path.exists("db"):
+        os.mkdir("db")
+    return "db"
+
+
+def get_db_path():
+    return get_db_dir() + "/local.db"
+
+
+def get_db_connection():
+    return sqlite3.connect(get_db_path())
+
+
+# TODO: check if it makes sense
+conn = get_db_connection()
+
+
+def create_tables():
+
+    c = conn.cursor()
+    c.execute(
+        "CREATE TABLE IF NOT EXISTS urls (id INTEGER PRIMARY KEY, url_type STRING, parent_id INTEGER DEFAULT 0, url TEXT UNIQUE, url_destiny TEXT, h1 TEXT, error TEXT, description TEXT, description_links INTEGER DEFAULT 0, json TEXT, ai_processed INTEGER DEFAULT 0, history INTEGER DEFAULT 0, last_touch DATETIME, created_at DATETIME)"
+    )
+    c.execute(
+        "CREATE TABLE IF NOT EXISTS ai_log (id INTEGER PRIMARY KEY, instructions STRING, response STRING, model STRING, created_at DATETIME)"
+    )
+
+    c.execute(
+        "CREATE TABLE IF NOT EXISTS urls_valid_prefix (id INTEGER PRIMARY KEY, url_prefix TEXT UNIQUE, url_type TEXT)"
+    )
+
+    return pd.read_sql_query("SELECT * FROM urls LIMIT 100", conn)
+
+
+# TODO: not sure this should be something. depends on the project
+def seeds():
+    create_tables()
+
+    add_urls_valid_prefix("https://%.linkedin.com/posts/%", "linkedin_post")
+    add_urls_valid_prefix("https://lnkd.in/%", "linkedin_redirect")
+    add_urls_valid_prefix("https://%.linkedin.com/jobs/view/%", "linkedin_job")
+    add_urls_valid_prefix("https://%.linkedin.com/feed/%", "linkedin_feed")
+    add_urls_valid_prefix("https://%.linkedin.com/company/%", "linkedin_company")
+
+    # add_urls_valid_prefix("%.pdf", "pdf")
+    # add_url('https://imazon.org.br/categorias/artigos-cientificos/')
+
+    return True
+
+
+def add_urls_valid_prefix(url_prefix, url_type):
+    conn = get_db_connection()
+
+    df = pd.read_sql_query(
+        f"SELECT * FROM urls_valid_prefix WHERE url_prefix = '{url_prefix}'", conn
+    )
+    if len(df) == 0:
+        c = conn.cursor()
+        c.execute(
+            "INSERT INTO urls_valid_prefix (url_prefix, url_type) VALUES (?, ?)",
+            (url_prefix, url_type),
+        )
+        conn.commit()
+
+
+def get_urls_valid_prefix_by_type(url_type):
+    df = pd.read_sql_query(
+        f"SELECT * FROM urls_valid_prefix WHERE url_type = '{url_type}'", conn
+    )
+    return df
+
+
+def get_urls_valid_prefix_by_id(id):
+    df = pd.read_sql_query(f"SELECT * FROM urls_valid_prefix WHERE id = '{id}'", conn)
+    return df
+
+
+# TODO: pagination required
+def get_urls_valid_prefix(limit=0):
+    if limit > 0:
+        df = pd.read_sql_query(f"SELECT * FROM urls_valid_prefix LIMIT {limit}", conn)
+    else:
+        df = pd.read_sql_query(f"SELECT * FROM urls_valid_prefix", conn)
+    return df
+
+
+# TODO: pagination required
+def get_urls(limit=0):
+    if limit > 0:
+        df = pd.read_sql_query(
+            f"SELECT * FROM urls LIMIT {limit} ORDER BY history ASC", conn
+        )
+    else:
+        df = pd.read_sql_query(f"SELECT * FROM urls ORDER BY history ASC", conn)
+    return df
+
+
+def get_urls_report():
+    sql = """
+    WITH parents_id AS (
+        SELECT parent_id FROM urls WHERE parent_id != 0 GROUP BY parent_id
+    ),
+    parents AS (
+        SELECT
+            u.id,
+            u.h1
+            FROM urls u
+                INNER JOIN parents_id p
+                    ON u.id = p.parent_id
+    )
+    SELECT
+        u.id,
+        u.url_type,
+        u.url,
+        u.h1,
+        p.id as parent_id,
+        p.h1 as parent_h1
+        FROM urls u
+        LEFT JOIN parents p
+            ON u.parent_id = p.id
+        WHERE u.history = 0 AND u.id NOT IN (SELECT id FROM parents)
+    """
+    df = pd.read_sql_query(sql, conn)
+
+    return df
+
+
+def get_url_by_url(url):
+    url = clean_url(url)
+    df = pd.read_sql_query(f"SELECT * FROM urls WHERE url = '{url}'", conn)
+
+    return df
+
+
+def get_url_by_id(id):
+    df = pd.read_sql_query(f"SELECT * FROM urls WHERE id = '{id}'", conn)
+
+    return df
+
+
+def get_urls_by_url_type(url_type):
+    df = pd.read_sql_query(
+        f"SELECT * FROM urls WHERE history = 0 AND url_type = '{url_type}'", conn
+    )
+    return df
+
+
+def get_urls_by_url_type_for_ai_process(url_type="linkedin_post", limit=10):
+    df = pd.read_sql_query(
+        f"SELECT * FROM urls WHERE history = 0 AND url_type = '{url_type}' AND ai_processed = 0 LIMIT {limit}",
+        conn,
+    )
+    return df
+
+
+def get_url_like_unclassified(like_condition):
+    df = pd.read_sql_query(
+        f"SELECT * FROM urls WHERE history = 0 AND url LIKE '{like_condition}' AND url_type IS NULL",
+        conn,
+    )
+    return df
+
+
+def add_url(url, h1=None, parent_id=0):
+    url = clean_url(url)
+    c = conn.cursor()
+
+    if h1 is not None:
+        h1 = h1.strip()
+
+    if parent_id is None:
+        parent_id = 0
+
+    parent_id = int(parent_id)
+
+    if len(get_url_by_url(url)) == 0:
+        c.execute(
+            "INSERT INTO urls (url, h1, parent_id, created_at, ai_processed, description_links, history) VALUES (?, ?, ?, ?, 0, 0, 0)",
+            (url, h1, parent_id, int(time.time())),
+        )
+        conn.commit()
+
+    return get_url_by_url(url)
+
+
+def add_ai_log(instructions, response, model):
+    c = conn.cursor()
+
+    c.execute(
+        "INSERT INTO ai_log (instructions, response, model, created_at) VALUES (?, ?, ?, ?)",
+        (instructions, response, model, int(time.time())),
+    )
+    conn.commit()
+
+
+def set_url_destiny(url, destiny):
+    url = clean_url(url)
+    destiny = clean_url(destiny)
+    c = conn.cursor()
+    url_obj = get_url_by_url(url)
+    c.execute("UPDATE urls SET url_destiny = ? WHERE url = ?", (destiny, url))
+    c.execute(
+        "UPDATE urls SET parent_id = ? WHERE url = ?",
+        (int(url_obj.iloc[0]["id"]), destiny),
+    )
+
+    conn.commit()
+
+
+def set_url_h1(url, value):
+    value = str(value).strip()
+    url = clean_url(url)
+    c = conn.cursor()
+    c.execute("UPDATE urls SET h1 = ? WHERE url = ?", (value, url))
+    conn.commit()
+
+
+def set_url_h1_by_id(id, value):
+    value = str(value).strip()
+
+    c = conn.cursor()
+    c.execute("UPDATE urls SET h1 = ? WHERE id = ?", (value, id))
+    conn.commit()
+
+
+def set_url_ai_processed_by_id(id):
+    value = 1
+    value = str(value).strip()
+    c = conn.cursor()
+    c.execute("UPDATE urls SET ai_processed = ? WHERE id = ?", (value, id))
+    conn.commit()
+
+
+def set_url_ai_processed_by_url(url):
+    value = 1
+    value = str(value).strip()
+    url = clean_url(url)
+    c = conn.cursor()
+    c.execute("UPDATE urls SET ai_processed = ? WHERE url = ?", (value, url))
+    conn.commit()
+
+
+def set_url_description(url, value):
+    url = clean_url(url)
+    c = conn.cursor()
+    c.execute("UPDATE urls SET description = ? WHERE url = ?", (value, url))
+    conn.commit()
+
+
+def set_url_description_links(url, value):
+    url = clean_url(url)
+    c = conn.cursor()
+    c.execute("UPDATE urls SET description_links = ? WHERE url = ?", (value, url))
+    conn.commit()
+
+
+def set_url_json(url, value):
+    url = clean_url(url)
+    c = conn.cursor()
+    c.execute("UPDATE urls SET json = ? WHERE url = ?", (value, url))
+    conn.commit()
+
+
+def set_url_error(url, value):
+    url = clean_url(url)
+    c = conn.cursor()
+    c.execute("UPDATE urls SET error = ? WHERE url = ?", (value, url))
+    conn.commit()
+
+
+def set_url_type_by_id(url_id, url_type):
+    c = conn.cursor()
+    c.execute(f"UPDATE urls SET url_type = '{url_type}' WHERE id = {url_id}")
+    conn.commit()
+
+
+def clean_url(url):
+    if url[0:7] == "http://":
+        url = "https://" + url[7:]
+
+    if url[0:8] != "https://":
+        url = "https://" + url
+    url = url.split("#")[0]
+    old_query = urlparse(url).query.split("&")
+    new_query = []
+    for i in old_query:
+        if i[0:4] != "utm_":
+            new_query.append(i)
+
+    url = urlunparse(urlparse(url)._replace(query="&".join(new_query))).replace("'", "")
+    return url
+
+
+def get_untouched_urls(
+    limit=10, randomize=True, ignore_valid_prefix=False, only_parents=True
+):
+    where_sql = ""
+    if not ignore_valid_prefix:
+        where_sql += " AND url_type IS NOT NULL "
+
+    if only_parents:
+        where_sql += " AND parent_id = 0 "
+
+    if randomize:
+        random_sql = " RANDOM() "
+    else:
+        random_sql = " created_at DESC "
+    sql = f"SELECT * FROM urls WHERE 1 = 1 AND history = 0 {where_sql} AND last_touch IS NULL ORDER BY {random_sql} LIMIT {limit}"
+    df = pd.read_sql_query(sql, conn)
+    return df
+
+
+def touch_url(url):
+    url = clean_url(url)
+    c = conn.cursor()
+    c.execute("UPDATE urls SET last_touch = ? WHERE url = ?", (int(time.time()), url))
+    conn.commit()
+
+
+def untouch_url(url):
+    url = clean_url(url)
+    c = conn.cursor()
+    c.execute("UPDATE urls SET last_touch = NULL WHERE url = ?", (url))
+    conn.commit()
+
+
+def untouch_all_urls():
+    c = conn.cursor()
+    c.execute("UPDATE urls SET last_touch = NULL WHERE history = 0")
+    conn.commit()
+
+
+def set_all_urls_as_history():
+    c = conn.cursor()
+    c.execute("UPDATE urls SET history = 1")
+    conn.commit()
+
+
+def merge_dbs() -> None:
+    production_db_file = get_db_path()
+    db_number = -1
+    dir = get_db_dir()
+    list_of_files = glob.glob(dir + "/*.db")
+    list_of_files.remove(production_db_file)
+    if len(list_of_files) > 0:
+        print("\nAvailable dbs:")
+        for index, file in enumerate(list_of_files):
+            print(index, ":", file)
+        while db_number < 0 or db_number >= len(list_of_files):
+            db_number = int(input("Choose the db to merge: "))
+
+        print(list_of_files[db_number])
+        source_conn = sqlite3.connect(list_of_files[db_number])
+        df = pd.read_sql_query("SELECT * FROM urls", source_conn)
+        for index, row in df.iterrows():
+            merge_url(
+                row["url"],
+                f"merged from {list_of_files[db_number]}",
+                row["last_touch"],
+                row["created_at"],
+                row["description"],
+                row["json"],
+            )
+        # ßmerge_url(df)
+
+
+def merge_url(url, h1, last_touch, created_at, description, json):
+    url = clean_url(url)
+    c = conn.cursor()
+
+    if h1 is not None:
+        h1 = h1.strip()
+
+    if len(get_url_by_url(url)) == 0:
+        c.execute(
+            "INSERT INTO urls (url, h1, last_touch , created_at, history, ai_processed, description_links, description, json) VALUES (?, ?, ?, ?, 1, 0, 0, ? , ?)",
+            (url, h1, last_touch, created_at, description, json),
+        )
+        conn.commit()

ohmyscrapper/modules/classify_urls.py

@@ -0,0 +1,23 @@
+import ohmyscrapper.models.urls_manager as urls_manager
+import pandas as pd
+import time
+
+
+def classify_urls(recursive=False):
+    urls_manager.seeds()
+    df = urls_manager.get_urls_valid_prefix()
+
+    keep_alive = True
+    while keep_alive:
+        print("waking up!")
+        for index, row_prefix in df.iterrows():
+            df_urls = urls_manager.get_url_like_unclassified(row_prefix["url_prefix"])
+            for index, row_urls in df_urls.iterrows():
+                urls_manager.set_url_type_by_id(row_urls["id"], row_prefix["url_type"])
+
+        if not recursive:
+            print("ending...")
+            keep_alive = False
+        else:
+            print("sleeping...")
+            time.sleep(10)

ohmyscrapper/modules/load_txt.py

@@ -0,0 +1,32 @@
+import os
+from urlextract import URLExtract
+import ohmyscrapper.models.urls_manager as urls_manager
+
+
+def load_txt(file_name="input/_chat.txt"):
+
+    if not os.path.exists("input"):
+        os.mkdir("input")
+
+    urls_manager.create_tables()
+    urls_manager.seeds()
+    # make it recursive for all files
+    text_file_content = open(file_name, "r").read()
+
+    put_urls_from_string(text_file_content)
+
+    # move_it_to_processed
+    print("--------------------")
+    print(file_name, "processed")
+
+
+def put_urls_from_string(text_to_process, parent_id=None):
+    if isinstance(text_to_process, str):
+        extractor = URLExtract()
+        for url in extractor.find_urls(text_to_process):
+            urls_manager.add_url(url, parent_id=parent_id)
+            print(url, "added")
+
+        return len(extractor.find_urls(text_to_process))
+    else:
+        return 0

ohmyscrapper/modules/merge_dbs.py

@@ -0,0 +1,6 @@
+import ohmyscrapper.models.urls_manager as urls_manager
+
+
+def merge_dbs():
+    urls_manager.merge_dbs()
+    return

ohmyscrapper/modules/process_with_ai.py

@@ -0,0 +1,160 @@
+import ohmyscrapper.models.urls_manager as urls_manager
+from bs4 import BeautifulSoup
+from google import genai
+from dotenv import load_dotenv
+import random
+import time
+import os
+import yaml
+
+load_dotenv()
+
+
+def process_with_ai(recursive=True):
+    prompt = _get_prompt()
+    if not prompt:
+        return
+
+    url_type = "linkedin_post"
+    df = urls_manager.get_urls_by_url_type_for_ai_process(url_type)
+
+    if len(df) == 0:
+        print("no urls to process with ai anymore")
+        return
+    texts = ""
+    for index, row in df.iterrows():
+        texts = (
+            texts
+            + f"""
+        <texto>
+        <id>{str(row['id'])}</id>
+        {row['description']}
+        </texto>
+        """
+        )
+    if texts == "":
+        print("no urls to process")
+        return
+
+    print("starting...")
+    print("prompt:", prompt["name"])
+    print("model:", prompt["model"])
+    print("description:", prompt["description"])
+    prompt["instrusctions"] = prompt["instrusctions"].replace("{ohmyscrapper_texts}", texts)
+
+    # The client gets the API key from the environment variable `GEMINI_API_KEY`.
+    client = genai.Client()
+    response = client.models.generate_content(model=prompt["model"], contents=prompt["instrusctions"])
+    response = str(response.text)
+    urls_manager.add_ai_log(instructions=prompt["instrusctions"], response=response, model=prompt["model"])
+    print(response)
+    print("^^^^^^")
+    soup = BeautifulSoup(response, "html.parser")
+    for vaga in soup.find_all(prompt["xml-item"]):
+
+        url = urls_manager.get_url_by_id(vaga.find("id").text)
+        if len(url) > 0:
+            url = url.iloc[0]
+        # TODO: make it dynamic
+        h1 = vaga.find("titulo").text
+        if (
+            vaga.find("contratante").text != "desconhecido"
+            and vaga.find("contratante").text != ""
+        ):
+            h1 = h1 + " - " + vaga.find("contratante").text
+        if url["description_links"] > 1 and vaga.find("id").text != "":
+            urls_manager.set_url_h1(vaga.find("url").text, h1)
+            urls_manager.set_url_ai_processed_by_url(vaga.find("url").text)
+
+            print("-- child updated -- ", vaga.find("url").text, h1)
+        elif url["description_links"] <= 1:
+            urls_manager.set_url_h1_by_id(vaga.find("id").text, h1)
+            urls_manager.set_url_ai_processed_by_id(vaga.find("id").text)
+            print("-- parent updated -- ", url["url"], h1)
+        else:
+            print("-- not updated -- ", url["url"], h1)
+
+    print("ending...")
+
+    if recursive:
+        wait = random.randint(1, 3)
+        print("sleeping for", wait, "seconds before next round")
+        time.sleep(wait)
+        process_with_ai(recursive=recursive)
+
+    return
+
+def _get_prompt():
+    prompts_path = "prompts"
+    default_prompt = """---
+model: "gemini-2.5-flash"
+name: "default-prompt"
+description: "Put here your prompt description."
+xml-item: "position"
+---
+Process with AI this prompt: {ohmyscrapper_texts}
+"""
+    if not os.path.exists(prompts_path):
+        os.mkdir(prompts_path)
+
+        open(f"{prompts_path}/prompt.md", "w").write(default_prompt)
+        print(f"You didn't have a prompt file. One was created in the /{prompts_path} folder. You can change it there.")
+        return False
+
+    prompt_files = os.listdir(prompts_path)
+    if len(prompt_files) == 0:
+        open(f"{prompts_path}/prompt.md", "w").write(default_prompt)
+        print(f"You didn't have a prompt file. One was created in the /{prompts_path} folder. You can change it there.")
+        return False
+
+    if len(prompt_files) == 1:
+        prompt = _parse_prompt(prompts_path, prompt_files[0])
+    else:
+        print("Choose a prompt:")
+        prompts = {}
+        for index, file in enumerate(prompt_files):
+            prompts[index] = _parse_prompt(prompts_path, file)
+            print(index, ":", prompts[index]['name'])
+        input_prompt = input("Type the number of the prompt you want to use or 'q' to quit: ")
+        if input_prompt == "q":
+            return False
+        try:
+            prompt = prompts[int(input_prompt)]
+        except:
+            print("! Invalid prompt\n")
+            prompt = _get_prompt()
+
+    return prompt
+
+def _parse_prompt(prompts_path, prompt_file):
+    prompt = {}
+    raw_prompt = open(f"{prompts_path}/{prompt_file}", "r").read().split("---")
+    prompt = yaml.safe_load(raw_prompt[1])
+    prompt["instrusctions"] = raw_prompt[2].strip()
+
+    return prompt
+# TODO: Separate gemini from basic function
+def _process_with_gemini(model, instructions):
+    response = """"""
+    return response
+
+
+def _process_with_openai(model, instructions):
+    # import os
+    # from openai import OpenAI
+
+    # client = OpenAI(
+    #    # This is the default and can be omitted
+    #    api_key=os.environ.get("OPENAI_API_KEY"),
+    # )
+
+    # response = client.responses.create(
+    #    model="gpt-4o",
+    #    instructions="You are a coding assistant that talks like a pirate.",
+    #    input="How do I check if a Python object is an instance of a class?",
+    # )
+
+    # print(response.output_text)
+
+    response = """"""
+    return response

ohmyscrapper/modules/scrap_urls.py

@@ -0,0 +1,179 @@
+import ohmyscrapper.models.urls_manager as urls_manager
+import ohmyscrapper.modules.sniff_url as sniff_url
+import ohmyscrapper.modules.load_txt as load_txt
+import ohmyscrapper.modules.classify_urls as classify_urls
+
+import time
+import random
+
+
+def process_linkedin_redirect(url_report, url):
+    print("linkedin_redirect")
+
+    if url_report["total-a-links"] < 5:
+        if "first-a-link" in url_report.keys():
+            url_destiny = url_report["first-a-link"]
+        else:
+            urls_manager.set_url_error(url["url"], "error: no first-a-link")
+            print("no url for:", url["url"])
+            return
+    else:
+        if "og:url" in url_report.keys():
+            url_destiny = url_report["og:url"]
+        else:
+            urls_manager.set_url_error(url["url"], "error: no og:url")
+            print("no url for:", url["url"])
+            return
+
+    print(url["url"], ">>", url_destiny)
+    urls_manager.add_url(url_destiny)
+    urls_manager.set_url_destiny(url["url"], url_destiny)
+
+
+def process_linkedin_feed(url_report, url):
+    print("linkedin_feed")
+
+    if "og:url" in url_report.keys():
+        url_destiny = url_report["og:url"]
+    else:
+        urls_manager.set_url_error(url["url"], "error: no og:url")
+        print("no url for:", url["url"])
+        return
+
+    print(url["url"], ">>", url_destiny)
+    urls_manager.add_url(url_destiny)
+    urls_manager.set_url_destiny(url["url"], url_destiny)
+
+
+def process_linkedin_job(url_report, url):
+    print("linkedin_job")
+    changed = False
+    if "h1" in url_report.keys():
+        print(url["url"], ": ", url_report["h1"])
+        urls_manager.set_url_h1(url["url"], url_report["h1"])
+        changed = True
+    elif "og:title" in url_report.keys():
+        print(url["url"], ": ", url_report["og:title"])
+        urls_manager.set_url_h1(url["url"], url_report["og:title"])
+        changed = True
+
+    if "description" in url_report.keys():
+        urls_manager.set_url_description(url["url"], url_report["description"])
+        changed = True
+    elif "og:description" in url_report.keys():
+        urls_manager.set_url_description(url["url"], url_report["og:description"])
+        changed = True
+    if not changed:
+        urls_manager.set_url_error(url["url"], "error: no h1 or description")
+
+
+def process_linkedin_post(url_report, url):
+    print("linkedin_post or generic")
+    print(url["url"])
+    changed = False
+    if "h1" in url_report.keys():
+        print(url["url"], ": ", url_report["h1"])
+        urls_manager.set_url_h1(url["url"], url_report["h1"])
+        changed = True
+    elif "og:title" in url_report.keys():
+        urls_manager.set_url_h1(url["url"], url_report["og:title"])
+        changed = True
+    description = None
+    if "description" in url_report.keys():
+        description = url_report["description"]
+        changed = True
+    elif "og:description" in url_report.keys():
+        description = url_report["og:description"]
+        changed = True
+
+    if description is not None:
+        urls_manager.set_url_description(url["url"], description)
+        description_links = load_txt.put_urls_from_string(description, url["id"])
+        urls_manager.set_url_description_links(url["url"], description_links)
+
+    if not changed:
+        urls_manager.set_url_error(url["url"], "error: no h1 or description")
+
+
+def scrap_url(url):
+    # TODO: Use get_urls_valid_prefix_by_id()
+    df = urls_manager.get_urls_valid_prefix()
+    url_valid_prefixes = df.set_index(df.id).T.to_dict()
+
+    # TODO: Need to change this
+
+    if url["url_type"] is None:
+        print("\n\ngeneric:", url["url"])
+        url["url_type"] = "generic"
+    else:
+        print("\n\n", url["url_type"] + ":", url["url"])
+    try:
+        url_report = sniff_url.get_tags(url["url"])
+    except Exception as e:
+        urls_manager.set_url_error(url["url"], "error")
+        urls_manager.touch_url(url["url"])
+        print("\n\n!!! ERROR FOR:", url["url"])
+        print(
+            "\n\n!!! you can check the URL using the command sniff-url",
+            url["url"],
+            "\n\n",
+        )
+        return
+
+    # linkedin_redirect - linkedin (https://lnkd.in/)
+    if url["url_type"] == "linkedin_redirect":
+        process_linkedin_redirect(url_report, url)
+
+    # linkedin_feed - linkedin (https://%.linkedin.com/feed/)
+    if url["url_type"] == "linkedin_feed":
+        process_linkedin_feed(url_report, url)
+
+    # linkedin_job - linkedin (https://www.linkedin.com/jobs/)
+    if url["url_type"] == "linkedin_job":
+        process_linkedin_job(url_report, url)
+
+    # linkedin_job - linkedin (https://www.linkedin.com/jobs/)
+    if url["url_type"] == "linkedin_post" or url["url_type"] == "generic":
+        process_linkedin_post(url_report, url)
+
+    urls_manager.set_url_json(url["url"], url_report["json"])
+    urls_manager.touch_url(url["url"])
+
+
+def isNaN(num):
+    return num != num
+
+
+def scrap_urls(
+    recursive=False, ignore_valid_prefix=False, randomize=False, only_parents=True
+):
+    classify_urls.classify_urls()
+    urls = urls_manager.get_untouched_urls(
+        ignore_valid_prefix=ignore_valid_prefix,
+        randomize=randomize,
+        only_parents=only_parents,
+    )
+    if len(urls) == 0:
+        print("no urls to scrap")
+        return
+    for index, url in urls.iterrows():
+        scrap_url(url)
+
+        wait = random.randint(15, 20)
+        wait = random.randint(1, 3)
+        print("sleeping for", wait, "seconds")
+        time.sleep(wait)
+
+    classify_urls.classify_urls()
+    if recursive:
+        wait = random.randint(5, 10)
+        print("sleeping for", wait, "seconds before next round")
+        time.sleep(wait)
+        scrap_urls(
+            recursive=recursive,
+            ignore_valid_prefix=ignore_valid_prefix,
+            randomize=randomize,
+            only_parents=only_parents,
+        )
+    else:
+        print("ending...")

ohmyscrapper/modules/seed.py

@@ -0,0 +1,7 @@
+import ohmyscrapper.models.urls_manager as urls_manager
+
+
+def seed():
+    urls_manager.seeds()
+    print("db seeded")
+    return

ohmyscrapper/modules/show.py

@@ -0,0 +1,116 @@
+import ohmyscrapper.models.urls_manager as urls_manager
+import math
+from rich.console import Console
+from rich.table import Table
+
+
+def export_urls(limit=0, csv_file="output/urls.csv", simplify=False):
+    df = urls_manager.get_urls(limit)
+
+    if simplify:
+        df.drop(columns=["description", "json"], inplace=True)
+
+    df.to_csv(csv_file, index=False)
+    print("--------------------")
+    print("Urls exported to", csv_file)
+
+    df.replace(
+        {
+            "description": {r"\n": " "},
+        },
+        regex=True,
+        inplace=True,
+    )
+    df.to_html(csv_file + "-preview.html", index=False)
+    print("Urls preview exported to", csv_file + "-preview.html")
+    print("--------------------")
+
+
+def export_report(csv_file="output/report.csv"):
+    df = urls_manager.get_urls_report()
+
+    df.to_csv(csv_file, index=False)
+    print("--------------------")
+    print("Urls report exported to", csv_file)
+
+    df.replace(
+        {
+            "description": {r"\n": " "},
+        },
+        regex=True,
+        inplace=True,
+    )
+    df.to_html(csv_file + "-preview.html", index=False)
+    print("Urls report preview exported to", csv_file + "-preview.html")
+    print("--------------------")
+
+
+def show_urls(limit=0, jump_to_page=0):
+    df = urls_manager.get_urls(limit)
+    df.drop(columns=["json", "description"], inplace=True)
+    # df = df.head(n=20)
+
+    # https://medium.com/@inzaniak/create-tables-in-your-terminal-with-python-6747d68d71a6
+
+    total_items = len(df)
+    items_per_page = 15
+    n_pages = math.ceil(total_items / items_per_page)
+
+    last_popped = 0
+    for page in range(n_pages):
+
+        df_page = df.head(n=items_per_page)
+        df_t = df.T
+        for i in range(items_per_page):
+            if last_popped < total_items:
+                df_t.pop(last_popped)
+            last_popped += 1
+        df = df_t.T
+        if page < jump_to_page:
+            continue
+        show_table(df_page)
+
+        print("Page", page + 1, "of", n_pages)
+        user_input = input("Press enter to continue or type q to quit: ")
+        if user_input == "q":
+            break
+        if user_input.isnumeric():
+            jump_to_page = math.ceil(int(user_input))
+            if jump_to_page > n_pages or jump_to_page < 1:
+                print("This page does not exist")
+                jump_to_page = 0
+            else:
+                jump_to_page = jump_to_page - 1
+                if page < jump_to_page:
+                    continue
+                elif jump_to_page >= 0:
+                    show_urls(limit=limit, jump_to_page=jump_to_page)
+                    break
+
+    return
+
+    return
+
+
+# TODO: Change place
+def show_table(df):
+    columns = df.columns.tolist()
+    df = df.to_dict(orient="records")
+    table = Table(show_header=True, header_style="bold magenta")
+    for column in columns:
+        table.add_column(column)
+
+    for row in df:
+        table.add_row(*[str(value) for value in row.values()])
+    console = Console()
+    console.print(table)
+
+
+def show_urls_valid_prefix(limit=0):
+    print(urls_manager.get_urls_valid_prefix(limit))
+    return
+
+
+def show_url(url):
+    print(urls_manager.get_url_by_url(url).T)
+    return

ohmyscrapper/modules/sniff_url.py

@@ -0,0 +1,88 @@
+import requests
+from bs4 import BeautifulSoup
+import json
+
+
+def sniff_url(url="https://www.linkedin.com/in/cesardesouzacardoso/", silent=False):
+    if not silent:
+        print("checking url:", url)
+    report_meta_tags = []
+    tags_to_search = [
+        "description",
+        "og:url",
+        "og:title",
+        "og:description",
+        "og:type",
+        "lnkd:url",
+    ]
+
+    r = requests.get(url=url)
+    soup = BeautifulSoup(r.text, "html.parser")
+
+    if not silent:
+        print("\n\n\n\n---- all <meta> tags ---\n")
+    i = 0
+    for meta_tag in soup.find_all("meta"):
+        if (
+            meta_tag.get("name") in tags_to_search
+            or meta_tag.get("property") in tags_to_search
+        ):
+            report_meta_tags.append(meta_tag)
+        i = i + 1
+        if not silent:
+            print("-- meta tag", i, "--")
+            print("name:", meta_tag.get("name"))
+            print("property:", meta_tag.get("property"))
+            print("content:", meta_tag.get("content"))
+            print("---------------- \n")
+
+    if not silent:
+        print("\n\n\n\n---- all <a> links ---")
+        i = 0
+        for a_tag in soup.find_all("a"):
+            i = i + 1
+            print("\n-- a link", i, "-- ")
+            print("target:", a_tag.get("target"))
+            print("text:", a_tag.text)
+            print("href:", a_tag.get("href"))
+            print("-------------- ")
+
+    final_report = {}
+    final_report["scrapped-url"] = url
+    if len(soup.find_all("h1")) > 0:
+        final_report["h1"] = soup.find("h1").text
+
+    for report_meta_tag in report_meta_tags:
+        if report_meta_tag.get("name") is not None:
+            final_report[report_meta_tag.get("name")] = report_meta_tag.get("content")
+        elif report_meta_tag.get("property") is not None:
+            final_report[report_meta_tag.get("property")] = report_meta_tag.get(
+                "content"
+            )
+
+    if len(soup.find_all("a")) > 0:
+        final_report["first-a-link"] = soup.find("a").get("href")
+        final_report["total-a-links"] = len(soup.find_all("a"))
+    else:
+        final_report["first-a-link"] = ""
+        final_report["total-a-links"] = 0
+
+    if len(soup.find_all("h2")) > 0:
+        final_report["h2"] = soup.find("h2").text
+
+    if len(soup.find_all("meta")) > 0:
+        final_report["total-meta-tags"] = len(soup.find_all("meta"))
+    else:
+        final_report["total-meta-tags"] = 0
+
+    final_report["json"] = json.dumps(final_report)
+    if not silent:
+        print("\n\n\n----report---\n")
+        for key in final_report:
+            print("* ", key, ":", final_report[key])
+
+    return final_report
+
+
+def get_tags(url):
+    return sniff_url(url, silent=True)

ohmyscrapper/modules/untouch_all.py

@@ -0,0 +1,7 @@
+import ohmyscrapper.models.urls_manager as urls_manager
+
+
+def untouch_all():
+    urls_manager.untouch_all_urls()
+    print("urls have been untouched")
+    return

ohmyscrapper-0.1.1.dist-info/METADATA

@@ -0,0 +1,109 @@
+Metadata-Version: 2.3
+Name: ohmyscrapper
+Version: 0.1.1
+Summary: This project aims to create a text-based scraper containing links to create a final PDF with general information about job openings.
+Author: Cesar Cardoso gh@bouli
+Author-email: Cesar Cardoso gh@bouli <hello@cesarcardoso.cc>
+Requires-Dist: beautifulsoup4>=4.14.3
+Requires-Dist: google-genai>=1.55.0
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: python-dotenv>=1.2.1
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: requests>=2.32.5
+Requires-Dist: rich>=14.2.0
+Requires-Dist: urlextract>=1.9.0
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# OhMyScrapper - v0.1.1
+
+This project aims to create a text-based scraper containing links to create a
+final PDF with general information about job openings.
+
+> This project is using [uv](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer) by default.
+
+## Scope
+
+- Read texts;
+- Extract links;
+- Use meta og:tags to extract information;
+
+## Installation
+
+I recomend to use the [uv](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer), so you can just use the command bellow and everything is installed:
+```shell
+uv sync
+```
+
+## How to use and test (development only)
+
+OhMyScrapper works in 3 stages:
+
+1. It collects and loads urls from a text (by default `input/_chat.txt`) in a database;
+2. It scraps/access the collected urls and read what is relevant. If it finds new urls, they are collected as well;
+3. Export a list of urls in CSV files;
+
+You can do 3 stages with the command:
+```shell
+make start
+```
+> Remember to add your text file in the folder `/input` with the name `_chat.txt`!
+
+You will find the exported files in the folder `/output` like this:
+- `/output/report.csv`
+- `/output/report.csv-preview.html`
+- `/output/urls-simplified.csv`
+- `/output/urls-simplified.csv-preview.html`
+- `/output/urls.csv`
+- `/output/urls.csv-preview.html`
+
+### BUT: if you want to do step by step, here it is:
+
+First we load a text file you would like to look for urls, the idea here is to
+use the whatsapp history, but it works with any txt file.
+
+The default file is `input/_chat.txt`. If you have the default file you just use
+the command `load`:
+```shell
+make load
+```
+or, if you have another file, just use the argument `-file` like this:
+```shell
+uv run main.py load -file=my-text-file.txt
+```
+That will create a database if it doesn't exist and store every url the oh-my-scrapper
+find. After that, let's scrap the urls with the command `scrap-urls`:
+
+```shell
+make scrap-urls
+```
+
+That will scrap only the linkedin urls we are interested in. For now they are:
+- linkedin_post: https://%.linkedin.com/posts/%
+- linkedin_redirect: https://lnkd.in/%
+- linkedin_job: https://%.linkedin.com/jobs/view/%
+- linkedin_feed" https://%.linkedin.com/feed/%
+- linkedin_company: https://%.linkedin.com/company/%
+
+But we can use every other one generically using the argument `--ignore-type`:
+```shell
+uv run main.py scrap-urls --ignore-type
+```
+
+And we can ask to make it recursively adding the argument `--recursive`:
+```shell
+uv run main.py scrap-urls --recursive
+```
+> !!! important: we are not sure about blocks we can have for excess of requests
+
+And we can finally export with the command:
+```shell
+make export
+```
+
+
+That's the basic usage!
+But you can understand more using the help:
+```shell
+uv run main.py --help
+```

ohmyscrapper-0.1.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+ohmyscrapper/__init__.py,sha256=ob4q1EP5nxpOA1uMW2MdyPQpgCGziGDjkJOAKien_i8,4917
+ohmyscrapper/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
+ohmyscrapper/models/urls_manager.py,sha256=nEGjd1SInCz7ccCuFT5Qh5GRoBejfVogYVHKgwIDCbI,10504
+ohmyscrapper/modules/classify_urls.py,sha256=LRYCD2Rmc7vz65uI9IjmFLyB5M-5fmyC5o-ZJZhmqOc,711
+ohmyscrapper/modules/load_txt.py,sha256=VycWtOM4BNFSYGqBLr5_7hAFjO3Fzyeb_xpDvWTWxaw,866
+ohmyscrapper/modules/merge_dbs.py,sha256=0pK3PPUGSbnaDkdpQUGCHemOVaKO37bfHwnsy_EVpWQ,115
+ohmyscrapper/modules/process_with_ai.py,sha256=oXeEth4vdzaQQoi3au8lDZ3uIJc0mQ_CF92w4CFckNk,5143
+ohmyscrapper/modules/scrap_urls.py,sha256=5Uk3dze6e4dRGlmOsyJ_A6Jat_L6vIVURtfll2hWGdw,5872
+ohmyscrapper/modules/seed.py,sha256=KeTSbmTdNTkVCtzk9iQmeuEqB0kG-rTZJb2a1WdROL4,129
+ohmyscrapper/modules/show.py,sha256=5VkDfFXWBik2YHk7xuneJCnndAroUHDAlWGRRNSSkYc,3194
+ohmyscrapper/modules/sniff_url.py,sha256=hsMd-mEZPPOqiyDCc-QW_ThJ1GIAL7vixiX3-fLO9AU,2716
+ohmyscrapper/modules/untouch_all.py,sha256=E1U9e3sOG7suzc8ZTWcYiQQo9mPmLJ0piXdXUjFLEd4,162
+ohmyscrapper-0.1.1.dist-info/WHEEL,sha256=xDCZ-UyfvkGuEHPeI7BcJzYKIZzdqN8A8o1M5Om8IyA,79
+ohmyscrapper-0.1.1.dist-info/entry_points.txt,sha256=BZud6D16XkfjelDa4Z33mji-KJbbZXgq2FoLrzjru5I,52
+ohmyscrapper-0.1.1.dist-info/METADATA,sha256=CBiGxmdZdR37mF1xh8CfRa7PQaII804RzcCGHLsGUlM,3381
+ohmyscrapper-0.1.1.dist-info/RECORD,,

ohmyscrapper-0.1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.9.17
+Root-Is-Purelib: true
+Tag: py3-none-any

ohmyscrapper-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+ohmyscrapper = ohmyscrapper:main
+