ohmyscrapper 0.2.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

ohmyscrapper/__init__.py

@@ -3,7 +3,7 @@ import argparse
 from ohmyscrapper.modules.classify_urls import classify_urls
 from ohmyscrapper.modules.sniff_url import sniff_url
 from ohmyscrapper.modules.load_txt import load_txt
-from ohmyscrapper.modules.seed import seed
+from ohmyscrapper.modules.seed import seed, export_url_types_to_file
 from ohmyscrapper.modules.scrap_urls import scrap_urls
 from ohmyscrapper.modules.show import (
     show_url,
@@ -15,23 +15,40 @@ from ohmyscrapper.modules.show import (
 from ohmyscrapper.modules.untouch_all import untouch_all
 from ohmyscrapper.modules.process_with_ai import process_with_ai, reprocess_ai_history
 from ohmyscrapper.modules.merge_dbs import merge_dbs
+from ohmyscrapper.core.config import update
 
 
 def main():
     parser = argparse.ArgumentParser(prog="ohmyscrapper")
-    parser.add_argument("--version", action="version", version="%(prog)s v0.2.1")
+    parser.add_argument("--version", action="version", version="%(prog)s v0.6.1")
 
+    update()
     subparsers = parser.add_subparsers(dest="command", help="Available commands")
+    start_parser = subparsers.add_parser(
+        "start",
+        help="Make the entire process of 📦 loading, 🐶 scraping and 📜🖋️ exporting with the default configuration.",
+    )
 
-    ai_process_parser = subparsers.add_parser(
-        "process-with-ai", help="Process with AI."
+    start_parser.add_argument(
+        "--ai",
+        default=False,
+        help="Make the entire process of loading, processing, reprocessing with AI and exporting with the default configuration.",
+        action="store_true",
     )
+
+    ai_process_parser = subparsers.add_parser("ai", help="Process with AI.")
     ai_process_parser.add_argument(
         "--history", default=False, help="Reprocess ai history", action="store_true"
     )
 
     seed_parser = subparsers.add_parser(
-        "seed", help="Seed database. Necessary to classify urls."
+        "seed", help="Seed database with `url_types` to classify the `urls`."
+    )
+    seed_parser.add_argument(
+        "--export",
+        default=False,
+        help="Add all `url_types` from the bank to the `/ohmyscrapper/url_types.yaml` file.",
+        action="store_true",
     )
     untouch_parser = subparsers.add_parser(
         "untouch-all", help="Untouch all urls. That resets classification"
@@ -44,12 +61,15 @@ def main():
         "--recursive", default=False, help="Run in recursive mode", action="store_true"
     )
 
-    load_txt_parser = subparsers.add_parser("load", help="Load txt file")
+    load_txt_parser = subparsers.add_parser("load", help="📦 Load txt file")
     load_txt_parser.add_argument(
-        "-file", default="input/_chat.txt", help="File path. Default is input/_chat.txt"
+        "-input", default=None, help="File/Folder path or url."
+    )
+    load_txt_parser.add_argument(
+        "--verbose", default=False, help="Run in verbose mode", action="store_true"
     )
 
-    scrap_urls_parser = subparsers.add_parser("scrap-urls", help="Scrap urls")
+    scrap_urls_parser = subparsers.add_parser("scrap-urls", help="🐶 Scrap urls")
     scrap_urls_parser.add_argument(
         "--recursive", default=False, help="Run in recursive mode", action="store_true"
     )
@@ -62,8 +82,11 @@ def main():
     scrap_urls_parser.add_argument(
         "--only-parents", default=False, help="Only parents urls", action="store_true"
     )
+    scrap_urls_parser.add_argument(
+        "--verbose", default=False, help="Run in verbose mode", action="store_true"
+    )
 
-    sniff_url_parser = subparsers.add_parser("sniff-url", help="Check url")
+    sniff_url_parser = subparsers.add_parser("sniff-url", help="🐕 Sniff/Check url")
     sniff_url_parser.add_argument(
         "url", default="https://cesarcardoso.cc/", help="Url to sniff"
     )
@@ -75,7 +98,7 @@ def main():
     show_urls_parser.add_argument("--limit", default=0, help="Limit of lines to show")
     show_urls_parser.add_argument("-url", default="", help="Url to show")
 
-    export_parser = subparsers.add_parser("export", help="Export urls to csv.")
+    export_parser = subparsers.add_parser("export", help="📊🖋️ Export urls to csv.")
     export_parser.add_argument("--limit", default=0, help="Limit of lines to export")
     export_parser.add_argument(
         "--file",
@@ -89,14 +112,11 @@ def main():
         action="store_true",
     )
 
-    report_parser = subparsers.add_parser("report", help="Export urls report to csv.")
+    report_parser = subparsers.add_parser(
+        "report", help="📜🖋️ Export urls report to csv."
+    )
     merge_parser = subparsers.add_parser("merge_dbs", help="Merge databases.")
 
-    # TODO: What is that?
-    # seed_parser.set_defaults(func=seed)
-    # classify_urls_parser.set_defaults(func=classify_urls)
-    # load_txt_parser.set_defaults(func=load_txt)
-
     args = parser.parse_args()
 
     if args.command == "classify-urls":
@@ -104,11 +124,14 @@ def main():
         return
 
     if args.command == "load":
-        load_txt(args.file)
+        load_txt(file_name=args.input, verbose=args.verbose)
         return
 
     if args.command == "seed":
-        seed()
+        if args.export:
+            export_url_types_to_file()
+        else:
+            seed()
         return
 
     if args.command == "untouch-all":
@@ -125,6 +148,7 @@ def main():
             ignore_valid_prefix=args.ignore_type,
             randomize=args.randomize,
             only_parents=args.only_parents,
+            verbose=args.verbose,
         )
         return
 
@@ -157,6 +181,21 @@ def main():
         merge_dbs()
         return
 
+    if args.command == "start":
+        load_txt()
+        scrap_urls(
+            recursive=True,
+            ignore_valid_prefix=True,
+            randomize=False,
+            only_parents=False,
+        )
+        if args.ai:
+            process_with_ai()
+        export_urls()
+        export_urls(csv_file="output/urls-simplified.csv", simplify=True)
+        export_report()
+        return
+
 
 if __name__ == "__main__":
     main()

ohmyscrapper/core/config.py

@@ -0,0 +1,95 @@
+import os
+from ohmyscrapper.core import config_files
+
+default_app_dir = "ohmyscrapper"
+
+
+def get_dir(param="ohmyscrapper"):
+    parent_param = "default_dirs"
+
+    if param == default_app_dir:
+        folder = "./" + param
+    else:
+        folder = config_files.get_param(
+            parent_param=parent_param, param=param, default_app_dir=default_app_dir
+        )
+    if not os.path.exists(folder):
+        os.mkdir(folder)
+    return folder
+
+
+def get_files(param):
+    parent_param = "default_files"
+    return config_files.get_param(
+        parent_param=parent_param, param=param, default_app_dir=default_app_dir
+    )
+
+
+def get_db(param="db_file"):
+    if param == "folder":
+        return get_dir(param="db")
+    return config_files.get_param(
+        parent_param="db", param=param, default_app_dir=default_app_dir
+    )
+
+
+def get_ai(param):
+    return config_files.get_param(
+        parent_param="ai", param=param, default_app_dir=default_app_dir
+    )
+
+
+def load_config(force_default=False):
+    config_file_name = "config.yaml"
+    config_params = config_files.create_and_read_config_file(
+        file_name=config_file_name,
+        default_app_dir=default_app_dir,
+        force_default=force_default,
+    )
+
+    if config_params is None or "default_dirs" not in config_params:
+        config_params = load_config(force_default=True)
+
+    return config_params
+
+
+def url_types_file_exists():
+    url_types_file = get_files("url_types")
+    return config_files.config_file_exists(
+        url_types_file, default_app_dir=default_app_dir
+    )
+
+
+def get_url_types():
+    url_types_file = get_files("url_types")
+    return config_files.create_and_read_config_file(
+        url_types_file, default_app_dir=default_app_dir
+    )
+
+
+def append_url_types(url_types):
+    url_types_file = get_files("url_types")
+    _append_config_file(url_types, url_types_file)
+
+
+def overwrite_config_file(data, file_name):
+    config_files.overwrite_config_file(data, file_name, default_app_dir=default_app_dir)
+
+
+def _append_config_file(data, file_name):
+    config_files.append_config_file(data, file_name, default_app_dir=default_app_dir)
+
+
+def update():
+    legacy_folder = "./customize"
+    new_folder = "./ohmyscrapper"
+    if os.path.exists(legacy_folder) and not os.path.exists(new_folder):
+        yes_no = input(
+            "We detected a legacy folder system for your OhMyScrapper, would you like to update? \n"
+            "If you don't update, a new version will be used and your legacy folder will be ignored. \n"
+            "[Y] for yes or  any other thing to ignore: "
+        )
+        if yes_no == "Y":
+            os.rename(legacy_folder, new_folder)
+        print(" You are up-to-date! =)")
+        print("")

ohmyscrapper/core/config_files.py

@@ -0,0 +1,73 @@
+import os
+import yaml
+
+
+def create_and_read_config_file(file_name, default_app_dir, force_default=False):
+    config_file = config_file_path(file_name, default_app_dir)
+    if force_default or not os.path.exists(config_file):
+        config_params = _get_default_file(default_file=file_name)
+        overwrite_config_file(
+            data=config_params, file_name=file_name, default_app_dir=default_app_dir
+        )
+    else:
+        with open(config_file, "r") as f:
+            config_params = yaml.safe_load(f.read())
+    if config_params is None:
+        config_params = create_and_read_config_file(
+            file_name=file_name, default_app_dir=default_app_dir, force_default=True
+        )
+    return config_params
+
+
+def overwrite_config_file(data, file_name, default_app_dir):
+    config_file = config_file_path(file_name, default_app_dir)
+    with open(config_file, "+w") as f:
+        f.write(yaml.safe_dump(data))
+
+
+def append_config_file(data, file_name, default_app_dir):
+    config_file = config_file_path(file_name, default_app_dir)
+    # append
+    with open(config_file, "+a") as f:
+        yaml.dump(data, f, allow_unicode=True)
+    # read
+    with open(config_file, "r") as f:
+        data = yaml.safe_load(f.read())
+    # overwrite preventing repetition
+    with open(config_file, "w") as f:
+        yaml.dump(data, f, allow_unicode=True)
+
+
+def get_param(parent_param, param, default_app_dir):
+    default_dirs = create_and_read_config_file(
+        file_name="config.yaml", default_app_dir=default_app_dir
+    )[parent_param]
+
+    if param in default_dirs:
+        return default_dirs[param]
+    else:
+        raise Exception(f"{param} do not exist in your params {parent_param}.")
+
+
+def config_file_exists(file_name, default_app_dir):
+    return os.path.exists(config_file_path(file_name, default_app_dir))
+
+
+def config_file_path(file_name, default_app_dir):
+    _ensure_default_app_dir(default_app_dir)
+    config_file = os.path.join(default_app_dir, file_name)
+    return config_file
+
+
+def _ensure_default_app_dir(default_app_dir):
+    if not os.path.exists(default_app_dir):
+        os.mkdir(default_app_dir)
+
+
+def _get_default_file(default_file):
+    default_files_dir = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "default_files"
+    )
+    default_file = os.path.join(default_files_dir, default_file)
+    with open(default_file, "r") as f:
+        return yaml.safe_load(f.read())

ohmyscrapper/core/default_files/config.yaml

@@ -0,0 +1,15 @@
+db:
+  db_file: local.db
+
+default_dirs:
+  db: ./db
+  input: ./input
+  output: ./output
+  prompts: ./prompts
+  templates: ./templates
+
+default_files:
+  url_types: url_types.yaml
+
+ai:
+  default_prompt_file: prompt.md

ohmyscrapper/core/default_files/url_types.yaml

@@ -0,0 +1,5 @@
+linkedin_company: https://%.linkedin.com/company/%
+linkedin_feed: https://%.linkedin.com/feed/%
+linkedin_job: https://%.linkedin.com/jobs/view/%
+linkedin_post: https://%.linkedin.com/posts/%
+linkedin_redirect: https://lnkd.in/%

ohmyscrapper/models/urls_manager.py

@@ -4,27 +4,37 @@ import time
 import glob
 import pandas as pd
 from urllib.parse import urlparse, urlunparse
+from ohmyscrapper.core import config
 
 
 def get_db_dir():
-    if not os.path.exists("db"):
-        os.mkdir("db")
-    return "db"
+    db_folder = config.get_dir("db")
+    if not os.path.exists(db_folder):
+        os.mkdir(db_folder)
+    return db_folder
 
 
 def get_db_path():
-    return get_db_dir() + "/local.db"
+    db_file = config.get_db()
+    return os.path.join(get_db_dir(), db_file)
 
 
 def get_db_connection():
+    if not os.path.exists(get_db_path()):
+        create_tables(sqlite3.connect(get_db_path()))
     return sqlite3.connect(get_db_path())
 
 
-# TODO: check if it makes sense
-conn = get_db_connection()
+def use_connection(func):
+    def provide_connection(*args, **kwargs):
+        global conn
+        with get_db_connection() as conn:
+            return func(*args, **kwargs)
 
+    return provide_connection
 
-def create_tables():
+
+def create_tables(conn):
 
     c = conn.cursor()
     c.execute(
@@ -38,27 +48,17 @@ def create_tables():
         "CREATE TABLE IF NOT EXISTS urls_valid_prefix (id INTEGER PRIMARY KEY, url_prefix TEXT UNIQUE, url_type TEXT)"
     )
 
-    return pd.read_sql_query("SELECT * FROM urls LIMIT 100", conn)
-
 
-# TODO: not sure this should be something. depends on the project
-def seeds():
-    create_tables()
+def seeds(seeds={}):
 
-    add_urls_valid_prefix("https://%.linkedin.com/posts/%", "linkedin_post")
-    add_urls_valid_prefix("https://lnkd.in/%", "linkedin_redirect")
-    add_urls_valid_prefix("https://%.linkedin.com/jobs/view/%", "linkedin_job")
-    add_urls_valid_prefix("https://%.linkedin.com/feed/%", "linkedin_feed")
-    add_urls_valid_prefix("https://%.linkedin.com/company/%", "linkedin_company")
-
-    # add_urls_valid_prefix("%.pdf", "pdf")
-    # add_url('https://imazon.org.br/categorias/artigos-cientificos/')
+    for url_type, url_prefix in seeds.items():
+        add_urls_valid_prefix(url_prefix, url_type)
 
     return True
 
 
+@use_connection
 def add_urls_valid_prefix(url_prefix, url_type):
-    conn = get_db_connection()
 
     df = pd.read_sql_query(
         f"SELECT * FROM urls_valid_prefix WHERE url_prefix = '{url_prefix}'", conn
@@ -72,6 +72,7 @@ def add_urls_valid_prefix(url_prefix, url_type):
         conn.commit()
 
 
+@use_connection
 def get_urls_valid_prefix_by_type(url_type):
     df = pd.read_sql_query(
         f"SELECT * FROM urls_valid_prefix WHERE url_type = '{url_type}'", conn
@@ -79,12 +80,14 @@ def get_urls_valid_prefix_by_type(url_type):
     return df
 
 
+@use_connection
 def get_urls_valid_prefix_by_id(id):
     df = pd.read_sql_query(f"SELECT * FROM urls_valid_prefix WHERE id = '{id}'", conn)
     return df
 
 
 # TODO: pagination required
+@use_connection
 def get_urls_valid_prefix(limit=0):
     if limit > 0:
         df = pd.read_sql_query(f"SELECT * FROM urls_valid_prefix LIMIT {limit}", conn)
@@ -94,6 +97,7 @@ def get_urls_valid_prefix(limit=0):
 
 
 # TODO: pagination required
+@use_connection
 def get_urls(limit=0):
     if limit > 0:
         df = pd.read_sql_query(
@@ -104,6 +108,7 @@ def get_urls(limit=0):
     return df
 
 
+@use_connection
 def get_urls_report():
     sql = """
     WITH parent_url AS (
@@ -138,6 +143,7 @@ def get_urls_report():
     return df
 
 
+@use_connection
 def get_url_by_url(url):
     url = clean_url(url)
     df = pd.read_sql_query(f"SELECT * FROM urls WHERE url = '{url}'", conn)
@@ -145,12 +151,14 @@ def get_url_by_url(url):
     return df
 
 
+@use_connection
 def get_url_by_id(id):
     df = pd.read_sql_query(f"SELECT * FROM urls WHERE id = '{id}'", conn)
 
     return df
 
 
+@use_connection
 def get_urls_by_url_type(url_type):
     df = pd.read_sql_query(
         f"SELECT * FROM urls WHERE history = 0 AND url_type = '{url_type}'", conn
@@ -158,6 +166,7 @@ def get_urls_by_url_type(url_type):
     return df
 
 
+@use_connection
 def get_urls_by_url_type_for_ai_process(url_type="linkedin_post", limit=10):
     df = pd.read_sql_query(
         f"SELECT * FROM urls WHERE history = 0 AND url_type = '{url_type}' AND ai_processed = 0 LIMIT {limit}",
@@ -166,6 +175,7 @@ def get_urls_by_url_type_for_ai_process(url_type="linkedin_post", limit=10):
     return df
 
 
+@use_connection
 def get_url_like_unclassified(like_condition):
     df = pd.read_sql_query(
         f"SELECT * FROM urls WHERE history = 0 AND url LIKE '{like_condition}' AND url_type IS NULL",
@@ -174,6 +184,7 @@ def get_url_like_unclassified(like_condition):
     return df
 
 
+@use_connection
 def add_url(url, h1=None, parent_url=None):
     url = clean_url(url)
     c = conn.cursor()
@@ -196,6 +207,7 @@ def add_url(url, h1=None, parent_url=None):
     return get_url_by_url(url)
 
 
+@use_connection
 def add_ai_log(instructions, response, model, prompt_file, prompt_name):
     c = conn.cursor()
 
@@ -205,10 +217,14 @@ def add_ai_log(instructions, response, model, prompt_file, prompt_name):
     )
     conn.commit()
 
+
+@use_connection
 def get_ai_log():
     df = pd.read_sql_query(f"SELECT * FROM ai_log", conn)
     return df
 
+
+@use_connection
 def set_url_destiny(url, destiny):
     url = clean_url(url)
     destiny = clean_url(destiny)
@@ -222,6 +238,7 @@ def set_url_destiny(url, destiny):
     conn.commit()
 
 
+@use_connection
 def set_url_h1(url, value):
     value = str(value).strip()
     url = clean_url(url)
@@ -230,6 +247,7 @@ def set_url_h1(url, value):
     conn.commit()
 
 
+@use_connection
 def set_url_h1_by_id(id, value):
     value = str(value).strip()
 
@@ -238,29 +256,44 @@ def set_url_h1_by_id(id, value):
     conn.commit()
 
 
+@use_connection
 def set_url_ai_processed_by_id(id, json_str):
     value = 1
     value = str(value).strip()
     c = conn.cursor()
-    c.execute("UPDATE urls SET ai_processed = ? , json_ai = ? WHERE id = ?", (value, json_str, id))
+    c.execute(
+        "UPDATE urls SET ai_processed = ? , json_ai = ? WHERE id = ?",
+        (value, json_str, id),
+    )
     conn.commit()
 
+
+@use_connection
 def set_url_empty_ai_processed_by_id(id, json_str="empty result"):
     value = 1
     value = str(value).strip()
     c = conn.cursor()
-    c.execute("UPDATE urls SET ai_processed = ? , json_ai = ? WHERE ai_processed = 0 AND id = ?", (value, json_str, id))
+    c.execute(
+        "UPDATE urls SET ai_processed = ? , json_ai = ? WHERE ai_processed = 0 AND id = ?",
+        (value, json_str, id),
+    )
     conn.commit()
 
+
+@use_connection
 def set_url_ai_processed_by_url(url, json_str):
     value = 1
     value = str(value).strip()
     url = clean_url(url)
     c = conn.cursor()
-    c.execute("UPDATE urls SET ai_processed = ?, json_ai = ? WHERE url = ?", (value, json_str, url))
+    c.execute(
+        "UPDATE urls SET ai_processed = ?, json_ai = ? WHERE url = ?",
+        (value, json_str, url),
+    )
     conn.commit()
 
 
+@use_connection
 def set_url_description(url, value):
     url = clean_url(url)
     c = conn.cursor()
@@ -268,6 +301,7 @@ def set_url_description(url, value):
     conn.commit()
 
 
+@use_connection
 def set_url_description_links(url, value):
     url = clean_url(url)
     c = conn.cursor()
@@ -275,6 +309,7 @@ def set_url_description_links(url, value):
     conn.commit()
 
 
+@use_connection
 def set_url_json(url, value):
     url = clean_url(url)
     c = conn.cursor()
@@ -282,6 +317,7 @@ def set_url_json(url, value):
     conn.commit()
 
 
+@use_connection
 def set_url_error(url, value):
     url = clean_url(url)
     c = conn.cursor()
@@ -289,6 +325,7 @@ def set_url_error(url, value):
     conn.commit()
 
 
+@use_connection
 def set_url_type_by_id(url_id, url_type):
     c = conn.cursor()
     c.execute(f"UPDATE urls SET url_type = '{url_type}' WHERE id = {url_id}")
@@ -312,6 +349,7 @@ def clean_url(url):
     return url
 
 
+@use_connection
 def get_untouched_urls(
     limit=10, randomize=True, ignore_valid_prefix=False, only_parents=True
 ):
@@ -331,6 +369,7 @@ def get_untouched_urls(
     return df
 
 
+@use_connection
 def touch_url(url):
     url = clean_url(url)
     c = conn.cursor()
@@ -338,6 +377,7 @@ def touch_url(url):
     conn.commit()
 
 
+@use_connection
 def untouch_url(url):
     url = clean_url(url)
     c = conn.cursor()
@@ -345,12 +385,14 @@ def untouch_url(url):
     conn.commit()
 
 
+@use_connection
 def untouch_all_urls():
     c = conn.cursor()
     c.execute("UPDATE urls SET last_touch = NULL WHERE history = 0")
     conn.commit()
 
 
+@use_connection
 def set_all_urls_as_history():
     c = conn.cursor()
     c.execute("UPDATE urls SET history = 1")
@@ -382,9 +424,9 @@ def merge_dbs() -> None:
                 row["description"],
                 row["json"],
             )
-        # ßmerge_url(df)
 
 
+@use_connection
 def merge_url(url, h1, last_touch, created_at, description, json):
     url = clean_url(url)
     c = conn.cursor()

ohmyscrapper/modules/classify_urls.py

@@ -9,15 +9,19 @@ def classify_urls(recursive=False):
 
     keep_alive = True
     while keep_alive:
-        print("waking up!")
+        print("#️⃣  URL Classifier woke up to classify urls!")
         for index, row_prefix in df.iterrows():
-            df_urls = urls_manager.get_url_like_unclassified(like_condition=row_prefix["url_prefix"])
+            df_urls = urls_manager.get_url_like_unclassified(
+                like_condition=row_prefix["url_prefix"]
+            )
             for index, row_urls in df_urls.iterrows():
-                urls_manager.set_url_type_by_id(url_id =row_urls["id"], url_type=row_prefix["url_type"])
+                urls_manager.set_url_type_by_id(
+                    url_id=row_urls["id"], url_type=row_prefix["url_type"]
+                )
 
         if not recursive:
-            print("ending...")
+            print("#️⃣  URL Classifier said: I'm done! See you soon...")
             keep_alive = False
         else:
-            print("sleeping...")
+            print("#️⃣  URL Classifier is taking a nap...")
             time.sleep(10)