ohmyscrapper 0.2.3__py3-none-any.whl → 0.7.0__py3-none-any.whl

ohmyscrapper/__init__.py

@@ -3,7 +3,7 @@ import argparse
 from ohmyscrapper.modules.classify_urls import classify_urls
 from ohmyscrapper.modules.sniff_url import sniff_url
 from ohmyscrapper.modules.load_txt import load_txt
-from ohmyscrapper.modules.seed import seed
+from ohmyscrapper.modules.seed import seed, export_url_types_to_file
 from ohmyscrapper.modules.scrap_urls import scrap_urls
 from ohmyscrapper.modules.show import (
     show_url,
@@ -15,30 +15,40 @@ from ohmyscrapper.modules.show import (
 from ohmyscrapper.modules.untouch_all import untouch_all
 from ohmyscrapper.modules.process_with_ai import process_with_ai, reprocess_ai_history
 from ohmyscrapper.modules.merge_dbs import merge_dbs
+from ohmyscrapper.core.config import update
 
 
 def main():
     parser = argparse.ArgumentParser(prog="ohmyscrapper")
-    parser.add_argument("--version", action="version", version="%(prog)s v0.2.3")
+    parser.add_argument("--version", action="version", version="%(prog)s v0.7.0")
 
+    update()
     subparsers = parser.add_subparsers(dest="command", help="Available commands")
     start_parser = subparsers.add_parser(
-        "start", help="Make the entire process of loading, processing and exporting with the default configuration."
+        "start",
+        help="Make the entire process of 📦 loading, 🐶 scraping and 📜🖋️ exporting with the default configuration.",
     )
 
     start_parser.add_argument(
-        "--ai", default=False, help="Make the entire process of loading, processing, reprocessing with AI and exporting with the default configuration.", action="store_true"
+        "--ai",
+        default=False,
+        help="Make the entire process of loading, processing, reprocessing with AI and exporting with the default configuration.",
+        action="store_true",
     )
 
-    ai_process_parser = subparsers.add_parser(
-        "ai", help="Process with AI."
-    )
+    ai_process_parser = subparsers.add_parser("ai", help="Process with AI.")
     ai_process_parser.add_argument(
         "--history", default=False, help="Reprocess ai history", action="store_true"
     )
 
     seed_parser = subparsers.add_parser(
-        "seed", help="Seed database. Necessary to classify urls."
+        "seed", help="Seed database with `url_types` to classify the `urls`."
+    )
+    seed_parser.add_argument(
+        "--export",
+        default=False,
+        help="Add all `url_types` from the bank to the `/ohmyscrapper/url_types.yaml` file.",
+        action="store_true",
     )
     untouch_parser = subparsers.add_parser(
         "untouch-all", help="Untouch all urls. That resets classification"
@@ -51,12 +61,15 @@ def main():
         "--recursive", default=False, help="Run in recursive mode", action="store_true"
     )
 
-    load_txt_parser = subparsers.add_parser("load", help="Load txt file")
+    load_txt_parser = subparsers.add_parser("load", help="📦 Load txt file")
+    load_txt_parser.add_argument(
+        "-input", default=None, help="File/Folder path or url."
+    )
     load_txt_parser.add_argument(
-        "-file", default="input/_chat.txt", help="File path. Default is input/_chat.txt"
+        "--verbose", default=False, help="Run in verbose mode", action="store_true"
     )
 
-    scrap_urls_parser = subparsers.add_parser("scrap-urls", help="Scrap urls")
+    scrap_urls_parser = subparsers.add_parser("scrap-urls", help="🐶 Scrap urls")
     scrap_urls_parser.add_argument(
         "--recursive", default=False, help="Run in recursive mode", action="store_true"
     )
@@ -69,8 +82,11 @@ def main():
     scrap_urls_parser.add_argument(
         "--only-parents", default=False, help="Only parents urls", action="store_true"
     )
+    scrap_urls_parser.add_argument(
+        "--verbose", default=False, help="Run in verbose mode", action="store_true"
+    )
 
-    sniff_url_parser = subparsers.add_parser("sniff-url", help="Check url")
+    sniff_url_parser = subparsers.add_parser("sniff-url", help="🐕 Sniff/Check url")
     sniff_url_parser.add_argument(
         "url", default="https://cesarcardoso.cc/", help="Url to sniff"
     )
@@ -82,7 +98,7 @@ def main():
     show_urls_parser.add_argument("--limit", default=0, help="Limit of lines to show")
     show_urls_parser.add_argument("-url", default="", help="Url to show")
 
-    export_parser = subparsers.add_parser("export", help="Export urls to csv.")
+    export_parser = subparsers.add_parser("export", help="📊🖋️ Export urls to csv.")
     export_parser.add_argument("--limit", default=0, help="Limit of lines to export")
     export_parser.add_argument(
         "--file",
@@ -96,14 +112,11 @@ def main():
         action="store_true",
     )
 
-    report_parser = subparsers.add_parser("report", help="Export urls report to csv.")
+    report_parser = subparsers.add_parser(
+        "report", help="📜🖋️ Export urls report to csv."
+    )
     merge_parser = subparsers.add_parser("merge_dbs", help="Merge databases.")
 
-    # TODO: What is that?
-    # seed_parser.set_defaults(func=seed)
-    # classify_urls_parser.set_defaults(func=classify_urls)
-    # load_txt_parser.set_defaults(func=load_txt)
-
     args = parser.parse_args()
 
     if args.command == "classify-urls":
@@ -111,11 +124,14 @@ def main():
         return
 
     if args.command == "load":
-        load_txt(args.file)
+        load_txt(file_name=args.input, verbose=args.verbose)
         return
 
     if args.command == "seed":
-        seed()
+        if args.export:
+            export_url_types_to_file()
+        else:
+            seed()
         return
 
     if args.command == "untouch-all":
@@ -132,6 +148,7 @@ def main():
             ignore_valid_prefix=args.ignore_type,
             randomize=args.randomize,
             only_parents=args.only_parents,
+            verbose=args.verbose,
         )
         return
 
@@ -166,7 +183,12 @@ def main():
 
     if args.command == "start":
         load_txt()
-        scrap_urls(recursive=True,ignore_valid_prefix=True,randomize=False,only_parents=False)
+        scrap_urls(
+            recursive=True,
+            ignore_valid_prefix=True,
+            randomize=False,
+            only_parents=False,
+        )
         if args.ai:
             process_with_ai()
         export_urls()

ohmyscrapper/core/config.py

@@ -0,0 +1,107 @@
+import os
+from ohmyscrapper.core import config_files
+
+default_app_dir = "ohmyscrapper"
+
+
+def get_dir(param="ohmyscrapper"):
+    parent_param = "default_dirs"
+
+    if param == default_app_dir:
+        folder = "./" + param
+    else:
+        folder = config_files.get_param(
+            parent_param=parent_param, param=param, default_app_dir=default_app_dir
+        )
+    if not os.path.exists(folder):
+        os.mkdir(folder)
+    return folder
+
+
+def get_files(param):
+    parent_param = "default_files"
+    return config_files.get_param(
+        parent_param=parent_param, param=param, default_app_dir=default_app_dir
+    )
+
+
+def get_db(param="db_file"):
+    if param == "folder":
+        return get_dir(param="db")
+    return config_files.get_param(
+        parent_param="db", param=param, default_app_dir=default_app_dir
+    )
+
+
+def get_ai(param):
+    return config_files.get_param(
+        parent_param="ai", param=param, default_app_dir=default_app_dir
+    )
+
+
+def load_config(force_default=False):
+    config_file_name = "config.yaml"
+    config_params = config_files.create_and_read_config_file(
+        file_name=config_file_name,
+        default_app_dir=default_app_dir,
+        force_default=force_default,
+    )
+
+    if config_params is None or "default_dirs" not in config_params:
+        config_params = load_config(force_default=True)
+
+    return config_params
+
+
+def url_types_file_exists():
+    url_types_file = get_files("url_types")
+    return config_files.config_file_exists(
+        url_types_file, default_app_dir=default_app_dir
+    )
+
+
+def get_url_types():
+    url_types_file = get_files("url_types")
+    return config_files.create_and_read_config_file(
+        url_types_file, default_app_dir=default_app_dir
+    )
+
+
+def get_url_sniffing():
+    file = get_files("url_sniffing")
+    return config_files.create_and_read_config_file(
+        file, default_app_dir=default_app_dir
+    )
+
+
+def append_url_sniffing(data):
+    file = get_files("url_sniffing")
+    _append_config_file(data, file)
+
+
+def append_url_types(url_types):
+    url_types_file = get_files("url_types")
+    _append_config_file(url_types, url_types_file)
+
+
+def overwrite_config_file(data, file_name):
+    config_files.overwrite_config_file(data, file_name, default_app_dir=default_app_dir)
+
+
+def _append_config_file(data, file_name):
+    config_files.append_config_file(data, file_name, default_app_dir=default_app_dir)
+
+
+def update():
+    legacy_folder = "./customize"
+    new_folder = "./ohmyscrapper"
+    if os.path.exists(legacy_folder) and not os.path.exists(new_folder):
+        yes_no = input(
+            "We detected a legacy folder system for your OhMyScrapper, would you like to update? \n"
+            "If you don't update, a new version will be used and your legacy folder will be ignored. \n"
+            "[Y] for yes or  any other thing to ignore: "
+        )
+        if yes_no == "Y":
+            os.rename(legacy_folder, new_folder)
+        print(" You are up-to-date! =)")
+        print("")

ohmyscrapper/core/config_files.py

@@ -0,0 +1,73 @@
+import os
+import yaml
+
+
+def create_and_read_config_file(file_name, default_app_dir, force_default=False):
+    config_file = config_file_path(file_name, default_app_dir)
+    if force_default or not os.path.exists(config_file):
+        config_params = _get_default_file(default_file=file_name)
+        overwrite_config_file(
+            data=config_params, file_name=file_name, default_app_dir=default_app_dir
+        )
+    else:
+        with open(config_file, "r") as f:
+            config_params = yaml.safe_load(f.read())
+    if config_params is None:
+        config_params = create_and_read_config_file(
+            file_name=file_name, default_app_dir=default_app_dir, force_default=True
+        )
+    return config_params
+
+
+def overwrite_config_file(data, file_name, default_app_dir):
+    config_file = config_file_path(file_name, default_app_dir)
+    with open(config_file, "+w") as f:
+        f.write(yaml.safe_dump(data))
+
+
+def append_config_file(data, file_name, default_app_dir):
+    config_file = config_file_path(file_name, default_app_dir)
+    # append
+    with open(config_file, "+a") as f:
+        yaml.dump(data, f, allow_unicode=True)
+    # read
+    with open(config_file, "r") as f:
+        data = yaml.safe_load(f.read())
+    # overwrite preventing repetition
+    with open(config_file, "w") as f:
+        yaml.dump(data, f, allow_unicode=True)
+
+
+def get_param(parent_param, param, default_app_dir):
+    default_dirs = create_and_read_config_file(
+        file_name="config.yaml", default_app_dir=default_app_dir
+    )[parent_param]
+
+    if param in default_dirs:
+        return default_dirs[param]
+    else:
+        raise Exception(f"{param} do not exist in your params {parent_param}.")
+
+
+def config_file_exists(file_name, default_app_dir):
+    return os.path.exists(config_file_path(file_name, default_app_dir))
+
+
+def config_file_path(file_name, default_app_dir):
+    _ensure_default_app_dir(default_app_dir)
+    config_file = os.path.join(default_app_dir, file_name)
+    return config_file
+
+
+def _ensure_default_app_dir(default_app_dir):
+    if not os.path.exists(default_app_dir):
+        os.mkdir(default_app_dir)
+
+
+def _get_default_file(default_file):
+    default_files_dir = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "default_files"
+    )
+    default_file = os.path.join(default_files_dir, default_file)
+    with open(default_file, "r") as f:
+        return yaml.safe_load(f.read())

ohmyscrapper/core/default_files/config.yaml

@@ -0,0 +1,16 @@
+db:
+  db_file: local.db
+
+default_dirs:
+  db: ./db
+  input: ./input
+  output: ./output
+  prompts: ./prompts
+  templates: ./templates
+
+default_files:
+  url_types: url_types.yaml
+  url_sniffing: url_sniffing.yaml
+
+ai:
+  default_prompt_file: prompt.md

ohmyscrapper/core/default_files/url_sniffing.yaml

@@ -0,0 +1,25 @@
+linkedin_feed:
+  metatags:
+    og:url: url_destiny
+
+linkedin_job:
+  bodytags:
+    h1: title
+  metatags:
+    og:title: title
+    og:description: description
+    description: description
+
+linkedin_post:
+  bodytags:
+    h1: title
+  metatags:
+    og:title: title
+    og:description: description
+    description: description
+
+linkedin_redirect:
+  metatags:
+    og:url: url_destiny
+  atags:
+    first-tag-as-url_destiny: 5

ohmyscrapper/core/default_files/url_types.yaml

@@ -0,0 +1,5 @@
+linkedin_company: https://%.linkedin.com/company/%
+linkedin_feed: https://%.linkedin.com/feed/%
+linkedin_job: https://%.linkedin.com/jobs/view/%
+linkedin_post: https://%.linkedin.com/posts/%
+linkedin_redirect: https://lnkd.in/%