ohmyscrapper 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

ohmyscrapper/__init__.py

@@ -20,7 +20,7 @@ from ohmyscrapper.core.config import update
 
 def main():
     parser = argparse.ArgumentParser(prog="ohmyscrapper")
-    parser.add_argument("--version", action="version", version="%(prog)s v0.6.1")
+    parser.add_argument("--version", action="version", version="%(prog)s v0.7.0")
 
     update()
     subparsers = parser.add_subparsers(dest="command", help="Available commands")

ohmyscrapper/core/config.py

@@ -67,6 +67,18 @@ def get_url_types():
     )
 
 
+def get_url_sniffing():
+    file = get_files("url_sniffing")
+    return config_files.create_and_read_config_file(
+        file, default_app_dir=default_app_dir
+    )
+
+
+def append_url_sniffing(data):
+    file = get_files("url_sniffing")
+    _append_config_file(data, file)
+
+
 def append_url_types(url_types):
     url_types_file = get_files("url_types")
     _append_config_file(url_types, url_types_file)

ohmyscrapper/core/default_files/config.yaml

@@ -10,6 +10,7 @@ default_dirs:
 
 default_files:
   url_types: url_types.yaml
+  url_sniffing: url_sniffing.yaml
 
 ai:
   default_prompt_file: prompt.md

ohmyscrapper/core/default_files/url_sniffing.yaml

@@ -0,0 +1,25 @@
+linkedin_feed:
+  metatags:
+    og:url: url_destiny
+
+linkedin_job:
+  bodytags:
+    h1: title
+  metatags:
+    og:title: title
+    og:description: description
+    description: description
+
+linkedin_post:
+  bodytags:
+    h1: title
+  metatags:
+    og:title: title
+    og:description: description
+    description: description
+
+linkedin_redirect:
+  metatags:
+    og:url: url_destiny
+  atags:
+    first-tag-as-url_destiny: 5

ohmyscrapper/models/urls_manager.py

@@ -29,7 +29,11 @@ def use_connection(func):
     def provide_connection(*args, **kwargs):
         global conn
         with get_db_connection() as conn:
-            return func(*args, **kwargs)
+            try:
+                return func(*args, **kwargs)
+            except:
+                update_db()
+                return func(*args, **kwargs)
 
     return provide_connection
 
@@ -38,7 +42,7 @@ def create_tables(conn):
 
     c = conn.cursor()
     c.execute(
-        "CREATE TABLE IF NOT EXISTS urls (id INTEGER PRIMARY KEY, url_type STRING, parent_url TEXT, url TEXT UNIQUE, url_destiny TEXT, h1 TEXT, error TEXT, description TEXT, description_links INTEGER DEFAULT 0, json TEXT, json_ai TEXT, ai_processed INTEGER DEFAULT 0, history INTEGER DEFAULT 0, last_touch DATETIME, created_at DATETIME)"
+        "CREATE TABLE IF NOT EXISTS urls (id INTEGER PRIMARY KEY, url_type STRING, parent_url TEXT, url TEXT UNIQUE, url_destiny TEXT, title TEXT, error TEXT, description TEXT, description_links INTEGER DEFAULT 0, json TEXT, json_ai TEXT, ai_processed INTEGER DEFAULT 0, history INTEGER DEFAULT 0, last_touch DATETIME, created_at DATETIME)"
     )
     c.execute(
         "CREATE TABLE IF NOT EXISTS ai_log (id INTEGER PRIMARY KEY, instructions STRING, response STRING, model STRING, prompt_file STRING, prompt_name STRING, created_at DATETIME)"
@@ -49,6 +53,14 @@ def create_tables(conn):
     )
 
 
+def update_db():
+    try:
+        c = conn.cursor()
+        c.execute("ALTER TABLE urls RENAME COLUMN h1 TO title")
+    except:
+        pass
+
+
 def seeds(seeds={}):
 
     for url_type, url_prefix in seeds.items():
@@ -118,7 +130,7 @@ def get_urls_report():
         SELECT
             u.id,
             u.url,
-            u.h1
+            u.title
             FROM urls u
                 INNER JOIN parent_url p
                     ON u.url = p.parent_url
@@ -127,9 +139,9 @@ def get_urls_report():
         u.id,
         u.url_type,
         u.url,
-        COALESCE(u.h1, p.h1) as h1,
+        COALESCE(u.title, p.title) as title,
         p.url as parent_url,
-        p.h1 as parent_h1
+        p.title as parent_title
         FROM urls u
         LEFT JOIN parents p
             ON u.parent_url = p.url
@@ -185,12 +197,12 @@ def get_url_like_unclassified(like_condition):
 
 
 @use_connection
-def add_url(url, h1=None, parent_url=None):
+def add_url(url, title=None, parent_url=None):
     url = clean_url(url)
     c = conn.cursor()
 
-    if h1 is not None:
-        h1 = h1.strip()
+    if title is not None:
+        title = title.strip()
 
     if parent_url is None:
         parent_url = None
@@ -199,8 +211,8 @@ def add_url(url, h1=None, parent_url=None):
 
     if len(get_url_by_url(url)) == 0:
         c.execute(
-            "INSERT INTO urls (url, h1, parent_url, created_at, ai_processed, description_links, history) VALUES (?, ?, ?, ?, 0, 0, 0)",
-            (url, h1, parent_url, int(time.time())),
+            "INSERT INTO urls (url, title, parent_url, created_at, ai_processed, description_links, history) VALUES (?, ?, ?, ?, 0, 0, 0)",
+            (url, title, parent_url, int(time.time())),
         )
         conn.commit()
 
@@ -239,20 +251,20 @@ def set_url_destiny(url, destiny):
 
 
 @use_connection
-def set_url_h1(url, value):
+def set_url_title(url, value):
     value = str(value).strip()
     url = clean_url(url)
     c = conn.cursor()
-    c.execute("UPDATE urls SET h1 = ? WHERE url = ?", (value, url))
+    c.execute("UPDATE urls SET title = ? WHERE url = ?", (value, url))
     conn.commit()
 
 
 @use_connection
-def set_url_h1_by_id(id, value):
+def set_url_title_by_id(id, value):
     value = str(value).strip()
 
     c = conn.cursor()
-    c.execute("UPDATE urls SET h1 = ? WHERE id = ?", (value, id))
+    c.execute("UPDATE urls SET title = ? WHERE id = ?", (value, id))
     conn.commit()
 
 
@@ -427,16 +439,16 @@ def merge_dbs() -> None:
 
 
 @use_connection
-def merge_url(url, h1, last_touch, created_at, description, json):
+def merge_url(url, title, last_touch, created_at, description, json):
     url = clean_url(url)
     c = conn.cursor()
 
-    if h1 is not None:
-        h1 = h1.strip()
+    if title is not None:
+        title = title.strip()
 
     if len(get_url_by_url(url)) == 0:
         c.execute(
-            "INSERT INTO urls (url, h1, last_touch , created_at, history, ai_processed, description_links, description, json) VALUES (?, ?, ?, ?, 1, 0, 0, ? , ?)",
-            (url, h1, last_touch, created_at, description, json),
+            "INSERT INTO urls (url, title, last_touch , created_at, history, ai_processed, description_links, description, json) VALUES (?, ?, ?, ?, 1, 0, 0, ? , ?)",
+            (url, title, last_touch, created_at, description, json),
         )
         conn.commit()

ohmyscrapper/modules/classify_urls.py

@@ -1,11 +1,15 @@
 import ohmyscrapper.models.urls_manager as urls_manager
+from ohmyscrapper.modules import seed
 import pandas as pd
 import time
 
 
 def classify_urls(recursive=False):
-    urls_manager.seeds()
     df = urls_manager.get_urls_valid_prefix()
+    if len(df) == 0:
+        seed.seed()
+        classify_urls(recursive=recursive)
+        return
 
     keep_alive = True
     while keep_alive:

ohmyscrapper/modules/process_with_ai.py

@@ -28,13 +28,13 @@ def process_ai_response(response):
             url_parent = urls_manager.get_url_by_id(url_child_xml["id"])
             if len(url_parent) > 0:
                 url_parent = url_parent.iloc[0]
-            h1 = url_child_xml.copy()
-            del h1["id"]
-            del h1["url"]
-            h1 = " - ".join(h1.values())
+            title = url_child_xml.copy()
+            del title["id"]
+            del title["url"]
+            title = " - ".join(title.values())
             if url_parent["description_links"] > 1 and url_child_xml["id"] != "":
-                print("-- child updated -- \n", url_child_xml["url"], ":", h1)
-                urls_manager.set_url_h1(url_child_xml["url"], h1)
+                print("-- child updated -- \n", url_child_xml["url"], ":", title)
+                urls_manager.set_url_title(url_child_xml["url"], title)
                 urls_manager.set_url_ai_processed_by_url(
                     url_child_xml["url"], str(json.dumps(url_child_xml))
                 )
@@ -43,8 +43,8 @@ def process_ai_response(response):
                         url_parent["url"], "children-update"
                     )
             else:
-                print("-- parent updated -- \n", url_parent["url"], ":", h1)
-                urls_manager.set_url_h1(url_parent["url"], h1)
+                print("-- parent updated -- \n", url_parent["url"], ":", title)
+                urls_manager.set_url_title(url_parent["url"], title)
                 urls_manager.set_url_ai_processed_by_url(
                     url_parent["url"], str(json.dumps(url_child_xml))
                 )

ohmyscrapper/modules/scrap_urls.py

@@ -2,154 +2,132 @@ import ohmyscrapper.models.urls_manager as urls_manager
 import ohmyscrapper.modules.sniff_url as sniff_url
 import ohmyscrapper.modules.load_txt as load_txt
 import ohmyscrapper.modules.classify_urls as classify_urls
+from ohmyscrapper.core import config
 
 import time
 import random
 
 
-def process_linkedin_redirect(url_report, url, verbose=False):
-    if verbose:
-        print("linkedin_redirect")
-
-    if url_report["total-a-links"] < 5:
-        if "first-a-link" in url_report.keys():
-            url_destiny = url_report["first-a-link"]
-        else:
-            urls_manager.set_url_error(url=url["url"], value="error: no first-a-link")
-            if verbose:
-                print("no url for:", url["url"])
-            return
-    else:
-        if "og:url" in url_report.keys():
-            url_destiny = url_report["og:url"]
-        else:
-            urls_manager.set_url_error(url=url["url"], value="error: no og:url")
-            if verbose:
-                print("no url for:", url["url"])
-            return
-    if verbose:
-        print(url["url"], ">>", url_destiny)
-    urls_manager.add_url(url=url_destiny)
-    urls_manager.set_url_destiny(url=url["url"], destiny=url_destiny)
-
+def scrap_url(url, verbose=False):
+    if url["url_type"] is None:
+        url["url_type"] = "generic"
 
-def process_linkedin_feed(url_report, url, verbose=False):
     if verbose:
-        print("linkedin_feed")
+        print("\n\n", url["url_type"] + ":", url["url"])
 
-    if "og:url" in url_report.keys():
-        url_destiny = url_report["og:url"]
-    else:
-        urls_manager.set_url_error(url=url["url"], value="error: no og:url")
+    try:
+        url_type = url["url_type"]
+        sniffing_config = config.get_url_sniffing()
+
+        if url_type not in sniffing_config:
+            default_type_sniffing = {
+                "bodytags": [{"h1": "title"}],
+                "metatags": [
+                    {"og:title": "title"},
+                    {"og:description": "description"},
+                    {"description": "description"},
+                ],
+            }
+            config.append_url_sniffing({url_type: default_type_sniffing})
+            sniffing_config = config.get_url_sniffing()
+
+        url_report = sniff_url.get_tags(
+            url=url["url"], sniffing_config=sniffing_config[url_type]
+        )
+    except Exception as e:
+        urls_manager.set_url_error(url=url["url"], value="error on scrapping")
+        urls_manager.touch_url(url=url["url"])
         if verbose:
-            print("no url for:", url["url"])
+            print("\n\n!!! ERROR FOR:", url["url"])
+            print(
+                "\n\n!!! you can check the URL using the command sniff-url",
+                url["url"],
+                "\n\n",
+            )
         return
 
-    if verbose:
-        print(url["url"], ">>", url_destiny)
-    urls_manager.add_url(url=url_destiny)
-    urls_manager.set_url_destiny(url=url["url"], destiny=url_destiny)
-
+    process_sniffed_url(
+        url_report=url_report,
+        url=url,
+        sniffing_config=sniffing_config[url_type],
+        verbose=verbose,
+    )
 
-def process_linkedin_job(url_report, url, verbose=False):
-    if verbose:
-        print("linkedin_job")
-    changed = False
-    if "h1" in url_report.keys():
-        if verbose:
-            print(url["url"], ": ", url_report["h1"])
-        urls_manager.set_url_h1(url=url["url"], value=url_report["h1"])
-        changed = True
-    elif "og:title" in url_report.keys():
-        if verbose:
-            print(url["url"], ": ", url_report["og:title"])
-        urls_manager.set_url_h1(url=url["url"], value=url_report["og:title"])
-        changed = True
+    urls_manager.set_url_json(url=url["url"], value=url_report["json"])
+    urls_manager.touch_url(url=url["url"])
 
-    if "description" in url_report.keys():
-        urls_manager.set_url_description(
-            url=url["url"], value=url_report["description"]
-        )
-        changed = True
-    elif "og:description" in url_report.keys():
-        urls_manager.set_url_description(
-            url=url["url"], value=url_report["og:description"]
-        )
-        changed = True
-    if not changed:
-        urls_manager.set_url_error(url=url["url"], value="error: no h1 or description")
+    return
 
 
-def process_linkedin_post(url_report, url, verbose=False):
+def process_sniffed_url(url_report, url, sniffing_config, verbose=False):
     if verbose:
-        print("linkedin_post or generic")
+        print(url["url_type"])
         print(url["url"])
     changed = False
-    if "h1" in url_report.keys():
-        if verbose:
-            print(url["url"], ": ", url_report["h1"])
-        urls_manager.set_url_h1(url=url["url"], value=url_report["h1"])
-        changed = True
-    elif "og:title" in url_report.keys():
-        urls_manager.set_url_h1(url=url["url"], value=url_report["og:title"])
-        changed = True
-    description = None
-    if "description" in url_report.keys():
-        description = url_report["description"]
-        changed = True
-    elif "og:description" in url_report.keys():
-        description = url_report["og:description"]
+
+    db_fields = {}
+    db_fields["title"] = None
+    db_fields["description"] = None
+    db_fields["url_destiny"] = None
+
+    if "metatags" in sniffing_config.keys():
+        for tag, bd_field in sniffing_config["metatags"].items():
+            if tag in url_report.keys():
+                if bd_field[:1] == "+":
+                    if db_fields[bd_field[1:]] is None:
+                        db_fields[bd_field[1:]] = ""
+                    db_fields[bd_field[1:]] = (
+                        db_fields[bd_field[1:]] + " " + url_report[tag]
+                    )
+                else:
+                    db_fields[bd_field] = url_report[tag]
+
+    if "bodytags" in sniffing_config.keys():
+        for tag, bd_field in sniffing_config["bodytags"].items():
+            if tag in url_report.keys():
+                if bd_field[:1] == "+":
+                    if db_fields[bd_field[1:]] is None:
+                        db_fields[bd_field[1:]] = ""
+                    db_fields[bd_field[1:]] = (
+                        db_fields[bd_field[1:]] + " " + url_report[tag]
+                    )
+                else:
+                    db_fields[bd_field] = url_report[tag]
+
+    if (
+        "atags" in sniffing_config.keys()
+        and "first-tag-as-url_destiny" in sniffing_config["atags"].keys()
+    ):
+        if (
+            url_report["total-a-links"]
+            < sniffing_config["atags"]["first-tag-as-url_destiny"]
+        ):
+            if "first-a-link" in url_report.keys():
+                db_fields["url_destiny"] = url_report["first-a-link"]
+
+    if db_fields["title"] is not None:
+        urls_manager.set_url_title(url=url["url"], value=db_fields["title"])
         changed = True
 
-    if description is not None:
-        urls_manager.set_url_description(url=url["url"], value=description)
+    if db_fields["description"] is not None:
+        urls_manager.set_url_description(url=url["url"], value=db_fields["description"])
         description_links = load_txt.put_urls_from_string(
-            text_to_process=description, parent_url=url["url"]
+            text_to_process=db_fields["description"], parent_url=url["url"]
         )
         urls_manager.set_url_description_links(url=url["url"], value=description_links)
 
-    if not changed:
-        urls_manager.set_url_error(url=url["url"], value="error: no h1 or description")
-
-
-def scrap_url(url, verbose=False):
-    # TODO: Need to change this
-
-    if url["url_type"] is None:
-        if verbose:
-            print("\n\ngeneric:", url["url"])
-        url["url_type"] = "generic"
-    else:
-        if verbose:
-            print("\n\n", url["url_type"] + ":", url["url"])
-    try:
-        url_report = sniff_url.get_tags(url=url["url"])
-    except Exception as e:
-        urls_manager.set_url_error(url=url["url"], value="error")
-        urls_manager.touch_url(url=url["url"])
-        if verbose:
-            print("\n\n!!! ERROR FOR:", url["url"])
-            print(
-                "\n\n!!! you can check the URL using the command sniff-url",
-                url["url"],
-                "\n\n",
-            )
-        return
-
-    if url["url_type"] == "linkedin_redirect":
-        process_linkedin_redirect(url_report=url_report, url=url, verbose=verbose)
-
-    if url["url_type"] == "linkedin_feed":
-        process_linkedin_feed(url_report=url_report, url=url, verbose=verbose)
-
-    if url["url_type"] == "linkedin_job":
-        process_linkedin_job(url_report=url_report, url=url, verbose=verbose)
+        changed = True
 
-    if url["url_type"] == "linkedin_post" or url["url_type"] == "generic":
-        process_linkedin_post(url_report=url_report, url=url, verbose=verbose)
+    if db_fields["url_destiny"] is not None:
+        urls_manager.add_url(url=db_fields["url_destiny"])
+        urls_manager.set_url_destiny(url=url["url"], destiny=db_fields["url_destiny"])
+        changed = True
 
-    urls_manager.set_url_json(url=url["url"], value=url_report["json"])
-    urls_manager.touch_url(url=url["url"])
+    if not changed:
+        urls_manager.set_url_error(
+            url=url["url"],
+            value="error: no title, url_destiny or description was founded",
+        )
 
 
 def isNaN(num):

ohmyscrapper/modules/sniff_url.py

@@ -3,39 +3,62 @@ from bs4 import BeautifulSoup
 import json
 
 
-def sniff_url(url="https://www.linkedin.com/in/cesardesouzacardoso/", silent=False):
+def sniff_url(
+    url="https://www.linkedin.com/in/cesardesouzacardoso/",
+    silent=False,
+    sniffing_config={},
+):
+    if "metatags" in sniffing_config:
+        metatags_to_search = sniffing_config["metatags"]
+    else:
+        metatags_to_search = [
+            "description",
+            "og:url",
+            "og:title",
+            "og:description",
+            "og:type",
+            "lnkd:url",
+        ]
+
+    if "bodytags" in sniffing_config:
+        body_tags_to_search = sniffing_config["bodytags"]
+    else:
+        body_tags_to_search = {
+            "h1": "",
+            "h2": "",
+        }
+
+    if type(metatags_to_search) is dict:
+        metatags_to_search = list(metatags_to_search.keys())
+
+    # force clean concatenate without any separator
+    if type(body_tags_to_search) is dict:
+        body_tags_to_search = list(body_tags_to_search.keys())
+
+    if type(body_tags_to_search) is list:
+        body_tags_to_search = dict.fromkeys(body_tags_to_search, " ")
+
     if not silent:
         print("checking url:", url)
 
     r = requests.get(url=url)
     soup = BeautifulSoup(r.text, "html.parser")
 
-    metatags_to_search = [
-        "description",
-        "og:url",
-        "og:title",
-        "og:description",
-        "og:type",
-        "lnkd:url",
-    ]
-
-    text_tags_to_search = {
-        "h1": "",
-        "h2": "|",
-    }
-
     final_report = {}
     final_report["scrapped-url"] = url
-    final_report.update(
-        _extract_meta_tags(
-            soup=soup, silent=silent, metatags_to_search=metatags_to_search
+    if len(metatags_to_search) > 0:
+        final_report.update(
+            _extract_meta_tags(
+                soup=soup, silent=silent, metatags_to_search=metatags_to_search
+            )
         )
-    )
-    final_report.update(
-        _extract_text_tags(
-            soup=soup, silent=silent, text_tags_to_search=text_tags_to_search
+
+    if len(body_tags_to_search) > 0:
+        final_report.update(
+            _extract_text_tags(
+                soup=soup, silent=silent, body_tags_to_search=body_tags_to_search
+            )
         )
-    )
     final_report["a_links"] = _extract_a_tags(soup=soup, silent=silent)
     final_report = _complementary_report(final_report, soup, silent).copy()
     final_report["json"] = json.dumps(final_report)
@@ -85,12 +108,12 @@ def _extract_meta_tags(soup, silent, metatags_to_search):
     return valid_meta_tags
 
 
-def _extract_text_tags(soup, silent, text_tags_to_search):
+def _extract_text_tags(soup, silent, body_tags_to_search):
     valid_text_tags = {}
     if not silent:
         print("\n\n\n\n---- all <text> tags ---\n")
     i = 0
-    for text_tag, separator in text_tags_to_search.items():
+    for text_tag, separator in body_tags_to_search.items():
         if len(soup.find_all(text_tag)) > 0:
             valid_text_tags[text_tag] = []
             for obj_tag in soup.find_all(text_tag):
@@ -128,5 +151,5 @@ def _complementary_report(final_report, soup, silent):
     return final_report
 
 
-def get_tags(url):
-    return sniff_url(url=url, silent=True)
+def get_tags(url, sniffing_config={}):
+    return sniff_url(url=url, silent=True, sniffing_config=sniffing_config)

{ohmyscrapper-0.6.1.dist-info → ohmyscrapper-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ohmyscrapper
-Version: 0.6.1
+Version: 0.7.0
 Summary: OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a final report with general information about job positions.
 Author: Cesar Cardoso
 Author-email: Cesar Cardoso <hello@cesarcardoso.cc>
@@ -16,7 +16,7 @@ Requires-Dist: urlextract>=1.9.0
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 
-# 🐶 OhMyScrapper - v0.6.1
+# 🐶 OhMyScrapper - v0.7.0
 
 OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a
 final report with general information about job positions.

ohmyscrapper-0.7.0.dist-info/RECORD

@@ -0,0 +1,21 @@
+ohmyscrapper/__init__.py,sha256=w5Ty9eszf8tEv72IQrFov0YbZWMqsraq448xhX3YGQs,6493
+ohmyscrapper/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
+ohmyscrapper/core/config.py,sha256=i_RA-zReNQIWWmsFar85qzRUqdqvTFMPeCP7Hya7ltU,2996
+ohmyscrapper/core/config_files.py,sha256=KC3yChTnlclclU9EKTqFBoAu9p6XdOKuegub5NPYDDY,2434
+ohmyscrapper/core/default_files/config.yaml,sha256=bgPBVlze2tOCbyrA47h_5BJ35UsXnqsjQszzy0vn-Pw,248
+ohmyscrapper/core/default_files/url_sniffing.yaml,sha256=MKdVR5HQ1i2yTRw2ijzxPSmIyhUno_R4L2k17r3EBBc,417
+ohmyscrapper/core/default_files/url_types.yaml,sha256=20kvv8_iWRT-pLa014RXYpAmPSonn6tDnG302rx7l-o,228
+ohmyscrapper/models/urls_manager.py,sha256=FC1j72M1gzNwC_PzPqnew986b-BI6s7zUv8Z7HiM1M0,11849
+ohmyscrapper/modules/classify_urls.py,sha256=GhiosAQUITy1DQe_PksYV9QRKVTgpkSE28dkutzbWVA,1038
+ohmyscrapper/modules/load_txt.py,sha256=dNkUZ2ehBiPx-q4fPczRiHFvnpzCrjeycFtexhWGmEE,3967
+ohmyscrapper/modules/merge_dbs.py,sha256=0pK3PPUGSbnaDkdpQUGCHemOVaKO37bfHwnsy_EVpWQ,115
+ohmyscrapper/modules/process_with_ai.py,sha256=kl39Jzl-PUwh6AfmTZ9SLFUYs9Sk4biqgt8rNz3X1FA,7255
+ohmyscrapper/modules/scrap_urls.py,sha256=CNoEC-d1r-u4qxnEVimm4ctP6MJGdU8y8VI2Nx0bBdM,6033
+ohmyscrapper/modules/seed.py,sha256=qDUE7TWx9iNQEzqThK4p7g8pTZjdpkmoqI8kOo_zdtk,983
+ohmyscrapper/modules/show.py,sha256=jsAs4g8ouA9wymkBfkDCbpVWKD-m_20uKG-m1cZAUGA,3877
+ohmyscrapper/modules/sniff_url.py,sha256=zJ2Uox2aUdQibL4UFLxg3t7GqJ7WwWEl0q3QSUbMEbc,4960
+ohmyscrapper/modules/untouch_all.py,sha256=DAwWYfqMFifHPtFCxSamu0AxHCgk6aJbTnBy6wLucXM,167
+ohmyscrapper-0.7.0.dist-info/WHEEL,sha256=xDCZ-UyfvkGuEHPeI7BcJzYKIZzdqN8A8o1M5Om8IyA,79
+ohmyscrapper-0.7.0.dist-info/entry_points.txt,sha256=BZud6D16XkfjelDa4Z33mji-KJbbZXgq2FoLrzjru5I,52
+ohmyscrapper-0.7.0.dist-info/METADATA,sha256=Doakf4oDT6oskPGdSlEoRJHBxUmm9FhWaHfDlNIfNuM,4096
+ohmyscrapper-0.7.0.dist-info/RECORD,,

ohmyscrapper-0.6.1.dist-info/RECORD

@@ -1,20 +0,0 @@
-ohmyscrapper/__init__.py,sha256=TGOizxll-06nyJdYSM8SRUccQ5Xhv6dDNW6sIbuH0Mk,6493
-ohmyscrapper/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
-ohmyscrapper/core/config.py,sha256=_me0T6IQqz7bA6Kh6IofNrb-o-07nipcLozUuPrz0l4,2722
-ohmyscrapper/core/config_files.py,sha256=KC3yChTnlclclU9EKTqFBoAu9p6XdOKuegub5NPYDDY,2434
-ohmyscrapper/core/default_files/config.yaml,sha256=9nMOhnnJUcZudXUq5WBEXCCgezfUKI3m4azIuSch_wQ,214
-ohmyscrapper/core/default_files/url_types.yaml,sha256=20kvv8_iWRT-pLa014RXYpAmPSonn6tDnG302rx7l-o,228
-ohmyscrapper/models/urls_manager.py,sha256=93WvHnk89hA2BfJfDsD2JlZBeRxo2T_F3FfypiRKKHs,11523
-ohmyscrapper/modules/classify_urls.py,sha256=4rt7_iPDcCGHhJg-f75wBfFmvjdvQj1xFFP-if_IeFM,926
-ohmyscrapper/modules/load_txt.py,sha256=dNkUZ2ehBiPx-q4fPczRiHFvnpzCrjeycFtexhWGmEE,3967
-ohmyscrapper/modules/merge_dbs.py,sha256=0pK3PPUGSbnaDkdpQUGCHemOVaKO37bfHwnsy_EVpWQ,115
-ohmyscrapper/modules/process_with_ai.py,sha256=Th-HMJzQYGQ4UBG8AGFsF5cCKIa1HlPATfmGLTTAE24,7222
-ohmyscrapper/modules/scrap_urls.py,sha256=dxpvPyJWtmQj1vZ6IgnhcICWw1eOxYOeplDfZzDTLw4,6864
-ohmyscrapper/modules/seed.py,sha256=qDUE7TWx9iNQEzqThK4p7g8pTZjdpkmoqI8kOo_zdtk,983
-ohmyscrapper/modules/show.py,sha256=jsAs4g8ouA9wymkBfkDCbpVWKD-m_20uKG-m1cZAUGA,3877
-ohmyscrapper/modules/sniff_url.py,sha256=dF6Nv54TC1Si-FRyqtw4V2WNk3NqaJ1h_PzwZm3UNzk,4126
-ohmyscrapper/modules/untouch_all.py,sha256=DAwWYfqMFifHPtFCxSamu0AxHCgk6aJbTnBy6wLucXM,167
-ohmyscrapper-0.6.1.dist-info/WHEEL,sha256=xDCZ-UyfvkGuEHPeI7BcJzYKIZzdqN8A8o1M5Om8IyA,79
-ohmyscrapper-0.6.1.dist-info/entry_points.txt,sha256=BZud6D16XkfjelDa4Z33mji-KJbbZXgq2FoLrzjru5I,52
-ohmyscrapper-0.6.1.dist-info/METADATA,sha256=k06ZCfkLkDuy_GvCj6jAFq2xfCUA5gN8cVlDH-2Q6Bs,4096
-ohmyscrapper-0.6.1.dist-info/RECORD,,