ohmyscrapper 0.2.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

ohmyscrapper/modules/sniff_url.py

@@ -6,8 +6,11 @@ import json
 def sniff_url(url="https://www.linkedin.com/in/cesardesouzacardoso/", silent=False):
     if not silent:
         print("checking url:", url)
-    report_meta_tags = []
-    tags_to_search = [
+
+    r = requests.get(url=url)
+    soup = BeautifulSoup(r.text, "html.parser")
+
+    metatags_to_search = [
         "description",
         "og:url",
         "og:title",
@@ -16,18 +19,62 @@ def sniff_url(url="https://www.linkedin.com/in/cesardesouzacardoso/", silent=Fal
         "lnkd:url",
     ]
 
-    r = requests.get(url=url)
-    soup = BeautifulSoup(r.text, "html.parser")
+    text_tags_to_search = {
+        "h1": "",
+        "h2": "|",
+    }
+
+    final_report = {}
+    final_report["scrapped-url"] = url
+    final_report.update(
+        _extract_meta_tags(
+            soup=soup, silent=silent, metatags_to_search=metatags_to_search
+        )
+    )
+    final_report.update(
+        _extract_text_tags(
+            soup=soup, silent=silent, text_tags_to_search=text_tags_to_search
+        )
+    )
+    final_report["a_links"] = _extract_a_tags(soup=soup, silent=silent)
+    final_report = _complementary_report(final_report, soup, silent).copy()
+    final_report["json"] = json.dumps(final_report)
+
+    return final_report
+
+
+def _extract_a_tags(soup, silent):
+    a_links = []
+    if not silent:
+        print("\n\n\n\n---- all <a> links ---")
+
+    i = 0
+    for a_tag in soup.find_all("a"):
+        i = i + 1
+        a_links.append({"text": a_tag.text, "href": a_tag.get("href")})
+        if not silent:
+            print("\n-- <a> link", i, "-- ")
+            print("target:", a_tag.get("target"))
+            print("text:", str(a_tag.text).strip())
+            print("href:", a_tag.get("href"))
+            print("-------------- ")
+    return a_links
 
+
+def _extract_meta_tags(soup, silent, metatags_to_search):
+    valid_meta_tags = {}
     if not silent:
         print("\n\n\n\n---- all <meta> tags ---\n")
     i = 0
     for meta_tag in soup.find_all("meta"):
         if (
-            meta_tag.get("name") in tags_to_search
-            or meta_tag.get("property") in tags_to_search
+            meta_tag.get("name") in metatags_to_search
+            or meta_tag.get("property") in metatags_to_search
         ):
-            report_meta_tags.append(meta_tag)
+            if meta_tag.get("name") is not None:
+                valid_meta_tags[meta_tag.get("name")] = meta_tag.get("content")
+            elif meta_tag.get("property") is not None:
+                valid_meta_tags[meta_tag.get("property")] = meta_tag.get("content")
         i = i + 1
         if not silent:
             print("-- meta tag", i, "--")
@@ -35,51 +82,48 @@ def sniff_url(url="https://www.linkedin.com/in/cesardesouzacardoso/", silent=Fal
             print("property:", meta_tag.get("property"))
             print("content:", meta_tag.get("content"))
             print("---------------- \n")
+    return valid_meta_tags
+
 
+def _extract_text_tags(soup, silent, text_tags_to_search):
+    valid_text_tags = {}
     if not silent:
-        print("\n\n\n\n---- all <a> links ---")
-        i = 0
-        for a_tag in soup.find_all("a"):
-            i = i + 1
-            print("\n-- a link", i, "-- ")
-            print("target:", a_tag.get("target"))
-            print("text:", a_tag.text)
-            print("href:", a_tag.get("href"))
-            print("-------------- ")
+        print("\n\n\n\n---- all <text> tags ---\n")
+    i = 0
+    for text_tag, separator in text_tags_to_search.items():
+        if len(soup.find_all(text_tag)) > 0:
+            valid_text_tags[text_tag] = []
+            for obj_tag in soup.find_all(text_tag):
+                valid_text_tags[text_tag].append(obj_tag.text.strip())
+            valid_text_tags[text_tag] = separator.join(valid_text_tags[text_tag])
+        i = i + 1
+        if not silent:
+            print("-- text tag", i, "--")
+            print("name:", text_tag)
+            print("separator:", separator)
+            print("texts:", valid_text_tags[text_tag])
+            print("---------------- \n")
+    return valid_text_tags
 
-    final_report = {}
-    final_report["scrapped-url"] = url
-    if len(soup.find_all("h1")) > 0:
-        final_report["h1"] = soup.find("h1").text
-
-    for report_meta_tag in report_meta_tags:
-        if report_meta_tag.get("name") is not None:
-            final_report[report_meta_tag.get("name")] = report_meta_tag.get("content")
-        elif report_meta_tag.get("property") is not None:
-            final_report[report_meta_tag.get("property")] = report_meta_tag.get(
-                "content"
-            )
-
-    if len(soup.find_all("a")) > 0:
-        final_report["first-a-link"] = soup.find("a").get("href")
-        final_report["total-a-links"] = len(soup.find_all("a"))
+
+def _complementary_report(final_report, soup, silent):
+
+    if len(final_report["a_links"]) > 0:
+        final_report["first-a-link"] = final_report["a_links"][0]["href"]
+        final_report["total-a-links"] = len(final_report["a_links"])
     else:
         final_report["first-a-link"] = ""
         final_report["total-a-links"] = 0
 
-    if len(soup.find_all("h2")) > 0:
-        final_report["h2"] = soup.find("h2").text
-
     if len(soup.find_all("meta")) > 0:
         final_report["total-meta-tags"] = len(soup.find_all("meta"))
     else:
         final_report["total-meta-tags"] = 0
-
-    final_report["json"] = json.dumps(final_report)
     if not silent:
         print("\n\n\n----report---\n")
         for key in final_report:
-            print("* ", key, ":", final_report[key])
+            if key != "a_links":
+                print("* ", key, ":", final_report[key])
 
     return final_report
 

ohmyscrapper/modules/untouch_all.py

@@ -3,5 +3,5 @@ import ohmyscrapper.models.urls_manager as urls_manager
 
 def untouch_all():
     urls_manager.untouch_all_urls()
-    print("urls have been untouched")
+    print("🙌 urls have been untouched")
     return

{ohmyscrapper-0.2.1.dist-info → ohmyscrapper-0.6.1.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.3
 Name: ohmyscrapper
-Version: 0.2.1
-Summary: This project aims to create a text-based scraper containing links to create a final PDF with general information about job openings.
-Author: Cesar Cardoso gh@bouli
-Author-email: Cesar Cardoso gh@bouli <hello@cesarcardoso.cc>
+Version: 0.6.1
+Summary: OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a final report with general information about job positions.
+Author: Cesar Cardoso
+Author-email: Cesar Cardoso <hello@cesarcardoso.cc>
 Requires-Dist: beautifulsoup4>=4.14.3
 Requires-Dist: google-genai>=1.55.0
 Requires-Dist: markdown>=3.10
@@ -16,39 +16,50 @@ Requires-Dist: urlextract>=1.9.0
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 
-# OhMyScrapper - v0.2.1
+# 🐶 OhMyScrapper - v0.6.1
 
-This project aims to create a text-based scraper containing links to create a
-final PDF with general information about job openings.
-
-> This project is using [uv](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer) by default.
+OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a
+final report with general information about job positions.
 
 ## Scope
 
 - Read texts;
-- Extract links;
-- Use meta og:tags to extract information;
+- Extract and load urls;
+- Scrapes the urls looking for og:tags and titles;
+- Export a list of links with relevant information;
 
 ## Installation
 
+You can install directly in your `pip`:
+```shell
+pip install ohmyscrapper
+```
+
 I recomend to use the [uv](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer), so you can just use the command bellow and everything is installed:
 ```shell
-uv sync
+uv add ohmyscrapper
+uv run ohmyscrapper --version
+```
+
+But you can use everything as a tool, for example:
+```shell
+uvx ohmyscrapper --version
 ```
 
+
 ## How to use and test (development only)
 
 OhMyScrapper works in 3 stages:
 
-1. It collects and loads urls from a text (by default `input/_chat.txt`) in a database;
+1. It collects and loads urls from a text in a database;
 2. It scraps/access the collected urls and read what is relevant. If it finds new urls, they are collected as well;
 3. Export a list of urls in CSV files;
 
 You can do 3 stages with the command:
 ```shell
-make start
+ohmyscrapper start
 ```
-> Remember to add your text file in the folder `/input` with the name `_chat.txt`!
+> Remember to add your text file in the folder `/input` with the name that finishes with `.txt`!
 
 You will find the exported files in the folder `/output` like this:
 - `/output/report.csv`
@@ -60,23 +71,28 @@ You will find the exported files in the folder `/output` like this:
 
 ### BUT: if you want to do step by step, here it is:
 
-First we load a text file you would like to look for urls, the idea here is to
-use the whatsapp history, but it works with any txt file.
+First we load a text file you would like to look for urls. It it works with any txt file.
 
-The default file is `input/_chat.txt`. If you have the default file you just use
-the command `load`:
+The default folder is `/input`. Put one or more text (finished with `.txt`) files
+in this folder and use the command `load`:
+```shell
+ohmyscrapper load
+```
+or, if you have another file in a different folder, just use the argument `-input` like this:
 ```shell
-make load
+ohmyscrapper load -input=my-text-file.txt
 ```
-or, if you have another file, just use the argument `-file` like this:
+In this case, you can add an url directly to the database, like this:
 ```shell
-uv run main.py load -file=my-text-file.txt
+ohmyscrapper load -input=https://cesarcardoso.cc/
 ```
+That will append the last url in the database to be scraped.
+
 That will create a database if it doesn't exist and store every url the oh-my-scrapper
 find. After that, let's scrap the urls with the command `scrap-urls`:
 
 ```shell
-make scrap-urls
+ohmyscrapper scrap-urls --recursive --ignore-type
 ```
 
 That will scrap only the linkedin urls we are interested in. For now they are:
@@ -88,23 +104,33 @@ That will scrap only the linkedin urls we are interested in. For now they are:
 
 But we can use every other one generically using the argument `--ignore-type`:
 ```shell
-uv run main.py scrap-urls --ignore-type
+ohmyscrapper scrap-urls --ignore-type
 ```
 
 And we can ask to make it recursively adding the argument `--recursive`:
 ```shell
-uv run main.py scrap-urls --recursive
+ohmyscrapper scrap-urls --recursive
 ```
 > !!! important: we are not sure about blocks we can have for excess of requests
 
 And we can finally export with the command:
 ```shell
-make export
+ohmyscrapper export
+ohmyscrapper export --file=output/urls-simplified.csv --simplify
+ohmyscrapper report
 ```
 
 
 That's the basic usage!
 But you can understand more using the help:
 ```shell
-uv run main.py --help
+ohmyscrapper --help
 ```
+
+## See Also
+
+- Github: https://github.com/bouli/ohmyscrapper
+- PyPI: https://pypi.org/project/ohmyscrapper/
+
+## License
+This package is distributed under the [MIT license](https://opensource.org/license/MIT).

ohmyscrapper-0.6.1.dist-info/RECORD

@@ -0,0 +1,20 @@
+ohmyscrapper/__init__.py,sha256=TGOizxll-06nyJdYSM8SRUccQ5Xhv6dDNW6sIbuH0Mk,6493
+ohmyscrapper/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
+ohmyscrapper/core/config.py,sha256=_me0T6IQqz7bA6Kh6IofNrb-o-07nipcLozUuPrz0l4,2722
+ohmyscrapper/core/config_files.py,sha256=KC3yChTnlclclU9EKTqFBoAu9p6XdOKuegub5NPYDDY,2434
+ohmyscrapper/core/default_files/config.yaml,sha256=9nMOhnnJUcZudXUq5WBEXCCgezfUKI3m4azIuSch_wQ,214
+ohmyscrapper/core/default_files/url_types.yaml,sha256=20kvv8_iWRT-pLa014RXYpAmPSonn6tDnG302rx7l-o,228
+ohmyscrapper/models/urls_manager.py,sha256=93WvHnk89hA2BfJfDsD2JlZBeRxo2T_F3FfypiRKKHs,11523
+ohmyscrapper/modules/classify_urls.py,sha256=4rt7_iPDcCGHhJg-f75wBfFmvjdvQj1xFFP-if_IeFM,926
+ohmyscrapper/modules/load_txt.py,sha256=dNkUZ2ehBiPx-q4fPczRiHFvnpzCrjeycFtexhWGmEE,3967
+ohmyscrapper/modules/merge_dbs.py,sha256=0pK3PPUGSbnaDkdpQUGCHemOVaKO37bfHwnsy_EVpWQ,115
+ohmyscrapper/modules/process_with_ai.py,sha256=Th-HMJzQYGQ4UBG8AGFsF5cCKIa1HlPATfmGLTTAE24,7222
+ohmyscrapper/modules/scrap_urls.py,sha256=dxpvPyJWtmQj1vZ6IgnhcICWw1eOxYOeplDfZzDTLw4,6864
+ohmyscrapper/modules/seed.py,sha256=qDUE7TWx9iNQEzqThK4p7g8pTZjdpkmoqI8kOo_zdtk,983
+ohmyscrapper/modules/show.py,sha256=jsAs4g8ouA9wymkBfkDCbpVWKD-m_20uKG-m1cZAUGA,3877
+ohmyscrapper/modules/sniff_url.py,sha256=dF6Nv54TC1Si-FRyqtw4V2WNk3NqaJ1h_PzwZm3UNzk,4126
+ohmyscrapper/modules/untouch_all.py,sha256=DAwWYfqMFifHPtFCxSamu0AxHCgk6aJbTnBy6wLucXM,167
+ohmyscrapper-0.6.1.dist-info/WHEEL,sha256=xDCZ-UyfvkGuEHPeI7BcJzYKIZzdqN8A8o1M5Om8IyA,79
+ohmyscrapper-0.6.1.dist-info/entry_points.txt,sha256=BZud6D16XkfjelDa4Z33mji-KJbbZXgq2FoLrzjru5I,52
+ohmyscrapper-0.6.1.dist-info/METADATA,sha256=k06ZCfkLkDuy_GvCj6jAFq2xfCUA5gN8cVlDH-2Q6Bs,4096
+ohmyscrapper-0.6.1.dist-info/RECORD,,

ohmyscrapper-0.2.1.dist-info/RECORD

@@ -1,16 +0,0 @@
-ohmyscrapper/__init__.py,sha256=OOoRFtkBKaTIf74FStI0MGtk-LUQOuN0QnBZRfRWauA,5145
-ohmyscrapper/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
-ohmyscrapper/models/urls_manager.py,sha256=xKql_xdwfRwgpMyriuIrZ0Srz4gYQGMfWClEWpGRtNE,11183
-ohmyscrapper/modules/classify_urls.py,sha256=eyHtTHDZp2pGmYw_X-7LrbeVOgDPcRQdhu0oEuwQtKA,743
-ohmyscrapper/modules/load_txt.py,sha256=mL60OGsh-R80P88vxyqvfBEFag9yhSFFbg5pwtu1f90,889
-ohmyscrapper/modules/merge_dbs.py,sha256=0pK3PPUGSbnaDkdpQUGCHemOVaKO37bfHwnsy_EVpWQ,115
-ohmyscrapper/modules/process_with_ai.py,sha256=TpumucIVNZulKOw2idy4hD3vG5IhG5pbhyJImYFP8g0,6844
-ohmyscrapper/modules/scrap_urls.py,sha256=KQVs3R03X80hmvvJAU1SqnNhwXEeVV99WlN8TxSKqA8,6097
-ohmyscrapper/modules/seed.py,sha256=KeTSbmTdNTkVCtzk9iQmeuEqB0kG-rTZJb2a1WdROL4,129
-ohmyscrapper/modules/show.py,sha256=u0L9uxgU8Xt_-myA3r7byuOmnX_-2gkpTtXWkXon1ns,3572
-ohmyscrapper/modules/sniff_url.py,sha256=jQDc7aSimuOOedw2fSXZlf6_o0OqQHOr6NsWb4n0XgI,2720
-ohmyscrapper/modules/untouch_all.py,sha256=E1U9e3sOG7suzc8ZTWcYiQQo9mPmLJ0piXdXUjFLEd4,162
-ohmyscrapper-0.2.1.dist-info/WHEEL,sha256=xDCZ-UyfvkGuEHPeI7BcJzYKIZzdqN8A8o1M5Om8IyA,79
-ohmyscrapper-0.2.1.dist-info/entry_points.txt,sha256=BZud6D16XkfjelDa4Z33mji-KJbbZXgq2FoLrzjru5I,52
-ohmyscrapper-0.2.1.dist-info/METADATA,sha256=Sl1HuVlxTSSAYz9ga0zJ9xUpWGY2NZOkNu1xTNtGUu8,3411
-ohmyscrapper-0.2.1.dist-info/RECORD,,