ohmyscrapper 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

ohmyscrapper/__init__.py

@@ -19,12 +19,19 @@ from ohmyscrapper.modules.merge_dbs import merge_dbs
 
 def main():
     parser = argparse.ArgumentParser(prog="ohmyscrapper")
-    parser.add_argument("--version", action="version", version="%(prog)s v0.2.1")
+    parser.add_argument("--version", action="version", version="%(prog)s v0.2.3")
 
     subparsers = parser.add_subparsers(dest="command", help="Available commands")
+    start_parser = subparsers.add_parser(
+        "start", help="Make the entire process of loading, processing and exporting with the default configuration."
+    )
+
+    start_parser.add_argument(
+        "--ai", default=False, help="Make the entire process of loading, processing, reprocessing with AI and exporting with the default configuration.", action="store_true"
+    )
 
     ai_process_parser = subparsers.add_parser(
-        "process-with-ai", help="Process with AI."
+        "ai", help="Process with AI."
     )
     ai_process_parser.add_argument(
         "--history", default=False, help="Reprocess ai history", action="store_true"
@@ -157,6 +164,16 @@ def main():
         merge_dbs()
         return
 
+    if args.command == "start":
+        load_txt()
+        scrap_urls(recursive=True,ignore_valid_prefix=True,randomize=False,only_parents=False)
+        if args.ai:
+            process_with_ai()
+        export_urls()
+        export_urls(csv_file="output/urls-simplified.csv", simplify=True)
+        export_report()
+        return
+
 
 if __name__ == "__main__":
     main()

{ohmyscrapper-0.2.1.dist-info → ohmyscrapper-0.2.3.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.3
 Name: ohmyscrapper
-Version: 0.2.1
+Version: 0.2.3
 Summary: This project aims to create a text-based scraper containing links to create a final PDF with general information about job openings.
-Author: Cesar Cardoso gh@bouli
-Author-email: Cesar Cardoso gh@bouli <hello@cesarcardoso.cc>
+Author: Cesar Cardoso
+Author-email: Cesar Cardoso <hello@cesarcardoso.cc>
 Requires-Dist: beautifulsoup4>=4.14.3
 Requires-Dist: google-genai>=1.55.0
 Requires-Dist: markdown>=3.10
@@ -16,13 +16,11 @@ Requires-Dist: urlextract>=1.9.0
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 
-# OhMyScrapper - v0.2.1
+# OhMyScrapper - v0.2.3
 
 This project aims to create a text-based scraper containing links to create a
 final PDF with general information about job openings.
 
-> This project is using [uv](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer) by default.
-
 ## Scope
 
 - Read texts;
@@ -31,11 +29,23 @@ final PDF with general information about job openings.
 
 ## Installation
 
+You can install directly in your `pip`:
+```shell
+pip install ohmyscrapper
+```
+
 I recomend to use the [uv](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer), so you can just use the command bellow and everything is installed:
 ```shell
-uv sync
+uv add ohmyscrapper
+uv run ohmyscrapper --version
 ```
 
+But you can use everything as a tool, for example:
+```shell
+uvx ohmyscrapper --version
+```
+
+
 ## How to use and test (development only)
 
 OhMyScrapper works in 3 stages:
@@ -46,7 +56,7 @@ OhMyScrapper works in 3 stages:
 
 You can do 3 stages with the command:
 ```shell
-make start
+ohmyscrapper start
 ```
 > Remember to add your text file in the folder `/input` with the name `_chat.txt`!
 
@@ -66,17 +76,17 @@ use the whatsapp history, but it works with any txt file.
 The default file is `input/_chat.txt`. If you have the default file you just use
 the command `load`:
 ```shell
-make load
+ohmyscrapper load
 ```
 or, if you have another file, just use the argument `-file` like this:
 ```shell
-uv run main.py load -file=my-text-file.txt
+ohmyscrapper load -file=my-text-file.txt
 ```
 That will create a database if it doesn't exist and store every url the oh-my-scrapper
 find. After that, let's scrap the urls with the command `scrap-urls`:
 
 ```shell
-make scrap-urls
+ohmyscrapper scrap-urls --recursive --ignore-type
 ```
 
 That will scrap only the linkedin urls we are interested in. For now they are:
@@ -88,23 +98,33 @@ That will scrap only the linkedin urls we are interested in. For now they are:
 
 But we can use every other one generically using the argument `--ignore-type`:
 ```shell
-uv run main.py scrap-urls --ignore-type
+ohmyscrapper scrap-urls --ignore-type
 ```
 
 And we can ask to make it recursively adding the argument `--recursive`:
 ```shell
-uv run main.py scrap-urls --recursive
+ohmyscrapper scrap-urls --recursive
 ```
 > !!! important: we are not sure about blocks we can have for excess of requests
 
 And we can finally export with the command:
 ```shell
-make export
+ohmyscrapper export
+ohmyscrapper export --file=output/urls-simplified.csv --simplify
+ohmyscrapper report
 ```
 
 
 That's the basic usage!
 But you can understand more using the help:
 ```shell
-uv run main.py --help
+ohmyscrapper --help
 ```
+
+## See Also
+
+- Github: https://github.com/bouli/ohmyscrapper
+- PyPI: https://pypi.org/project/ohmyscrapper/
+
+## License
+This package is distributed under the [MIT license](https://opensource.org/license/MIT).

{ohmyscrapper-0.2.1.dist-info → ohmyscrapper-0.2.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ohmyscrapper/__init__.py,sha256=OOoRFtkBKaTIf74FStI0MGtk-LUQOuN0QnBZRfRWauA,5145
+ohmyscrapper/__init__.py,sha256=6k-fyuKkTefy5lJiRJFDP7wfkGhYRC8qgdfYlNB_3sk,5841
 ohmyscrapper/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
 ohmyscrapper/models/urls_manager.py,sha256=xKql_xdwfRwgpMyriuIrZ0Srz4gYQGMfWClEWpGRtNE,11183
 ohmyscrapper/modules/classify_urls.py,sha256=eyHtTHDZp2pGmYw_X-7LrbeVOgDPcRQdhu0oEuwQtKA,743
@@ -10,7 +10,7 @@ ohmyscrapper/modules/seed.py,sha256=KeTSbmTdNTkVCtzk9iQmeuEqB0kG-rTZJb2a1WdROL4,
 ohmyscrapper/modules/show.py,sha256=u0L9uxgU8Xt_-myA3r7byuOmnX_-2gkpTtXWkXon1ns,3572
 ohmyscrapper/modules/sniff_url.py,sha256=jQDc7aSimuOOedw2fSXZlf6_o0OqQHOr6NsWb4n0XgI,2720
 ohmyscrapper/modules/untouch_all.py,sha256=E1U9e3sOG7suzc8ZTWcYiQQo9mPmLJ0piXdXUjFLEd4,162
-ohmyscrapper-0.2.1.dist-info/WHEEL,sha256=xDCZ-UyfvkGuEHPeI7BcJzYKIZzdqN8A8o1M5Om8IyA,79
-ohmyscrapper-0.2.1.dist-info/entry_points.txt,sha256=BZud6D16XkfjelDa4Z33mji-KJbbZXgq2FoLrzjru5I,52
-ohmyscrapper-0.2.1.dist-info/METADATA,sha256=Sl1HuVlxTSSAYz9ga0zJ9xUpWGY2NZOkNu1xTNtGUu8,3411
-ohmyscrapper-0.2.1.dist-info/RECORD,,
+ohmyscrapper-0.2.3.dist-info/WHEEL,sha256=xDCZ-UyfvkGuEHPeI7BcJzYKIZzdqN8A8o1M5Om8IyA,79
+ohmyscrapper-0.2.3.dist-info/entry_points.txt,sha256=BZud6D16XkfjelDa4Z33mji-KJbbZXgq2FoLrzjru5I,52
+ohmyscrapper-0.2.3.dist-info/METADATA,sha256=uwthvf7vwhb6H14KdbPLibzp1c3PaVmcJIYePmV8cRc,3832
+ohmyscrapper-0.2.3.dist-info/RECORD,,