ohmyscrapper 0.3.4__tar.gz → 0.4.0__tar.gz

{ohmyscrapper-0.3.4 → ohmyscrapper-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ohmyscrapper
-Version: 0.3.4
+Version: 0.4.0
 Summary: OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a final report with general information about job positions.
 Author: Cesar Cardoso
 Author-email: Cesar Cardoso <hello@cesarcardoso.cc>
@@ -16,7 +16,7 @@ Requires-Dist: urlextract>=1.9.0
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 
-# 🐶 OhMyScrapper - v0.3.4
+# 🐶 OhMyScrapper - v0.4.0
 
 OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a
 final report with general information about job positions.
@@ -78,10 +78,16 @@ in this folder and use the command `load`:
 ```shell
 ohmyscrapper load
 ```
-or, if you have another file in a different folder, just use the argument `-file` like this:
+or, if you have another file in a different folder, just use the argument `-input` like this:
 ```shell
-ohmyscrapper load -file=my-text-file.txt
+ohmyscrapper load -input=my-text-file.txt
 ```
+In this case, you can add an url directly to the database, like this:
+```shell
+ohmyscrapper load -input=https://cesarcardoso.cc/
+```
+That will append the last url in the database to be scraped.
+
 That will create a database if it doesn't exist and store every url the oh-my-scrapper
 find. After that, let's scrap the urls with the command `scrap-urls`:
 

{ohmyscrapper-0.3.4 → ohmyscrapper-0.4.0}/README.md

@@ -1,4 +1,4 @@
-# 🐶 OhMyScrapper - v0.3.4
+# 🐶 OhMyScrapper - v0.4.0
 
 OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a
 final report with general information about job positions.
@@ -60,10 +60,16 @@ in this folder and use the command `load`:
 ```shell
 ohmyscrapper load
 ```
-or, if you have another file in a different folder, just use the argument `-file` like this:
+or, if you have another file in a different folder, just use the argument `-input` like this:
 ```shell
-ohmyscrapper load -file=my-text-file.txt
+ohmyscrapper load -input=my-text-file.txt
 ```
+In this case, you can add an url directly to the database, like this:
+```shell
+ohmyscrapper load -input=https://cesarcardoso.cc/
+```
+That will append the last url in the database to be scraped.
+
 That will create a database if it doesn't exist and store every url the oh-my-scrapper
 find. After that, let's scrap the urls with the command `scrap-urls`:
 

{ohmyscrapper-0.3.4 → ohmyscrapper-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ohmyscrapper"
-version = "0.3.4"
+version = "0.4.0"
 description = "OhMyScrapper scrapes texts and urls looking for links and jobs-data to create a final report with general information about job positions."
 readme = "README.md"
 authors = [

{ohmyscrapper-0.3.4 → ohmyscrapper-0.4.0}/src/ohmyscrapper/__init__.py

@@ -19,7 +19,7 @@ from ohmyscrapper.modules.merge_dbs import merge_dbs
 
 def main():
     parser = argparse.ArgumentParser(prog="ohmyscrapper")
-    parser.add_argument("--version", action="version", version="%(prog)s v0.3.4")
+    parser.add_argument("--version", action="version", version="%(prog)s v0.4.0")
 
     subparsers = parser.add_subparsers(dest="command", help="Available commands")
     start_parser = subparsers.add_parser(
@@ -54,7 +54,7 @@ def main():
     )
 
     load_txt_parser = subparsers.add_parser("load", help="📦 Load txt file")
-    load_txt_parser.add_argument("-file", default=None, help="File path.")
+    load_txt_parser.add_argument("-input", default=None, help="File path or url.")
     load_txt_parser.add_argument(
         "--verbose", default=False, help="Run in verbose mode", action="store_true"
     )
@@ -114,7 +114,7 @@ def main():
         return
 
     if args.command == "load":
-        load_txt(file_name=args.file, verbose=args.verbose)
+        load_txt(file_name=args.input, verbose=args.verbose)
         return
 
     if args.command == "seed":

{ohmyscrapper-0.3.4 → ohmyscrapper-0.4.0}/src/ohmyscrapper/modules/load_txt.py

@@ -22,11 +22,15 @@ def load_txt(file_name=None, verbose=False):
     if file_name is not None:
         print(f"📖 reading file `{file_name}`... ")
         if not os.path.exists(file_name):
-            print(f"\n file `{file_name}` not found.")
-            return
-        text_file_content = _increment_file_name(
-            text_file_content=text_file_content, file_name=file_name
-        )
+            if file_name.startswith("https://") or file_name.startswith("http://"):
+                text_file_content = " " + file_name + " "
+            else:
+                print(f"\n file `{file_name}` not found.")
+                return
+        else:
+            text_file_content = _increment_file_name(
+                text_file_content=text_file_content, file_name=file_name
+            )
     else:
         print("📂 reading /input directory... ")
         dir_files = "input"