jaseci 1.4.0.16__tar.gz → 1.4.0.18__tar.gz

{jaseci-1.4.0.16 → jaseci-1.4.0.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jaseci
-Version: 1.4.0.16
+Version: 1.4.0.18
 Home-page: https://github.com/Jaseci-Labs/jaseci
 Author: Jason Mars
 Author-email: jason@jaseci.org

jaseci-1.4.0.18/jaseci/VERSION

@@ -0,0 +1 @@
+1.4.0.18

{jaseci-1.4.0.16 → jaseci-1.4.0.18}/jaseci/extens/act_lib/tests/test_webtool.py

@@ -26,3 +26,19 @@ class WebtoolTest(CoreTest):
     @jac_testcase("webtool.jac", "get_meta_invalid")
     def test_get_meta_invalid(self, ret):
         self.assertTrue("Failed at getting metadata" in ret["report"][0])
+
+    @jac_testcase("webtool.jac", "get_meta_timeout")
+    def test_get_meta_timeout(self, ret):
+        self.assertTrue("Failed at getting metadata" in ret["report"][0])
+
+    @jac_testcase("webtool.jac", "get_meta_need_header")
+    def test_get_meta_need_header(self, ret):
+        self.assertTrue(ret["success"])
+        expected_tags = set(["og:image", "og:type", "og:title"])
+        tags = set(
+            [
+                meta["property"] if "property" in meta else ""
+                for meta in ret["report"][0]
+            ]
+        )
+        self.assertTrue(tags.issuperset(expected_tags))

jaseci-1.4.0.18/jaseci/extens/act_lib/webtool.py

@@ -0,0 +1,30 @@
+"""Built in actions for Jaseci"""
+import requests
+from jaseci.jsorc.live_actions import jaseci_action
+from bs4 import BeautifulSoup
+
+
+@jaseci_action()
+def get_page_meta(url: str, timeout: int = 3, parser: str = "lxml"):
+    """
+    Util to parse metadata out of urls and html documents
+    Parser option: lxml (default), html5lib, html.parser
+    Comparison between parsers: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser
+    """
+    try:
+        webpage = requests.get(
+            url,
+            timeout=timeout,
+            headers={
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0"
+            },
+        )
+        soup = BeautifulSoup(webpage.content, features=parser)
+        meta = soup.find_all("meta")
+        meta_list = []
+        for tag in meta:
+            meta_list.append(dict(tag.attrs))
+        return meta_list
+    except Exception as e:
+        print("Failed")
+        return f"Failed at getting metadata for {url}: {str(e)}"

{jaseci-1.4.0.16 → jaseci-1.4.0.18}/jaseci/utils/utils.py

@@ -17,8 +17,9 @@ import traceback
 import inspect
 import unittest
 from time import time
-
+from pathlib import Path
 from pprint import pformat
+from typing import Union
 
 from jaseci.utils.log_utils import LimitedSlidingBuffer
 
@@ -331,3 +332,16 @@ def find_first_api(api_name, **api_endpoints):
         if api:
             return path, api
     raise InvalidApiException(f"api {api_name} is not existing!")
+
+
+cache_root = Path(Path.home(), ".jaseci/models")
+
+
+def model_base_path(cache_dir: Union[str, Path]) -> Path:
+    cache_dir = Path(cache_dir)
+
+    if not os.path.isabs(cache_dir):
+        model_cache = cache_root / cache_dir
+    else:
+        model_cache = cache_dir
+    return model_cache

{jaseci-1.4.0.16 → jaseci-1.4.0.18}/jaseci.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jaseci
-Version: 1.4.0.16
+Version: 1.4.0.18
 Home-page: https://github.com/Jaseci-Labs/jaseci
 Author: Jason Mars
 Author-email: jason@jaseci.org

{jaseci-1.4.0.16 → jaseci-1.4.0.18}/jaseci.egg-info/requires.txt

@@ -24,3 +24,4 @@ pygls
 mock
 beautifulsoup4<4.13.0,>=4.12.2
 lxml<4.10.0,>=4.9.2
+html5lib<1.2,>=1.1

{jaseci-1.4.0.16 → jaseci-1.4.0.18}/setup.py

@@ -38,6 +38,7 @@ setup(
         "mock",
         "beautifulsoup4>=4.12.2, <4.13.0",
         "lxml>=4.9.2, <4.10.0",
+        "html5lib>=1.1, <1.2",
     ],
     package_data={
         "": ["*.ini", "*.yaml", "jac.g4", "VERSION"],

jaseci-1.4.0.16/jaseci/VERSION

@@ -1 +0,0 @@
-1.4.0.16

jaseci-1.4.0.16/jaseci/extens/act_lib/webtool.py

@@ -1,22 +0,0 @@
-"""Built in actions for Jaseci"""
-import requests
-from jaseci.jsorc.live_actions import jaseci_action
-from bs4 import BeautifulSoup
-
-
-@jaseci_action()
-def get_page_meta(url: str):
-    """
-    Util to parse metadata out of urls and html documents
-    """
-    try:
-        webpage = requests.get(url)
-        soup = BeautifulSoup(webpage.content, features="lxml")
-        meta = soup.find_all("meta")
-        meta_list = []
-        for tag in meta:
-            meta_list.append(dict(tag.attrs))
-        return meta_list
-    except Exception as e:
-        print("Failed")
-        return f"Failed at getting metadata for {url}: {str(e)}"