jaseci 1.4.0.16__py3-none-any.whl → 1.4.0.18__py3-none-any.whl

jaseci/VERSION

@@ -1 +1 @@
-1.4.0.16
+1.4.0.18

jaseci/extens/act_lib/tests/test_webtool.py

@@ -26,3 +26,19 @@ class WebtoolTest(CoreTest):
     @jac_testcase("webtool.jac", "get_meta_invalid")
     def test_get_meta_invalid(self, ret):
         self.assertTrue("Failed at getting metadata" in ret["report"][0])
+
+    @jac_testcase("webtool.jac", "get_meta_timeout")
+    def test_get_meta_timeout(self, ret):
+        self.assertTrue("Failed at getting metadata" in ret["report"][0])
+
+    @jac_testcase("webtool.jac", "get_meta_need_header")
+    def test_get_meta_need_header(self, ret):
+        self.assertTrue(ret["success"])
+        expected_tags = set(["og:image", "og:type", "og:title"])
+        tags = set(
+            [
+                meta["property"] if "property" in meta else ""
+                for meta in ret["report"][0]
+            ]
+        )
+        self.assertTrue(tags.issuperset(expected_tags))

jaseci/extens/act_lib/webtool.py

@@ -5,13 +5,21 @@ from bs4 import BeautifulSoup
 
 
 @jaseci_action()
-def get_page_meta(url: str):
+def get_page_meta(url: str, timeout: int = 3, parser: str = "lxml"):
     """
     Util to parse metadata out of urls and html documents
+    Parser option: lxml (default), html5lib, html.parser
+    Comparison between parsers: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser
     """
     try:
-        webpage = requests.get(url)
-        soup = BeautifulSoup(webpage.content, features="lxml")
+        webpage = requests.get(
+            url,
+            timeout=timeout,
+            headers={
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0"
+            },
+        )
+        soup = BeautifulSoup(webpage.content, features=parser)
         meta = soup.find_all("meta")
         meta_list = []
         for tag in meta:

jaseci/utils/utils.py

@@ -17,8 +17,9 @@ import traceback
 import inspect
 import unittest
 from time import time
-
+from pathlib import Path
 from pprint import pformat
+from typing import Union
 
 from jaseci.utils.log_utils import LimitedSlidingBuffer
 
@@ -331,3 +332,16 @@ def find_first_api(api_name, **api_endpoints):
         if api:
             return path, api
     raise InvalidApiException(f"api {api_name} is not existing!")
+
+
+cache_root = Path(Path.home(), ".jaseci/models")
+
+
+def model_base_path(cache_dir: Union[str, Path]) -> Path:
+    cache_dir = Path(cache_dir)
+
+    if not os.path.isabs(cache_dir):
+        model_cache = cache_root / cache_dir
+    else:
+        model_cache = cache_dir
+    return model_cache

{jaseci-1.4.0.16.dist-info → jaseci-1.4.0.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jaseci
-Version: 1.4.0.16
+Version: 1.4.0.18
 Home-page: https://github.com/Jaseci-Labs/jaseci
 Author: Jason Mars
 Author-email: jason@jaseci.org
@@ -31,4 +31,5 @@ Requires-Dist: pygls
 Requires-Dist: mock
 Requires-Dist: beautifulsoup4 (<4.13.0,>=4.12.2)
 Requires-Dist: lxml (<4.10.0,>=4.9.2)
+Requires-Dist: html5lib (<1.2,>=1.1)
 

{jaseci-1.4.0.16.dist-info → jaseci-1.4.0.18.dist-info}/RECORD

@@ -1,4 +1,4 @@
-jaseci/VERSION,sha256=dg0mtsTYEtoBjdH8cy9UJ_askmcWcS75Jlu9IOKgct4,9
+jaseci/VERSION,sha256=wt2cDwcEID1HJJqP4EFUnrYbbWFI95p3NNCVHoBS-L4,9
 jaseci/__init__.py,sha256=qkJ3-ufZPGervTKYsKVNxdKvdMbrCmae8pgFAcWk5Pg,1101
 jaseci/cli_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaseci/cli_tools/book_tools.py,sha256=4xeNRhKufKJgAaKI5miPTezdBDWeA-iB_H5F5cqyIB8,16103
@@ -22,7 +22,7 @@ jaseci/extens/act_lib/stripe.py,sha256=Aal_0jmt8nGSWM4PFiKsSAXPN779d51nV-tnFr6_p
 jaseci/extens/act_lib/task.py,sha256=2OlsgYDprGyDwEzJ7943piqrqo7soCDBZRpqgGVMhUk,355
 jaseci/extens/act_lib/url.py,sha256=SC0fgrCA28UFznEAA_vOLnB51_N0MXU0801V_MVhDQg,1853
 jaseci/extens/act_lib/vector.py,sha256=ekTtYx3sHP6j23m1otVeKfqWynfLNsGg6271IIvfE-I,4636
-jaseci/extens/act_lib/webtool.py,sha256=YYZSaBWLUg646OX88_ZeB52AlfkFBsShk4PSA6F5Hv8,634
+jaseci/extens/act_lib/webtool.py,sha256=1gJJJ8r86YQpRmSAY6_L6JUwLGl8ZcGWKtydSRemDnY,1038
 jaseci/extens/act_lib/zip.py,sha256=re6jUmkFZaOsrCpum8BMnLFltavrclv-rjfvhbl67i4,942
 jaseci/extens/act_lib/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaseci/extens/act_lib/tests/std_test_code.py,sha256=_pXoqkD4_s2LQHQBs_FFphYseJ5mCaPSe_LDvLW6p_Y,782
@@ -35,7 +35,7 @@ jaseci/extens/act_lib/tests/test_std.py,sha256=wSNganrnB7XVNYKBD8n2emXp1jrJJ5IiN
 jaseci/extens/act_lib/tests/test_std_lib.py,sha256=3RO3Q0y32A_D3bAokYRWSE8iHeyD2VRie8rmEpyZ9kM,1204
 jaseci/extens/act_lib/tests/test_url.py,sha256=TfCWRtSrB-wjDOxRpfT9yjkSVRW6Ro4_IynbPir00XM,904
 jaseci/extens/act_lib/tests/test_vector.py,sha256=Lte-EZnheAGiJUpCE0mlXzhCH3dTMW000pgYbiLeHtY,1285
-jaseci/extens/act_lib/tests/test_webtool.py,sha256=RbtT8wSR7p-07jxAuENmZSJqZV0YJ0UGg1Vmzg-eJzM,928
+jaseci/extens/act_lib/tests/test_webtool.py,sha256=BfRQOdCUk3ciZj_wP6yzxVevcieqH5fQ6NhnbhllBH8,1529
 jaseci/extens/act_lib/tests/test_zlib.py,sha256=iG5FRkUsUDfJWKEhrgAJ21HWNZRfiWxGrctsZXpSxPw,840
 jaseci/extens/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaseci/extens/api/actions_api.py,sha256=Ml2wRROmXofEXWTOXy0PQCxRcjDLkxz163p7Z13zfKs,6923
@@ -159,14 +159,14 @@ jaseci/utils/id_list.py,sha256=L1uzwBrLRxnw3EaapOsEMA7QCsbY50RnK0nFTu2ruWo,5328
 jaseci/utils/json_handler.py,sha256=P1fY7BnGBfA_mkiKjeJvAwfqGXWt3cpY0JcmPVL86bQ,2135
 jaseci/utils/log_utils.py,sha256=U46KspmCbX6S7YzEk1LB20iB4G1DPHfpV5RFim04Mbg,1629
 jaseci/utils/test_core.py,sha256=DG0mW2GRc2l0M1A1-vCgTUFnZh8ip_0-h7eOfKMxlXQ,1549
-jaseci/utils/utils.py,sha256=-BBpaeZxVH3LgOHMrCO8R6S65Bm67FDoCLZr3aK39i8,8761
+jaseci/utils/utils.py,sha256=iGK-Ovf5u9Y5cQUc95GJmyGpQQa1cnF7QOCseCf42EQ,9101
 jaseci/utils/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaseci/utils/actions/actions_manager.py,sha256=sKfWk9zRpDpkLo3-mKpVMKrqx7kpRG7hRi5tfDTcqb8,8527
 jaseci/utils/actions/actions_optimizer.py,sha256=gs2spE-HFzdfW8WNqctw9DBldUxv3vj3DPpU5txZllE,20786
 jaseci/utils/actions/actions_state.py,sha256=5p080-DtDXS9QUd2FwkrtN6-7LBqT2tmJDOKVKjDEbU,2963
-jaseci-1.4.0.16.dist-info/LICENSE,sha256=2b_qOOd7ak6x49HwOzCAmRLSPJehiBJdqj9HDX1Jp7U,1072
-jaseci-1.4.0.16.dist-info/METADATA,sha256=JauKjHm-pvZgNW9fykwhejRHEYgbx2S0QF6TvgO26lY,1022
-jaseci-1.4.0.16.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-jaseci-1.4.0.16.dist-info/entry_points.txt,sha256=nryZyKQLUJzlTOcA9knRGoXHGmNUCQxvvuDIEj5ePyg,88
-jaseci-1.4.0.16.dist-info/top_level.txt,sha256=0WZh7RF_ruiaZHQSi8IWOiLRUvKKDlhcnLe-by6EyFc,7
-jaseci-1.4.0.16.dist-info/RECORD,,
+jaseci-1.4.0.18.dist-info/LICENSE,sha256=2b_qOOd7ak6x49HwOzCAmRLSPJehiBJdqj9HDX1Jp7U,1072
+jaseci-1.4.0.18.dist-info/METADATA,sha256=wh0bSDOWuDVz46_MXv15eNHZM0NUbXw0VByq0TRUGO8,1059
+jaseci-1.4.0.18.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+jaseci-1.4.0.18.dist-info/entry_points.txt,sha256=nryZyKQLUJzlTOcA9knRGoXHGmNUCQxvvuDIEj5ePyg,88
+jaseci-1.4.0.18.dist-info/top_level.txt,sha256=0WZh7RF_ruiaZHQSi8IWOiLRUvKKDlhcnLe-by6EyFc,7
+jaseci-1.4.0.18.dist-info/RECORD,,