parsagon 0.9.8__tar.gz → 0.10.2__tar.gz

{parsagon-0.9.8 → parsagon-0.10.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: parsagon
-Version: 0.9.8
+Version: 0.10.2
 Summary: Allows you to create browser automations with natural language
 Author-email: Sandy Suh <sandy@parsagon.io>
 Project-URL: Homepage, https://parsagon.io
@@ -8,7 +8,22 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
+Requires-Dist: selenium==4.9.1
+Requires-Dist: lxml==4.9.2
+Requires-Dist: httpx==0.24.1
+Requires-Dist: halo==0.0.31
+Requires-Dist: pandas==1.4.2
+Requires-Dist: PyVirtualDisplay==3.0
+Requires-Dist: selenium-wire==5.1.0
+Requires-Dist: cssselect==1.1.0
+Requires-Dist: undetected-chromedriver==3.5.2
+Requires-Dist: webdriver-manager==4.0.0
+Requires-Dist: jsonpath-ng==1.5.3
+Requires-Dist: usaddress==0.5.10
+Requires-Dist: simplejson==3.19.1
 Provides-Extra: dev
+Requires-Dist: pytest==7.3.2; extra == "dev"
+Requires-Dist: pytest-mock==3.11.1; extra == "dev"
 
 # parsagon
 

{parsagon-0.9.8 → parsagon-0.10.2}/pyproject.toml

@@ -16,7 +16,7 @@ line-length = 120
 
 [project]
 name = "parsagon"
-version = "0.9.8"
+version = "0.10.2"
 description = "Allows you to create browser automations with natural language"
 readme = "README.md"
 requires-python = ">=3.8"

{parsagon-0.9.8 → parsagon-0.10.2}/src/parsagon/__init__.py

@@ -1 +1 @@
-from parsagon.main import create, update, detail, run, delete, get_product
+from parsagon.main import create, update, detail, run, delete, get_product, get_review_article, get_article_list

{parsagon-0.9.8 → parsagon-0.10.2}/src/parsagon/api.py

@@ -58,7 +58,7 @@ def get_program_sketches(description):
     """
     Gets a program sketches (full and abridged) from a description.
     :param description: Description in natural language that will be used to generate the scraping program.
-    :return: A dict with keys "full" and "abridged" for the respective program ASTs.
+    :return: A dict with keys "full", "abridged", and "pseudocode" for the respective program ASTs and pseudocode.
     """
     return _api_call(httpx.post, "/transformers/get-program-sketch/", json={"description": description})
 
@@ -131,9 +131,9 @@ def get_bool_about_data(data, question):
     return data["result"]
 
 
-def create_pipeline(name, description, program_sketch):
+def create_pipeline(name, description, program_sketch, pseudocode):
     return _api_call(
-        httpx.post, "/pipelines/", json={"name": name, "description": description, "program_sketch": program_sketch}
+        httpx.post, "/pipelines/", json={"name": name, "description": description, "program_sketch": program_sketch, "pseudocode": pseudocode}
     )
 
 
@@ -199,5 +199,5 @@ def get_run(run_id):
     )
 
 
-def poll_product(url):
-    return _api_call(httpx.post, "/extract/product/", json={"url": url, "page_type": "PRODUCT_DETAIL"})
+def poll_data(url, page_type):
+    return _api_call(httpx.post, "/extract/", json={"url": url, "page_type": page_type})

{parsagon-0.9.8 → parsagon-0.10.2}/src/parsagon/main.py

@@ -17,7 +17,7 @@ from parsagon.api import (
     get_pipelines,
     get_pipeline_code,
     get_run,
-    poll_product,
+    poll_data,
     APIException,
 )
 from parsagon.exceptions import ParsagonException
@@ -190,10 +190,11 @@ def create(task=None, program_name=None, headless=False, infer=False, verbose=Fa
 
     logger.info("Analyzing task description...")
     program_sketches = get_program_sketches(task)
-    logger.info("Created a program based on task description. Now demonstrating what the program does:\n")
 
     full_program = program_sketches["full"]
     abridged_program = program_sketches["abridged"]
+    pseudocode = program_sketches["pseudocode"]
+    logger.info(f"Created a program based on task description. Program does the following:\n\n{pseudocode}\n\nNow executing the program to identify web elements to be scraped:\n")
     logger.debug("Program:\n%s", abridged_program)
     abridged_program += "\n\noutput = func()\nprint(f'Program finished and returned a value of:\\n{output}\\n')\n"  # Make the program runnable
 
@@ -208,7 +209,7 @@ def create(task=None, program_name=None, headless=False, infer=False, verbose=Fa
         if program_name:
             logger.info(f"Saving program as {program_name}")
             try:
-                pipeline = create_pipeline(program_name, task, full_program)
+                pipeline = create_pipeline(program_name, task, full_program, pseudocode)
             except APIException as e:
                 if isinstance(e.value, list) and "Program with name already exists" in e.value:
                     logger.info("A program with this name already exists. Please choose another name.")
@@ -354,13 +355,25 @@ def setup(verbose=False):
     logger.info("Setup complete.")
 
 
-def get_product(url, timeout=300):
+def _get_data(url, page_type, timeout):
     start_time = time.time()
     with Halo(text="Extracting data...", spinner="dots"):
         while time.time() - start_time <= timeout:
-            result = poll_product(url)
+            result = poll_data(url, page_type)
             if result["done"]:
                 return result["result"]
-            time.sleep(5)
+            time.sleep(15)
     logger.info("No data found")
     return None
+
+
+def get_product(url, timeout=300):
+    return _get_data(url, "PRODUCT_DETAIL", timeout)
+
+
+def get_review_article(url, timeout=300):
+    return _get_data(url, "REVIEW_ARTICLE_DETAIL", timeout)
+
+
+def get_article_list(url, timeout=300):
+    return _get_data(url, "ARTICLE_LIST", timeout)

{parsagon-0.9.8 → parsagon-0.10.2}/src/parsagon.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: parsagon
-Version: 0.9.8
+Version: 0.10.2
 Summary: Allows you to create browser automations with natural language
 Author-email: Sandy Suh <sandy@parsagon.io>
 Project-URL: Homepage, https://parsagon.io
@@ -8,7 +8,22 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
+Requires-Dist: selenium==4.9.1
+Requires-Dist: lxml==4.9.2
+Requires-Dist: httpx==0.24.1
+Requires-Dist: halo==0.0.31
+Requires-Dist: pandas==1.4.2
+Requires-Dist: PyVirtualDisplay==3.0
+Requires-Dist: selenium-wire==5.1.0
+Requires-Dist: cssselect==1.1.0
+Requires-Dist: undetected-chromedriver==3.5.2
+Requires-Dist: webdriver-manager==4.0.0
+Requires-Dist: jsonpath-ng==1.5.3
+Requires-Dist: usaddress==0.5.10
+Requires-Dist: simplejson==3.19.1
 Provides-Extra: dev
+Requires-Dist: pytest==7.3.2; extra == "dev"
+Requires-Dist: pytest-mock==3.11.1; extra == "dev"
 
 # parsagon