parsagon 0.13.0__tar.gz → 0.14.0__tar.gz

{parsagon-0.13.0 → parsagon-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: parsagon
-Version: 0.13.0
+Version: 0.14.0
 Summary: Allows you to create browser automations with natural language
 Author-email: Sandy Suh <sandy@parsagon.io>
 Project-URL: Homepage, https://parsagon.io

{parsagon-0.13.0 → parsagon-0.14.0}/pyproject.toml

@@ -16,7 +16,7 @@ line-length = 120
 
 [project]
 name = "parsagon"
-version = "0.13.0"
+version = "0.14.0"
 description = "Allows you to create browser automations with natural language"
 readme = "README.md"
 requires-python = ">=3.8"

{parsagon-0.13.0 → parsagon-0.14.0}/src/parsagon/api.py

@@ -1,5 +1,6 @@
 from json import JSONDecodeError
 import time
+from urllib.parse import quote
 
 import httpx
 
@@ -166,7 +167,13 @@ def create_pipeline(name, description, program_sketch, pseudocode, secrets):
     return _api_call(
         httpx.post,
         "/pipelines/",
-        json={"name": name, "description": description, "program_sketch": program_sketch, "pseudocode": pseudocode, "secrets": secrets},
+        json={
+            "name": name,
+            "description": description,
+            "program_sketch": program_sketch,
+            "pseudocode": pseudocode,
+            "secrets": secrets,
+        },
     )
 
 
@@ -195,11 +202,21 @@ def add_examples_to_custom_function(pipeline_id, call_id, custom_function, remov
     )
 
 
+def double_quote(string):
+    """
+    A bug in Django disallows URLs even with quoted slashes as in:
+    /api/pipelines/name/stripe%2Fstuff/code/ HTTP/1.1" 405
+    Therefore we must double quote
+    """
+    return quote(quote(string, safe=""), safe="")
+
+
 def get_pipeline(pipeline_name):
+    escaped_pipeline_name = double_quote(pipeline_name)
     with RaiseProgramNotFound(pipeline_name):
         return _api_call(
             httpx.get,
-            f"/pipelines/name/{pipeline_name}/",
+            f"/pipelines/name/{escaped_pipeline_name}/?double_escaped=True",
         )
 
 
@@ -207,14 +224,16 @@ def get_pipelines():
     return _api_call(httpx.get, f"/pipelines/")
 
 
-def get_pipeline_code(pipeline_name, variables, headless):
+def get_pipeline_code(pipeline_name, variables, headless, use_uc):
+    escaped_pipeline_name = double_quote(pipeline_name)
     with RaiseProgramNotFound(pipeline_name):
         return _api_call(
             httpx.post,
-            f"/pipelines/name/{pipeline_name}/code/",
+            f"/pipelines/name/{escaped_pipeline_name}/code/?double_escaped=True",
             json={
                 "variables": variables,
                 "headless": headless,
+                "use_uc": use_uc,
             },
         )
 
@@ -246,11 +265,17 @@ def get_run(run_id):
 
 
 def send_assistant_message(message, thread_id=None):
-    return _api_call(httpx.post, "/transformers/send-assistant-message/", json={"message": message, "thread_id": thread_id})
+    return _api_call(
+        httpx.post, "/transformers/send-assistant-message/", json={"message": message, "thread_id": thread_id}
+    )
 
 
 def send_assistant_function_outputs(outputs, thread_id, run_id):
-    return _api_call(httpx.post, "/transformers/send-assistant-function-outputs/", json={"outputs": outputs, "thread_id": thread_id, "run_id": run_id})
+    return _api_call(
+        httpx.post,
+        "/transformers/send-assistant-function-outputs/",
+        json={"outputs": outputs, "thread_id": thread_id, "run_id": run_id},
+    )
 
 
 def poll_extract(url, page_type):

{parsagon-0.13.0 → parsagon-0.14.0}/src/parsagon/assistant.py

@@ -4,10 +4,10 @@ from parsagon.create import create_program
 from parsagon.executor import Executor
 from parsagon.print import assistant_print, assistant_spinner, browser_print, error_print
 from rich.prompt import Prompt
-from parsagon.runs import run, batch_runs, run_with_file_output
+from parsagon.runs import run, batch_runs
 
 
-def assist(headless=False, infer=False, verbose=False):
+def assist(verbose=False):
     task = Prompt.ask("Type what do you want to do")
     with assistant_spinner():
         response = send_assistant_message(task)
@@ -37,11 +37,11 @@ def assist(headless=False, infer=False, verbose=False):
                     output["output"] = html
                     outputs.append(output)
                 elif name == "create_program":
-                    result = create_program(args["description"], headless=headless, infer=infer)
+                    result = create_program(args["description"])
                     output["output"] = json.dumps(result)
                     outputs.append(output)
                 elif name == "run_program":
-                    result = run_with_file_output(**args)
+                    result = run(**args)
                     output["output"] = json.dumps(result)
                     outputs.append(output)
                 elif name == "batch_runs":

{parsagon-0.13.0 → parsagon-0.14.0}/src/parsagon/create.py

@@ -6,7 +6,7 @@ from parsagon.secrets import extract_secrets
 from rich.prompt import Prompt
 
 
-def create_program(task, headless=False, infer=False):
+def create_program(task, headless=False, infer=False, undetected=False):
     assistant_print("Creating a program based on your specifications...")
     task, secrets = extract_secrets(task)
     program_sketches = get_program_sketches(task)
@@ -28,7 +28,7 @@ def create_program(task, headless=False, infer=False):
     abridged_program += f"\n\noutput = func({args})\n"  # Make the program runnable
 
     # Execute the abridged program to gather examples
-    executor = Executor(headless=headless, infer=infer)
+    executor = Executor(headless=headless, infer=infer, use_uc=undetected)
     executor.execute(abridged_program)
 
     # The user must select a name

{parsagon-0.13.0 → parsagon-0.14.0}/src/parsagon/executor.py

@@ -54,6 +54,7 @@ ELEMENT_TYPES = {
     "html": "HTML",
     "element": "ACTION",
     "textarea": "TEXT",
+    "text_markdown": "TEXT",
     "markdown": "TEXT",
     "elem_id": "ACTION",
 }

{parsagon-0.13.0 → parsagon-0.14.0}/src/parsagon/main.py

@@ -17,28 +17,18 @@ from parsagon.assistant import assist
 from parsagon.create import create_program
 from parsagon.exceptions import ParsagonException
 from parsagon.executor import Executor, custom_functions_to_descriptions
-from parsagon.runs import run_with_file_output
+from parsagon.runs import run
 from parsagon.settings import get_api_key, save_setting, configure_logging
 
 console = Console()
 logger = logging.getLogger(__name__)
 
 
-def get_args():
+def get_args(argv):
     parser = argparse.ArgumentParser(
         prog="parsagon", description="Scrapes and interacts with web pages based on natural language.", add_help=False
     )
     parser.add_argument("-v", "--verbose", action="store_true", help="run the task in verbose mode")
-    parser.add_argument(
-        "--headless",
-        action="store_true",
-        help="run the browser in headless mode",
-    )
-    parser.add_argument(
-        "--infer",
-        action="store_true",
-        help="let Parsagon infer all elements to be scraped",
-    )
     subparsers = parser.add_subparsers()
 
     # Create
@@ -53,6 +43,11 @@ def get_args():
         action="store_true",
         help="let Parsagon infer all elements to be scraped",
     )
+    parser_create.add_argument(
+        "--undetected",
+        action="store_true",
+        help="run in undetected mode",
+    )
     parser_create.set_defaults(func=create_cli)
 
     # Detail
@@ -133,7 +128,17 @@ def get_args():
         action="store_true",
         help="output log data from the run",
     )
-    parser_run.set_defaults(func=run_with_file_output)
+    parser_run.add_argument(
+        "--output_file",
+        type=str,
+        help="write the data to the given file path",
+    )
+    parser_run.add_argument(
+        "--undetected",
+        action="store_true",
+        help="run in undetected mode",
+    )
+    parser_run.set_defaults(func=run)
 
     # Delete
     parser_delete = subparsers.add_parser(
@@ -164,20 +169,16 @@ def get_args():
     )
     parser_help.set_defaults(func=help, parser=parser)
 
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
     kwargs = vars(args)
     return kwargs, parser
 
 
-def main():
-    kwargs, parser = get_args()
+def main(argv=None):
+    kwargs, parser = get_args(argv)
     func = kwargs.pop("func", None)
     if func is None:
         func = assist
-    else:
-        # Pop assist-only arguments
-        kwargs.pop("infer")
-        kwargs.pop("headless")
     verbose = kwargs["verbose"]
     configure_logging(verbose)
 
@@ -188,9 +189,9 @@ def main():
         logger.error(error_message)
 
 
-def create_cli(headless=False, infer=False, verbose=False):
+def create_cli(headless=False, infer=False, undetected=False, verbose=False):
     task = Prompt.ask("Enter a detailed scraping task")
-    create_program(task, headless=headless, infer=infer)
+    create_program(task, headless=headless, infer=infer, undetected=undetected)
 
 
 def update(program_name, variables={}, headless=False, infer=False, replace=False, verbose=False):

{parsagon-0.13.0 → parsagon-0.14.0}/src/parsagon/runs.py

@@ -1,5 +1,4 @@
 import datetime
-import datetime
 import json
 import logging.config
 import time
@@ -24,18 +23,7 @@ console = Console()
 logger = logging.getLogger(__name__)
 
 
-def run_with_file_output(*args, **kwargs):
-    dump_path = Prompt.ask("Please enter a path/filename to save the output (in JSON format)")
-    if not dump_path.endswith(".json"):
-        dump_path += ".json"
-    result = run(*args, **kwargs)
-    with open(dump_path, "w") as f:
-        json.dump(result, f, indent=4)
-    print(f"Output saved to {dump_path}")
-    return result
-
-
-def run(program_name, variables={}, headless=False, remote=False, output_log=False, verbose=False):
+def run(program_name, variables={}, headless=False, remote=False, output_log=False, output_file=None, undetected=False, verbose=False):
     """
     Executes pipeline code
     """
@@ -56,7 +44,13 @@ def run(program_name, variables={}, headless=False, remote=False, output_log=Fal
                 status = run["status"]
 
                 if output_log and status in ("FINISHED", "ERROR"):
-                    return {k: v for k, v in run.items() if k in ("output", "status", "log", "warnings", "error")}
+                    result = {k: v for k, v in run.items() if k in ("output", "status", "log", "warnings", "error")}
+                    if output_file:
+                        with open(output_file, "w") as f:
+                            json.dump(result, f, indent=4)
+                        return
+                    else:
+                        return result
 
                 if status == "FINISHED":
                     if verbose:
@@ -64,7 +58,13 @@ def run(program_name, variables={}, headless=False, remote=False, output_log=Fal
                         for warning in run["warnings"]:
                             logger.warning(warning)
                     logger.info("Program finished running.")
-                    return run["output"]
+                    result = run["output"]
+                    if output_file:
+                        with open(output_file, "w") as f:
+                            json.dump(result, f, indent=4)
+                        return
+                    else:
+                        return result
                 elif status == "ERROR":
                     raise ParsagonException(f"Program failed to run: {run['error']}")
                 elif status == "CANCELED":
@@ -73,7 +73,7 @@ def run(program_name, variables={}, headless=False, remote=False, output_log=Fal
                 time.sleep(5)
 
     run = create_pipeline_run(pipeline_id, variables, True)
-    code = get_pipeline_code(program_name, variables, headless)["code"]
+    code = get_pipeline_code(program_name, variables, headless, undetected)["code"]
     start_time = datetime.datetime.now(datetime.timezone.utc).isoformat()
     run_data = {"start_time": start_time}
 
@@ -107,11 +107,18 @@ def run(program_name, variables={}, headless=False, remote=False, output_log=Fal
                 continue
         run = update_pipeline_run(run["id"], run_data)
     logger.info("Done.")
+
+    result = globals_locals["output"]
     if output_log:
         if "error" not in run_data:
             run["output"] = globals_locals["output"]
-        return {k: v for k, v in run.items() if k in ("output", "status", "log", "warnings", "error")}
-    return globals_locals["output"]
+        result = {k: v for k, v in run.items() if k in ("output", "status", "log", "warnings", "error")}
+    if output_file:
+        with open(output_file, "w") as f:
+            json.dump(result, f, indent=4)
+        return
+    else:
+        return result
 
 
 def batch_runs(

{parsagon-0.13.0 → parsagon-0.14.0}/src/parsagon/tests/api_mocks.py

@@ -53,7 +53,7 @@ def mock_httpx_method_func(*args, **kwargs):
             },
         )
 
-    if match := re.search(r"/pipelines/name/(.+)/$", url):
+    if match := re.search(r"/pipelines/name/(.+)/", url):
         assert method == "get"
         pipeline_name = match.group(1)
         if pipeline_name == not_found_pipeline_name:

{parsagon-0.13.0 → parsagon-0.14.0}/src/parsagon/tests/test_invalid_args.py

@@ -1,23 +1,13 @@
 import pytest
 
-from parsagon import run
-from parsagon.tests.cli_mocks import call_cli
+from parsagon.main import main
 
 
 def test_headless_remote_run_invalid(mocker, debug_logs):
     """
     Tests that we are unable to run a program in headless mode when the environment is remote, and that this is logged to the user.
     """
-    call_cli(
-        mocker,
-        {
-            "func": run,
-            "program_name": "test_program",
-            "headless": True,
-            "remote": True,
-            "verbose": False,
-        },
-    )
+    main(["run", "test_program", "--headless", "--remote"])
     debug_logs_lower = debug_logs.text.lower()
     assert "error" in debug_logs_lower
     assert "headless" in debug_logs_lower

{parsagon-0.13.0 → parsagon-0.14.0}/src/parsagon/tests/test_pipeline_operations.py

@@ -6,7 +6,6 @@ import pytest
 from parsagon import delete, run
 from parsagon.main import main
 from parsagon.tests.api_mocks import install_api_mocks, not_found_pipeline_name
-from parsagon.tests.cli_mocks import call_cli
 
 
 def test_pipeline_delete(mocker):
@@ -18,24 +17,10 @@ def test_pipeline_not_found(mocker, debug_logs):
     install_api_mocks(mocker, {"code_to_return": 'raise Exception("Should not exec this code if pipeline not found.")'})
 
     # On delete
-    call_cli(
-        mocker,
-        {
-            "func": delete,
-            "program_name": not_found_pipeline_name,
-            "verbose": False,
-        },
-    )
+    main(["delete", not_found_pipeline_name, "-y"])
     assert f"A program with name {not_found_pipeline_name} does not exist." in debug_logs.text
     debug_logs.clear()
 
     # On attempted run
-    call_cli(
-        mocker,
-        {
-            "func": run,
-            "program_name": not_found_pipeline_name,
-            "verbose": False,
-        },
-    )
+    main(["run", not_found_pipeline_name])
     assert f"A program with name {not_found_pipeline_name} does not exist." in debug_logs.text

{parsagon-0.13.0 → parsagon-0.14.0}/src/parsagon.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: parsagon
-Version: 0.13.0
+Version: 0.14.0
 Summary: Allows you to create browser automations with natural language
 Author-email: Sandy Suh <sandy@parsagon.io>
 Project-URL: Homepage, https://parsagon.io

{parsagon-0.13.0 → parsagon-0.14.0}/src/parsagon.egg-info/SOURCES.txt

@@ -22,7 +22,6 @@ src/parsagon.egg-info/requires.txt
 src/parsagon.egg-info/top_level.txt
 src/parsagon/tests/__init__.py
 src/parsagon/tests/api_mocks.py
-src/parsagon/tests/cli_mocks.py
 src/parsagon/tests/conftest.py
 src/parsagon/tests/test_executor.py
 src/parsagon/tests/test_invalid_args.py

parsagon-0.13.0/src/parsagon/tests/cli_mocks.py

@@ -1,16 +0,0 @@
-from parsagon.main import main
-
-
-def call_cli(mocker, args):
-    """
-    Uses the mocker to pretend that the args passed are coming from argparse, then calls the main function.
-    """
-
-    mocker.patch(
-        "parsagon.main.get_args",
-        lambda: (
-            args,
-            None,
-        ),
-    )
-    return main()