stoobly-agent 0.34.12__tar.gz → 1.0.7__tar.gz

{stoobly_agent-0.34.12 → stoobly_agent-1.0.7}/PKG-INFO

@@ -1,27 +1,28 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: stoobly-agent
-Version: 0.34.12
+Version: 1.0.7
 Summary: Record, mock, and test HTTP(s) requests. CLI agent for Stoobly
 License: Apache-2.0
 Author: Matt Le
 Author-email: themathewle@gmail.com
-Requires-Python: >=3.8,<4.0
+Requires-Python: >=3.10,<4.0
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: click (>=7.0.0,<8.0.0)
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: click (>=8.1.0,<9.0.0)
 Requires-Dist: diff-match-patch (>=20230430,<20230431)
 Requires-Dist: distro (>=1.6.0,<1.7.0)
+Requires-Dist: dnspython (>=2.6.1,<2.7.0)
 Requires-Dist: httptools (>=0.4.0)
 Requires-Dist: jmespath (>=1.0.0)
 Requires-Dist: mergedeep (>=1.3.0,<1.3.4)
-Requires-Dist: mitmproxy (>=8.0.0,<=8.1.0)
+Requires-Dist: mitmproxy (>=10.0.0,<11.0.0)
 Requires-Dist: multipart (>=0.2.5,<0.3.0)
-Requires-Dist: openapi-core (>=0.17.0,<0.18.0)
+Requires-Dist: openapi-core (>=0.19.0,<0.20.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: pyyaml (>=6.0.1)
 Requires-Dist: requests (>=2.31.0)
 Requires-Dist: stoobly_orator (>=0.9.12)
@@ -55,7 +56,7 @@ See our docs for more detailed information! https://docs.stoobly.com
 
 ## Prerequisite
 
-- Python >= 3.8, < 3.11
+- Python 3.10, 3.11, 3.12
 
 ## Installation
 

{stoobly_agent-0.34.12 → stoobly_agent-1.0.7}/README.md

@@ -21,7 +21,7 @@ See our docs for more detailed information! https://docs.stoobly.com
 
 ## Prerequisite
 
-- Python >= 3.8, < 3.11
+- Python 3.10, 3.11, 3.12
 
 ## Installation
 

{stoobly_agent-0.34.12 → stoobly_agent-1.0.7}/pyproject.toml

@@ -10,19 +10,22 @@ description = "Record, mock, and test HTTP(s) requests. CLI agent for Stoobly"
 license = "Apache License 2.0"
 name = "stoobly-agent"
 readme = "README.md"
-version = "0.34.12"
+version = "1.0.7"
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.10"
 
-click = ">=7.0.0,<8.0.0"
+click = "^8.1.0"
 diff-match-patch = "^20230430"
 distro = ">=1.6.0,<1.7.0"
+dnspython = ">=2.6.1,<2.7.0"
 httptools = ">=0.4.0"
 jmespath = ">=1.0.0"
 mergedeep = ">=1.3.0,<1.3.4"
-mitmproxy = ">=8.0.0,<=8.1.0"
-openapi-core = ">=0.17.0,<0.18.0"
+mitmproxy = ">=10.0.0,<11.0.0"
+openapi-core = ">=0.19.0,<0.20.0"
+multipart = "^0.2.5"
+python-dotenv = "^1.0.1"
 pyyaml = ">=6.0.1"
 requests = ">=2.31.0"
 stoobly_orator = ">=0.9.12"
@@ -30,7 +33,6 @@ tabulate = ">=0.8.8"
 watchdog = ">=2.1.0,<=2.1.3"
 werkzeug = "<3.0.0" # Dependency resolution
 yamale = ">=4.0.0,<=4.0.2"
-multipart = "^0.2.5"
 
 [tool.poetry.group.test.dependencies]
 mock = ">=5.0.0"

{stoobly_agent-0.34.12 → stoobly_agent-1.0.7}/stoobly_agent/__init__.py

@@ -1,2 +1,2 @@
 COMMAND = 'stoobly-agent'
-VERSION = '0.34.12'
+VERSION = '1.0.7'

{stoobly_agent-0.34.12 → stoobly_agent-1.0.7}/stoobly_agent/app/api/__init__.py

@@ -3,28 +3,36 @@ import pdb
 import threading
 
 from http.server import HTTPServer
+from urllib.parse import urlparse
 
 from stoobly_agent.app.settings import Settings
 from stoobly_agent.config.constants import env_vars
+from stoobly_agent.lib.logger import Logger
 
 from .application_http_request_handler import ApplicationHTTPRequestHandler
 
+LOG_ID = 'UI'
+
 def start_server(host, port):
     httpd = HTTPServer((host, port), ApplicationHTTPRequestHandler)
     httpd.serve_forever()
 
-def run(**kwargs):
-    ui_host = kwargs['ui_host']
-    ui_port = kwargs['ui_port']
+def initialize(**kwargs):
     url = f"http://{kwargs['ui_host']}:{kwargs['ui_port']}"
 
     os.environ[env_vars.AGENT_URL] = url
 
+    # TODO: debug why the following is needed or else logger won't print
     settings = Settings.instance()
     settings.ui.url = url
     settings.commit()
 
-    print(f"UI server listening at http://{ui_host}:{ui_port}\n")
+    return url
+
+def run(url):
+    parsed_url = urlparse(url)
+
+    Logger.instance(LOG_ID).info(f"Server listening at {url}\n")
 
-    thread = threading.Thread(target=start_server, args=(ui_host, ui_port))
+    thread = threading.Thread(target=start_server, args=(parsed_url.hostname, parsed_url.port))
     thread.start()

{stoobly_agent-0.34.12 → stoobly_agent-1.0.7}/stoobly_agent/app/cli/__init__.py

@@ -8,6 +8,7 @@ from .main_group import MainGroup
 from .project_cli import project
 from .report_cli import report
 from .request_cli import request
+from .scaffold_cli import scaffold
 from .scenario_cli import scenario
 from .snapshot_cli import snapshot
 from .trace_cli import trace

stoobly_agent-1.0.7/stoobly_agent/app/cli/ca_cert_cli.py

@@ -0,0 +1,46 @@
+import click
+import pdb
+import sys
+
+from stoobly_agent.config.data_dir import DataDir
+
+from .helpers.certificate_authority import CertificateAuthority
+
+@click.group(
+    epilog="Run 'stoobly-agent COMMAND --help' for more information on a command.",
+    help="Manage CA certificate"
+)
+@click.pass_context
+def ca_cert(ctx):
+    pass
+
+@ca_cert.command()
+@click.option('--ca-certs-dir-path', default=DataDir.instance().mitmproxy_conf_dir_path, help='Path to ca certs directory used to sign SSL certs. Defaults to ~/.mitmproxy')
+@click.option('--certs-dir-path', default=DataDir.instance().certs_dir_path, help='Output directory.')
+@click.argument('hostname')
+def mkcert(**kwargs):
+    mitmproxy_ca_certs_dir = kwargs['ca_certs_dir_path']
+
+    installer = CertificateAuthority(mitmproxy_ca_certs_dir)
+
+    try:
+        installer.sign(kwargs['hostname'], kwargs['certs_dir_path'])
+    except Exception as e:
+        print(e, file=sys.stderr)
+        sys.exit(1)
+
+@ca_cert.command()
+@click.option('--ca-certs-dir-path', default=DataDir.instance().mitmproxy_conf_dir_path, help='Path to ca certs directory.')
+def install(**kwargs):
+    mitmproxy_ca_certs_dir = kwargs['ca_certs_dir_path']
+    installer = CertificateAuthority(mitmproxy_ca_certs_dir)
+
+    try:
+        installer.install()
+    except Exception as e:
+        print(e, file=sys.stderr)
+        sys.exit(1)
+
+@ca_cert.command()
+def uninstall():
+    print("Not yet implemented. Stay tuned!")

stoobly_agent-1.0.7/stoobly_agent/app/cli/helpers/certificate_authority.py

@@ -0,0 +1,201 @@
+import distro
+import os
+import pdb
+import socket
+import ssl
+import subprocess
+
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from cryptography.x509.oid import NameOID
+from mitmproxy.certs import CertStore
+from pathlib import Path
+
+from stoobly_agent.config.data_dir import DataDir
+from stoobly_agent.lib.logger import Logger
+
+LOG_ID = 'CertificateAuthority'
+
+MITMPROXY_CN = 'mitmproxy'
+
+class CertificateAuthority():
+
+    def __init__(self, certs_dir = DataDir.instance().mitmproxy_conf_dir_path, cn = MITMPROXY_CN):
+        self.certs_dir = certs_dir
+        self.cn = cn
+        self.key_size = 2048
+
+    @property
+    def certs_generated(self):
+        if not os.path.exists(self.ca_cert_path('.pem')):
+            return False
+
+        if not os.path.exists(self.ca_cert_path('.cer')):
+            return False
+
+        if not os.path.exists(self.ca_path('.pem')):
+            return False
+
+        return True
+
+    def ca_file_name(self, extension: str):
+        return '{cn}-ca{extension}'.format(cn=self.cn, extension=extension)
+
+    def ca_path(self, extension: str):
+        file_name = self.ca_file_name(extension)
+        return os.path.join(self.certs_dir, file_name)
+
+    def ca_cert_file_name(self, extension: str):
+        return '{cn}-ca-cert{extension}'.format(cn=self.cn, extension=extension)
+
+    def ca_cert_path(self, extension: str):
+        file_name = self.ca_cert_file_name(extension)
+        return os.path.join(self.certs_dir, file_name)
+
+    def generate_certs(self):
+        if not os.path.exists(self.certs_dir):
+            os.makedirs(self.certs_dir)
+
+        path = Path(self.certs_dir)
+        CertStore.create_store(path, self.cn, self.key_size)
+
+    # https://askubuntu.com/a/94861
+    def handle_debian(self):
+        extra_ca_certs_dir = '/usr/local/share/ca-certificates/extra'
+
+        ca_cert_path = self.ca_cert_path('.cer')
+        self.__ensure_exists(ca_cert_path)
+
+        extra_crt_absolute_path = os.path.join(extra_ca_certs_dir, self.ca_cert_file_name('.crt'))
+
+        subprocess.run(f"sudo mkdir -p {extra_ca_certs_dir}".split(), check=True)
+        subprocess.run(f"sudo cp {ca_cert_path} {extra_crt_absolute_path}".split(), check=True)
+        subprocess.run("sudo update-ca-certificates".split(), check=True)
+
+    # https://www.dssw.co.uk/reference/security.html
+    def handle_darwin(self):
+        system_keychain_path = '/Library/Keychains/System.keychain'
+
+        ca_cert_path = self.ca_cert_path('.pem')
+        self.__ensure_exists(ca_cert_path)
+
+        subprocess.run(f"sudo security add-trusted-cert -d -p ssl -p basic -k {system_keychain_path} {ca_cert_path}".split(), check=True)
+
+    def handle_rhel(self):
+        ca_trust_dir = '/etc/pki/ca-trust/source/anchors'
+
+        ca_cert_path = self.ca_cert_path('.cer')
+        self.__ensure_exists(ca_cert_path)
+
+        ca_trust_crt_absolute_path = os.path.join(ca_trust_dir, self.crt_file_name)
+
+        subprocess.run(f"sudo cp {ca_cert_path} {ca_trust_crt_absolute_path}".split(), check=True)
+        subprocess.run("sudo update-ca-trust extract".split(), check=True)
+
+    def install(self):
+        distro_name = distro.name(pretty=True)
+
+        # Ubuntu or other Debian based
+        if distro.like() == 'debian':
+            self.handle_debian()
+        # MacOS
+        elif distro.id() == 'darwin':
+            self.handle_darwin()
+        # elif distro.id() == 'rhel':
+        #     return
+        else:
+            raise TypeError(f"{distro_name} is not supported yet for automatic CA cert installation.")
+
+    def sign(self, hostname: str, dest = None, port = 443):
+        dest = dest or self.certs_dir
+
+        if self.signed(hostname, dest):
+            return False
+
+        store = CertStore.from_store(self.certs_dir, self.cn, self.key_size)
+        organization, alt_names = self.__cert_details(hostname, port)
+        cert_entry = store.get_cert(alt_names[0], alt_names, organization)
+
+        crt_path = os.path.join(dest, f"{hostname}.crt")
+        with open(crt_path, 'wb') as fp:
+            cert = cert_entry.cert
+            crt = cert.to_pem()
+            fp.write(crt)
+
+        key_path = os.path.join(dest, f"{hostname}.key")
+        with open(key_path, 'wb') as fp:
+            private_key = cert_entry.privatekey
+            key = private_key.private_bytes(
+                encoding=serialization.Encoding.PEM,
+                format=serialization.PrivateFormat.TraditionalOpenSSL,
+                encryption_algorithm=serialization.NoEncryption(),
+            )
+            fp.write(key)
+
+        joined_path = os.path.join(dest, f"{hostname}-joined.pem")
+        with open(joined_path, 'wb') as fp:
+            fp.write(b"\n".join([crt, key]))
+
+        return True
+
+    def signed(self, hostname: str, dest = None):
+        dest = dest or self.certs_dir
+
+        crt_path = os.path.join(dest, f"{hostname}.crt")
+        if not os.path.exists(crt_path):
+            return False
+
+        key_path = os.path.join(dest, f"{hostname}.key")
+        if not os.path.exists(key_path):
+            return False
+
+        joined_path = os.path.join(dest, f"{hostname}-joined.pem")
+        if not os.path.exists(joined_path):
+            return False
+
+        return True
+
+    def __ensure_exists(self, file_path):
+        if not os.path.exists(file_path):
+            self.generate_certs()
+
+    def __cert_details(self, domain: str, port: int = 443):
+        """
+        Retrieves the organization name and subject alternative names from an HTTPS domain's certificate.
+
+        :param domain: The domain name (e.g., 'example.com').
+        :param port: The port number (default: 443 for HTTPS).
+        :return: A tuple containing the organization name and a list of alternative names (or None if not found).
+        """
+        alt_names = [domain]
+        try:
+            # Connect to the server and get the certificate
+            context = ssl.create_default_context()
+            context.minimum_version = ssl.TLSVersion.TLSv1_2
+            timeout = 2.5
+            with socket.create_connection((domain, port), timeout) as sock:
+                with context.wrap_socket(sock, server_hostname=domain) as ssock:
+                    cert_der = ssock.getpeercert(binary_form=True)
+
+            # Parse the certificate
+            cert = x509.load_der_x509_certificate(cert_der, default_backend())
+
+            # Extract the organization name
+            try:
+                org_name = cert.subject.get_attributes_for_oid(NameOID.ORGANIZATION_NAME)[0].value
+            except IndexError:
+                org_name = None
+
+            # Extract the Subject Alternative Names (SANs)
+            try:
+                ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName)
+                alt_names.extend(ext.value.get_values_for_type(x509.DNSName))
+            except x509.ExtensionNotFound:
+                alt_names = []
+
+            return org_name, alt_names
+
+        except Exception as e:
+            Logger.instance(LOG_ID).warn(f"Could not retrieve certificate for {domain}: {e}")
+            return None, alt_names

{stoobly_agent-0.34.12 → stoobly_agent-1.0.7}/stoobly_agent/app/cli/helpers/openapi_endpoint_adapter.py

@@ -4,7 +4,8 @@ import re
 import yaml
 
 from functools import reduce
-from openapi_core import Spec
+from jsonschema_path import SchemaPath
+from openapi_core import OpenAPI
 from pprint import pprint
 from typing import Dict, List, Union
 from urllib.parse import urlparse
@@ -35,11 +36,12 @@ class OpenApiEndpointAdapter():
       if missing_oauth2_scopes:
         self.__add_oauth2_default_scopes(file_data)
 
-      spec = Spec.from_dict(file_data)
+      openApi = OpenAPI.from_dict(file_data)
+      spec = openApi.spec
 
     return self.adapt(spec)
 
-  def adapt(self, spec: Spec) -> List[EndpointShowResponse]:
+  def adapt(self, spec: SchemaPath) -> List[EndpointShowResponse]:
     endpoints = []
     endpoint_counter = 0
     components = spec.get("components", {})
@@ -364,7 +366,7 @@ class OpenApiEndpointAdapter():
   
 
   # Returns the schema object located at the given reference path
-  def __dereference(self, components: Spec, reference: str):
+  def __dereference(self, components: SchemaPath, reference: str):
     # '#/components/schemas/NewPet'
     if not reference.startswith('#'):
       print('external references are not supported yet')
@@ -475,7 +477,7 @@ class OpenApiEndpointAdapter():
     num = url.count('{')
     return num
 
-  def __evaluate_servers(self, servers: Spec) -> List[dict]:
+  def __evaluate_servers(self, servers: SchemaPath) -> List[dict]:
     result = []
 
     if not servers:
@@ -524,7 +526,7 @@ class OpenApiEndpointAdapter():
     
     return result
 
-  def __parse_responses(self, endpoint: EndpointShowResponse, responses: Spec, components: Spec):
+  def __parse_responses(self, endpoint: EndpointShowResponse, responses: SchemaPath, components: SchemaPath):
     for response_code, response_definition in responses.items():
       # Only support status code 200 for now
       if response_code != '200':
@@ -566,4 +568,4 @@ class OpenApiEndpointAdapter():
           
         if not endpoint.get('response_header_names'):
           endpoint['response_header_names'] = []
-        endpoint['response_header_names'].append(response_header_name)	  
+        endpoint['response_header_names'].append(response_header_name)	  

stoobly_agent-1.0.7/stoobly_agent/app/cli/helpers/shell.py

@@ -0,0 +1,28 @@
+import subprocess
+import sys
+
+from dotenv import load_dotenv
+
+def exec_stream(command):
+  load_dotenv()
+
+  # Start the process
+  process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+  
+  # Stream output line by line
+  while True:
+      stdout = process.stdout.readline()
+      stderr = process.stderr.readline()
+
+      if stdout == '' and stderr == '' and process.poll() is not None:
+          break
+
+      if stdout:
+          print(stdout, end='')
+
+      if stderr:
+          print(stderr, end='', file=sys.stderr)
+  
+  # Get the remaining output (if any)
+  rc = process.poll()
+  return rc

{stoobly_agent-0.34.12 → stoobly_agent-1.0.7}/stoobly_agent/app/cli/main_group.py

@@ -26,7 +26,7 @@ class MainGroup(click.Group):
     command_groups: List[CommandGroup] = [
       {
         'name': 'Commands',
-        'commands': ['dev-tools', 'exec', 'feature', 'init', 'mock', 'record'],
+        'commands': ['dev-tools', 'exec', 'feature', 'init', 'mock', 'record', 'scaffold'],
       },
       {
         'name': 'Proxy Commands',

stoobly_agent-1.0.7/stoobly_agent/app/cli/scaffold/app.py

@@ -0,0 +1,134 @@
+import os
+import shutil
+
+from stoobly_agent.config.data_dir import DataDir, DATA_DIR_NAME
+
+class App():
+
+  def __init__(self, path: str, namespace: str, **kwargs):
+    path = path or os.getcwd()
+    data_dir: DataDir = DataDir.instance(path) 
+
+    self.__scaffold_dir_path = data_dir.path
+    self.__ca_certs_dir_path = kwargs.get('ca_certs_dir_path') or data_dir.mitmproxy_conf_dir_path
+    self.__certs_dir_path = data_dir.certs_dir_path
+    self.__context_dir_path = data_dir.context_dir_path
+    self.__dir_path = path
+    self.__name = os.path.basename(self.__dir_path)
+    self.__network = os.path.basename(self.__dir_path)
+    self.__namespace = namespace
+    self.__skip_validate_path = not not kwargs.get('skip_validate_path')
+
+  @property
+  def ca_certs_dir_path(self):
+    return self.__ca_certs_dir_path
+
+  @ca_certs_dir_path.setter
+  def ca_certs_dir_path(self, v: str):
+    self.__validate_path(v)
+    self.__certs_dir_path = v
+
+  @property
+  def certs_dir_path(self):
+    return self.__certs_dir_path
+
+  @certs_dir_path.setter
+  def certs_dir_path(self, v: str):
+    self.__validate_path(v)
+    self.__certs_dir_path = v
+
+  @property
+  def context_dir_path(self):
+    return self.__context_dir_path
+
+  @context_dir_path.setter
+  def context_dir_path(self, v: str):
+    self.__validate_path(v)
+    self.__context_dir_path = v 
+
+  @property
+  def data_dir_path(self):
+    return os.path.join(self.context_dir_path, DATA_DIR_NAME)
+
+  @property
+  def exists(self):
+    return os.path.exists(self.dir_path)
+
+  @property
+  def name(self):
+    return self.__name
+
+  @name.setter
+  def name(self, v: str):
+    self.__name = v
+
+  @property
+  def network(self):
+    return self.__network
+
+  @network.setter
+  def network(self, v: str):
+    self.__network = v
+
+  @property
+  def namespace(self):
+    return self.__namespace
+
+  @property
+  def namespace_path(self):
+    return os.path.join(self.data_dir_path, self.namespace)
+
+  @property
+  def dir_path(self):
+    return self.__dir_path
+
+  @property
+  def scaffold_dir_path(self):
+    return self.__scaffold_dir_path
+
+  @property
+  def scaffold_namespace_path(self):
+    return os.path.join(self.scaffold_dir_path, self.namespace)
+
+  @property
+  def services(self):
+    return list(map(lambda path: os.path.basename(path), self.service_paths))
+
+  @property
+  def service_paths(self):
+    services_dir = os.path.join(self.scaffold_dir_path, self.namespace)
+
+    services = []
+    for filename in os.listdir(services_dir):
+      path = os.path.join(services_dir, filename)
+      if not os.path.isdir(path):
+        continue
+      
+      services.append(path)
+
+    return services
+
+  def copy_folders_and_hidden_files(self, src, dst):
+      os.makedirs(dst, exist_ok=True)
+
+      # Walk through the source directory
+      for root, dirs, files in os.walk(src):
+          # Copy hidden files only
+          for file_name in files:
+              src_file_path = os.path.join(root, file_name)
+              dst_file_path = os.path.join(dst, os.path.relpath(root, src), file_name)
+
+              if not file_name.startswith('.'):
+                  if os.path.exists(dst_file_path):
+                      continue
+
+              os.makedirs(os.path.dirname(dst_file_path), exist_ok=True)  # Create directories in destination
+              shutil.copy2(src_file_path, dst_file_path)
+
+  def __validate_path(self, v: str):
+    if not isinstance(v, str):
+      raise TypeError('Expected a str')
+
+    if not self.__skip_validate_path:
+      if not os.path.exists(v):
+        raise ValueError(f"{v} does not exist")