folioflex 1.2.1__tar.gz → 1.2.2__tar.gz

{folioflex-1.2.1/folioflex.egg-info → folioflex-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: folioflex
-Version: 1.2.1
+Version: 1.2.2
 Summary: A collection of portfolio tracking capabilities
 Author-email: John Koestner <johnkoestner@outlook.com>
 License: The MIT License (MIT)
@@ -42,6 +42,7 @@ Requires-Dist: pandas-market-calendars>=4.1.4
 Requires-Dist: pyxirr>=0.7.2
 Requires-Dist: sqlalchemy>=2.0.23
 Requires-Dist: yfinance>=0.2.32
+Requires-Dist: tzlocal>=5.2
 Requires-Dist: dash>=1.0.2
 Requires-Dist: dash_bootstrap_components>=1.6.0
 Requires-Dist: dash-core-components>=1.0.0
@@ -58,6 +59,7 @@ Requires-Dist: redis>=3.3.8
 Requires-Dist: g4f==0.2.4.1
 Requires-Dist: hugchat>=0.3.8
 Requires-Dist: openai>=1.3.7
+Requires-Dist: pyautogui>=0.9.53
 Requires-Dist: seleniumbase>=4.22.0
 Requires-Dist: emoji>=2.9.0
 Requires-Dist: gensim>=4.3.2
@@ -194,7 +196,7 @@ services:
   folioflex-web:
     image: dmbymdt/folioflex:latest
     container_name: folioflex-web
-    command: gunicorn -b 0.0.0.0:8001 app:server
+    command: gunicorn -b 0.0.0.0:8001 folioflex.dashboard.app:server
     restart: unless-stopped
     environment:
       FFX_CONFIG_PATH: /code/folioflex/configs

{folioflex-1.2.1 → folioflex-1.2.2}/README.md

@@ -118,7 +118,7 @@ services:
   folioflex-web:
     image: dmbymdt/folioflex:latest
     container_name: folioflex-web
-    command: gunicorn -b 0.0.0.0:8001 app:server
+    command: gunicorn -b 0.0.0.0:8001 folioflex.dashboard.app:server
     restart: unless-stopped
     environment:
       FFX_CONFIG_PATH: /code/folioflex/configs

folioflex-1.2.2/folioflex/chatbot/scraper.py

@@ -0,0 +1,363 @@
+"""
+Scraper module.
+
+This module contains functions to scrape the html of a website to be able
+to share to a gpt.
+
+"""
+
+import http.client
+import os
+import re
+
+import requests
+from bs4 import BeautifulSoup
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.remote.webdriver import WebDriver
+from seleniumbase import SB, Driver
+
+from folioflex.utils import config_helper, custom_logger
+
+logger = custom_logger.setup_logging(__name__)
+
+
+def scrape_html(
+    url,
+    scraper="selenium",
+    **kwargs,
+):
+    """
+    Scrape the html of a website.
+
+    Parameters
+    ----------
+    url : str
+        url of the website to scrape
+    scraper : str (optional)
+        scraper to use, seleniumbase by default
+    **kwargs : dict (optional)
+        keyword arguments for the options of the driver
+
+    Returns
+    -------
+    scrape_results : dict
+        dictionary with the url and the text of the website
+
+    """
+    scrape_results = {"url": url, "text": None}
+    logger.info(f"scraping '{url}' with '{scraper}'")
+    if scraper == "selenium":
+        soup, url = scrape_selenium(url, **kwargs)
+    elif scraper == "bee":
+        soup = scrape_bee(url, **kwargs)
+
+    # removing the html tags
+    scrape_text = soup.get_text(separator=" ", strip=True)
+    scrape_text = scrape_text.replace("\xa0", " ").replace("\\", "")
+
+    if url.startswith("https://www.wsj.com/livecoverage"):
+        # Use regex to find everything between "LIVE UPDATES"
+        # and "What to Read Next"
+        logger.info("cleaning the text")
+        pattern = r"LIVE(.*?)By "
+        match = re.search(pattern, scrape_text, re.DOTALL)
+        try:
+            scrape_text = match.group(1)
+        except AttributeError:
+            logger.info("no match found")
+
+    scrape_results = {"url": url, "text": scrape_text}
+
+    return scrape_results
+
+
+def close_windows(sb, url):
+    """
+    Close windows except url.
+
+    Parameters
+    ----------
+    sb : seleniumbase.SB
+        seleniumbase instance
+    url : str
+        url of the website to scrape
+
+    """
+    open_windows = sb.driver.window_handles
+    nbr_windows = len(open_windows)
+    logger.info(f"close {nbr_windows-1} open windows")
+    for window in open_windows:
+        sb.driver.switch_to.window(window)
+        nbr_windows = len(sb.driver.window_handles)
+        if url not in sb.get_current_url() and nbr_windows > 1:
+            sb.driver.close()
+    open_windows = sb.driver.window_handles
+    sb.driver.switch_to.window(open_windows[0])
+
+
+def scrape_selenium(
+    url,
+    xvfb=None,
+    screenshot=False,
+    port=None,
+    **kwargs,
+):
+    """
+    Scrape the html of a website using seleniumbase.
+
+    seleniumbase has a lot of methods that can be used in kwargs shown here:
+    https://github.com/seleniumbase/SeleniumBase/blob/af3d9545473e55b2a25cdbab8be0b1ed5e1f6afa/seleniumbase/plugins/sb_manager.py
+
+
+
+    Parameters
+    ----------
+    url : str
+        url of the website to scrape
+    xvfb : bool (optional)
+        It's recommended if using uc=True to not run the headless2=True option and to
+        have xvfb=True if running in linux.
+    screenshot : bool (optional)
+        take a screenshot of the website
+    port : int (optional)
+        port to use for debugging
+    **kwargs : dict (optional)
+        keyword arguments for the options of the driver
+
+    Returns
+    -------
+    soup : bs4.BeautifulSoup
+        beautiful soup object with the html of the website
+    url : str
+        url of the website
+
+    """
+    # kwargs defaults
+    binary_location = kwargs.pop("binary_location", None)
+    extension_dir = kwargs.pop("extension_dir", None)
+    headless2 = kwargs.pop("headless2", False)
+    wait_time = kwargs.pop("wait_time", 10)
+    proxy = kwargs.pop("proxy", None)
+    chromium_arg = None
+
+    # use xvfb if running a linux os and xvfb is not specified
+    if xvfb is None and os.name == "posix" and not headless2:
+        logger.info("using xvfb for browser")
+        xvfb = True
+    if port:
+        logger.info(f"using port {port} for browser debugging")
+        chromium_arg = f"--remote-debugging-port={port}"
+
+    if binary_location or config_helper.BROWSER_LOCATION:
+        logger.info("using binary location for browser")
+        binary_location = binary_location or config_helper.BROWSER_LOCATION
+    if extension_dir or config_helper.BROWSER_EXTENSION:
+        logger.info("using extension location for browser")
+        extension_dir = extension_dir or config_helper.BROWSER_EXTENSION
+    if proxy:
+        logger.info("using proxy for browser")
+
+    # specific landing page
+    #
+    # this breaks frequently. added the following due to breaks
+    # good issue describing the detection:
+    # https://github.com/seleniumbase/SeleniumBase/issues/2842
+    #
+    # incognito=True, to avoid detection
+    # xvfb=True, uc works better when display is shown and linux usually needs xvfb
+    # headless2=False, uc works better when display is shown
+    if re.match(r"https://www\.w.j\.com/finance", url, re.IGNORECASE):
+        with SB(
+            uc=True,
+            incognito=True,
+            xvfb=xvfb,
+            headless2=headless2,
+            binary_location=binary_location,
+            extension_dir=extension_dir,
+            proxy=proxy,
+            chromium_arg=chromium_arg,
+            **kwargs,
+        ) as sb:
+            sb.sleep(2)
+            close_windows(sb, url)
+            logger.info("obtaining the landing page")
+            sb.driver.uc_open_with_reconnect(url, reconnect_time=wait_time + 5)
+            try:
+                sb.driver.uc_click(
+                    "(//p[contains(text(), 'View All')])[1]", reconnect_time=wait_time
+                )
+                url = sb.get_current_url()
+                logger.info(f"scraping {url}")
+                soup = sb.get_beautiful_soup()
+                if screenshot:
+                    logger.info("screenshot saved to 'screenshot.png'")
+                    sb.driver.save_screenshot("screenshot.png")
+            except Exception:
+                logger.error("probably flagged bot: returning None")
+                html_content = "<html><body><p>could not scrape</p></body></html>"
+                soup = BeautifulSoup(html_content, "html.parser")
+                if screenshot:
+                    logger.info("screenshot saved to 'screenshot.png'")
+                    sb.driver.save_screenshot("screenshot.png")
+                return soup, url
+
+    else:
+        with SB(
+            uc=True,
+            incognito=True,
+            xvfb=xvfb,
+            headless2=headless2,
+            binary_location=binary_location,
+            extension_dir=extension_dir,
+            proxy=proxy,
+            chromium_arg=chromium_arg,
+            **kwargs,
+        ) as sb:
+            sb.sleep(2)
+            close_windows(sb, url)
+            logger.info("initializing the driver")
+            sb.driver.uc_open_with_reconnect(url, reconnect_time=wait_time)
+            logger.info(f"scraping {url}")
+            soup = sb.get_beautiful_soup()
+            if screenshot:
+                logger.info("screenshot saved to 'screenshot.png'")
+                sb.driver.save_screenshot("screenshot.png")
+
+    logger.info("scraped")
+
+    return soup, url
+
+
+def scrape_bee(url, **kwargs):
+    """
+    Scrape the html of a website using scrapingbee.
+
+    Parameters
+    ----------
+    url : str
+        url of the website to scrape
+    **kwargs : dict (optional)
+        keyword arguments for the options of the driver
+
+    Returns
+    -------
+    soup : bs4.BeautifulSoup
+        beautiful soup object with the html of the website
+
+    """
+    # increase the max headers to avoid error
+    http.client._MAXHEADERS = 1000
+    api_key = config_helper.SCRAPINGBEE_API
+    stealth_proxy = kwargs.get("stealth_proxy", "true")
+    response = requests.get(
+        url="https://app.scrapingbee.com/api/v1/",
+        params={
+            "api_key": api_key,
+            "url": url,
+            "stealth_proxy": stealth_proxy,
+        },
+    )
+    soup = BeautifulSoup(response.content, "html.parser")
+
+    return soup
+
+
+def scrape_test(url, **kwargs):
+    """
+    Test basic functionality of seleniumbase scraper.
+
+    When debugging a website it is useful if able to able to find the cause of
+    the error from the function or from another source. This function creates
+    a pause when connecting to the website that will temporarily stop the selenium
+    driver.
+
+    Here are some common test sites:
+    - pixelscan.net
+    - fingerprint.com/products/bot-detection/
+    - nowsecure.nl
+
+    Parameters
+    ----------
+    url : str
+        url of the website to scrape
+    **kwargs : dict (optional)
+        keyword arguments for the options of the driver
+
+    Returns
+    -------
+    soup : bs4.BeautifulSoup
+        beautiful soup object with the html of the website
+
+    """
+    driver = Driver(
+        uc=True,
+        headless2=False,
+    )
+    driver.sleep(2)
+    close_windows(driver, url)
+    logger.info("connecting to website")
+    driver.uc_open_with_reconnect(url, reconnect_time="breakpoint")
+    logger.info("exit site")
+    driver.quit()
+
+
+def attach_to_session(executor_url, session_id, options=None):
+    """
+    Attach to an existing browser session.
+
+    Parameters
+    ----------
+    executor_url : str
+        The URL of the WebDriver server to connect to.
+    session_id : str
+        The ID of the session to attach to.
+    options : Options, optional
+        The options to use when attaching to the session.
+
+    Returns
+    -------
+    WebDriver
+        A WebDriver instance that is attached to the existing session.
+
+    """
+    # save the original execute method of driver
+    original_execute = WebDriver.execute
+
+    def override_execute_method(self, command, params=None):
+        """
+        Override for the newSession command.
+
+        Parameters
+        ----------
+        self : WebDriver
+            The WebDriver instance to execute the command on.
+        command : str
+            The name of the command to execute.
+        params : dict, optional
+            The parameters for the command.
+
+        Returns
+        -------
+        dict
+            The result of the command execution.
+
+        """
+        # point to the existing session
+        if command == "newSession":
+            return {"value": {"sessionId": session_id, "capabilities": {}}}
+        else:
+            return original_execute(self, command, params)
+
+    # override the execute method with the original session
+    WebDriver.execute = override_execute_method
+
+    # create the driver
+    if options is None:
+        options = Options()
+    driver = WebDriver(command_executor=executor_url, options=options)
+    driver.session_id = session_id
+
+    # restore the method
+    WebDriver.execute = original_execute
+
+    return driver

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex/configs/config.ini

@@ -10,6 +10,7 @@ ffx_password = strongpassword
 
 [api]
 fred_api = None
+scrapingbee_api = None
 yodlee_client_id = None
 yodlee_secret = None
 yodlee_endpoint = None

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex/dashboard/app.py

@@ -27,11 +27,12 @@ from folioflex.utils import custom_logger
 #  /_/   \_\_|   |_|
 
 dbc_css = "https://cdn.jsdelivr.net/gh/AnnMarieW/dash-bootstrap-templates/dbc.min.css"
+server = auth.server
 
 # app configs
 app = dash.Dash(
     __name__,
-    server=auth.server,
+    server=server,
     use_pages=True,
     external_stylesheets=[
         dbc_css,

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex/dashboard/pages/macro.py

@@ -148,3 +148,10 @@ def layout():
             ),
         ]
     )
+
+
+#    ____      _ _ _                _
+#   / ___|__ _| | | |__   __ _  ___| | _____
+#  | |   / _` | | | '_ \ / _` |/ __| |/ / __|
+#  | |__| (_| | | | |_) | (_| | (__|   <\__ \
+#   \____\__,_|_|_|_.__/ \__,_|\___|_|\_\___/

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex/portfolio/broker.py

@@ -181,7 +181,7 @@ def fidelity(broker_file, output_file=None, broker="fidelity"):
     start_df_len = len(df)
 
     # update date column type
-    df["date"] = pd.to_datetime(df["run_date"], format="%m/%d/%Y").dt.date
+    df["date"] = pd.to_datetime(df["run_date"], format="%b-%d-%Y").dt.date
 
     # Loop through each type_lkup and update type
     for string, tag in type_lkup.items():

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex/portfolio/helper.py

@@ -12,12 +12,12 @@ import pandas as pd
 import pandas_market_calendars as mcal
 from dateutil.parser import parse
 
-from folioflex.utils import custom_logger
+from folioflex.utils import config_helper, custom_logger
 
 logger = custom_logger.setup_logging(__name__)
 
 
-def check_stock_dates(tx_df, fix=False, timezone="US/Eastern", warning=True):
+def check_stock_dates(tx_df, fix=False, warning=True, timezone="local"):
     """
     Check that the transaction dates are valid.
 
@@ -35,10 +35,10 @@ def check_stock_dates(tx_df, fix=False, timezone="US/Eastern", warning=True):
         transactions dataframe
     fix : bool (optional)
         if True then the dates will be fixed to previous valid date
-    timezone : str (optional)
-        timezone to use for checking dates
     warning : bool (optional)
         if True then a warning will be logged if dates are fixed
+    timezone : str (optional)
+        timezone to use for checking dates
 
     Returns
     -------
@@ -67,14 +67,22 @@ def check_stock_dates(tx_df, fix=False, timezone="US/Eastern", warning=True):
             "The date column is not a date object, please convert it to a date object"
         )
 
+    # get the timezone
+    if timezone == "local":
+        timezone = config_helper.LOCAL_TIMEZONE
+
     # date checks
-    tx_df_min = tx_df_min = tx_df["date"].min() - timedelta(days=7)
+    tx_df_min = tx_df["date"].min() - timedelta(days=7)
     tx_df_max = tx_df["date"].max()
     stock_dates = mcal.get_calendar("NYSE").schedule(
         start_date=tx_df_min, end_date=tx_df_max
     )
-    stock_dates["market_open"] = stock_dates["market_open"].dt.tz_convert(timezone)
-    stock_dates["market_close"] = stock_dates["market_close"].dt.tz_convert(timezone)
+    stock_dates["market_open"] = convert_date_to_timezone(
+        stock_dates["market_open"], timezone
+    )
+    stock_dates["market_close"] = convert_date_to_timezone(
+        stock_dates["market_close"], timezone
+    )
     stock_dates = pd.to_datetime(stock_dates["market_open"]).dt.date
 
     # change datetime to date
@@ -110,6 +118,37 @@ def check_stock_dates(tx_df, fix=False, timezone="US/Eastern", warning=True):
     return {"invalid_dt": invalid_dt, "fix_tx_df": fix_tx_df}
 
 
+def convert_date_to_timezone(date_series, timezone="local"):
+    """
+    Convert date column to a specific timezone.
+
+    Parameters
+    ----------
+    date_series : series
+        series with date column
+    timezone : str (optional)
+        timezone to convert the date to
+
+    Returns
+    -------
+    converted_date : series
+        series with date column converted to timezone
+
+    """
+    if not isinstance(date_series, pd.Series):
+        raise ValueError("date_series must be a pandas Series")
+
+    if timezone == "local":
+        timezone = config_helper.LOCAL_TIMEZONE
+    elif timezone is None and date_series.dt.tz:
+        converted_date = pd.to_datetime(date_series).dt.tz_localize(timezone)
+    elif date_series.dt.tz:
+        converted_date = pd.to_datetime(date_series).dt.tz_convert(timezone)
+    else:
+        converted_date = pd.to_datetime(date_series).dt.tz_localize(timezone)
+    return converted_date
+
+
 def most_recent_stock_date():
     """Get the most recent stock date."""
     stock_dates = mcal.get_calendar("NYSE").schedule(
@@ -133,6 +172,7 @@ def prettify_dataframe(dataframe):
     -------
     DataFrame
         prettified dataframe
+
     """
     if not isinstance(dataframe, pd.DataFrame):
         raise ValueError("dataframe must be a pandas DataFrame")
@@ -159,6 +199,7 @@ def convert_lookback(lookback):
     -------
     converted_lookback : int
         converted lookback
+
     """
 
     def is_string_a_date(string):

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex/portfolio/portfolio.py

@@ -28,7 +28,11 @@ import pandas_market_calendars as mcal
 import plotly.express as px
 from pyxirr import xirr
 
-from folioflex.portfolio.helper import check_stock_dates, convert_lookback
+from folioflex.portfolio.helper import (
+    check_stock_dates,
+    convert_date_to_timezone,
+    convert_lookback,
+)
 from folioflex.portfolio.wrappers import Yahoo
 from folioflex.utils import config_helper, custom_logger
 
@@ -321,6 +325,9 @@ class Portfolio:
         )
 
         transactions["date"] = pd.to_datetime(transactions["date"], format="%m/%d/%Y")
+        transactions["date"] = convert_date_to_timezone(
+            transactions["date"], timezone=None
+        )
 
         transactions["price"] = (transactions["cost"] / transactions["units"]) * -1
         transactions.loc[transactions["ticker"] == "Cash", "price"] = 1

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex/portfolio/wrappers.py

@@ -275,7 +275,9 @@ class TradingView:
             "countries": ["US"],
             "minImportance": minImportance,
         }
-        response = requests.get(url, params=payload).json()
+        # headers are now required as 07/24/2024
+        headers = {"Origin": "https://us.tradingview.com"}
+        response = requests.get(url, params=payload, headers=headers).json()
         calendar = pd.DataFrame(response["result"])
         calendar["date"] = pd.to_datetime(calendar["date"]).dt.date
         # select columns to keep
@@ -485,6 +487,9 @@ class Yahoo:
         cols = ["ticker", "date", "adj_close", "stock_splits"]
         stock_data = stock_data[cols]
         stock_data = stock_data.rename(columns={"adj_close": "last_price"})
+        stock_data["date"] = helper.convert_date_to_timezone(
+            stock_data["date"], timezone=None
+        )
 
         return stock_data
 

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex/utils/cli.py

@@ -213,6 +213,22 @@ def _create_argparser():
         help=("The proxy to use for the chatbot - user:password@ip:port"),
     )
 
+    _email_parser.add_argument(
+        "-pt",
+        "--port",
+        type=int,
+        default=None,
+        help=("The remote debugging port to use for the chatbot"),
+    )
+
+    _email_parser.add_argument(
+        "-s",
+        "--scraper",
+        type=str,
+        default="selenium",
+        help=("The scraper to use for the chatbot - 'bee' or 'selenium'"),
+    )
+
     # subparser: dashboard
     _dash_parser = _subparsers.add_parser("dash", help="dashboard command")
 
@@ -249,6 +265,9 @@ def cli():
             manager_performance=args.manager_performance,
             portfolio_performance=args.portfolio_performance,
             chatbot=args.chatbot,
+            proxy=args.proxy,
+            port=args.port,
+            scraper=args.scraper,
         )
         print(f"status sent: {email_status}")
 

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex/utils/config_helper.py

@@ -5,6 +5,8 @@ import configparser
 import os
 from pathlib import Path
 
+import tzlocal
+
 from folioflex.utils import custom_logger
 
 ROOT_PATH = Path(__file__).resolve().parent.parent.parent
@@ -14,6 +16,7 @@ CONFIG_PATH = (
     else ROOT_PATH / "folioflex" / "configs"
 )
 TESTS_PATH = ROOT_PATH / "tests" / "files"
+LOCAL_TIMEZONE = tzlocal.get_localzone()
 
 
 logger = custom_logger.setup_logging(__name__)
@@ -175,6 +178,7 @@ FFX_PASSWORD = get_config_options(config_file, "credentials").get("ffx_password"
 
 # apis
 FRED_API = get_config_options(config_file, "api").get("fred_api", None)
+SCRAPINGBEE_API = get_config_options(config_file, "api").get("scrapingbee_api", None)
 YODLEE_CLIENT_ID = get_config_options(config_file, "api").get("yodlee_client_id", None)
 YODLEE_SECRET = get_config_options(config_file, "api").get("yodlee_secret", None)
 YODLEE_ENDPOINT = get_config_options(config_file, "api").get("yodlee_endpoint", None)

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex/utils/mailer.py

@@ -92,6 +92,8 @@ def generate_report(
     portfolio_performance=None,
     chatbot=False,
     proxy=None,
+    port=None,
+    scraper="bee",
 ):
     """
     Generate report of portfolio performance and send to email.
@@ -138,6 +140,10 @@ def generate_report(
         Whether to use the chatbot to get the query
     proxy : str (optional)
         Proxy to use for the chatbot
+    port : int (optional)
+        Port to use for the chatbot
+    scraper : str (optional)
+        Scraper to use for the chatbot, "bee" by default
 
     Returns
     -------
@@ -315,26 +321,16 @@ def generate_report(
 
     if chatbot:
         chatbot = providers.GPTchat(provider="openai")
-        # get todays date and make sure it's a valid trading day to use in url
-        now = datetime.datetime.now()
-        start_hour = 8
-        if now.hour < start_hour:
-            today = datetime.date.today() - datetime.timedelta(days=1)
-        else:
-            today = datetime.date.today()
-        today = helper.check_stock_dates(today, fix=True, warning=False)["fix_tx_df"][
-            "date"
-        ][0]
-        formatted_today = today.strftime("%m-%d-%Y")
-
         # get the url to scrape
         scrape_url = "https://www.wsj.com/finance"
 
         # get the query
         response = chatbot.chat(
-            query=f"could you summarize this for me? {formatted_today}",
+            query="could you summarize this for me?",
             scrape_url=scrape_url,
             proxy=proxy,
+            scraper=scraper,
+            port=port,
         )
         response = response.replace("\n", "<br>")
 

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex/version.py

@@ -1,3 +1,3 @@
 """Version of app."""
 
-version = "1.2.1"
+version = "1.2.2"

{folioflex-1.2.1 → folioflex-1.2.2/folioflex.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: folioflex
-Version: 1.2.1
+Version: 1.2.2
 Summary: A collection of portfolio tracking capabilities
 Author-email: John Koestner <johnkoestner@outlook.com>
 License: The MIT License (MIT)
@@ -42,6 +42,7 @@ Requires-Dist: pandas-market-calendars>=4.1.4
 Requires-Dist: pyxirr>=0.7.2
 Requires-Dist: sqlalchemy>=2.0.23
 Requires-Dist: yfinance>=0.2.32
+Requires-Dist: tzlocal>=5.2
 Requires-Dist: dash>=1.0.2
 Requires-Dist: dash_bootstrap_components>=1.6.0
 Requires-Dist: dash-core-components>=1.0.0
@@ -58,6 +59,7 @@ Requires-Dist: redis>=3.3.8
 Requires-Dist: g4f==0.2.4.1
 Requires-Dist: hugchat>=0.3.8
 Requires-Dist: openai>=1.3.7
+Requires-Dist: pyautogui>=0.9.53
 Requires-Dist: seleniumbase>=4.22.0
 Requires-Dist: emoji>=2.9.0
 Requires-Dist: gensim>=4.3.2
@@ -194,7 +196,7 @@ services:
   folioflex-web:
     image: dmbymdt/folioflex:latest
     container_name: folioflex-web
-    command: gunicorn -b 0.0.0.0:8001 app:server
+    command: gunicorn -b 0.0.0.0:8001 folioflex.dashboard.app:server
     restart: unless-stopped
     environment:
       FFX_CONFIG_PATH: /code/folioflex/configs

{folioflex-1.2.1 → folioflex-1.2.2}/folioflex.egg-info/requires.txt

@@ -10,6 +10,7 @@ pandas-market-calendars>=4.1.4
 pyxirr>=0.7.2
 sqlalchemy>=2.0.23
 yfinance>=0.2.32
+tzlocal>=5.2
 dash>=1.0.2
 dash_bootstrap_components>=1.6.0
 dash-core-components>=1.0.0
@@ -26,6 +27,7 @@ redis>=3.3.8
 g4f==0.2.4.1
 hugchat>=0.3.8
 openai>=1.3.7
+pyautogui>=0.9.53
 seleniumbase>=4.22.0
 emoji>=2.9.0
 gensim>=4.3.2

{folioflex-1.2.1 → folioflex-1.2.2}/pyproject.toml

@@ -24,6 +24,7 @@ dependencies = [
     "pyxirr>=0.7.2",
     "sqlalchemy>=2.0.23",
     "yfinance>=0.2.32",
+    "tzlocal>=5.2",
 
     # web
     "dash>=1.0.2",
@@ -46,6 +47,7 @@ dependencies = [
     "g4f==0.2.4.1",         # this package has loose testing support
     "hugchat>=0.3.8",
     "openai>=1.3.7",
+    "pyautogui>=0.9.53",
     "seleniumbase>=4.22.0",
 
     # budget

{folioflex-1.2.1 → folioflex-1.2.2}/tests/test_chatbot.py

@@ -20,8 +20,6 @@ def test_g4f():
     assert chatbot.chatbot["provider"] == g4f.Provider.Bing, "Default provider not set."
     assert chatbot.chatbot["auth"] == False, "Default auth not set."
     assert chatbot.chatbot["access_token"] is None, "Default access token not set."
-    response = chatbot.chat("return back 'test' for test purpose")
-    assert isinstance(response, str), "Response not a string."
 
 
 # openai has billing so not testing unless needed

folioflex-1.2.1/folioflex/chatbot/scraper.py

@@ -1,136 +0,0 @@
-"""
-Scraper module.
-
-This module contains functions to scrape the html of a website to be able
-to share to a gpt.
-
-"""
-
-import re
-
-from seleniumbase import SB
-
-from folioflex.utils import config_helper, custom_logger
-
-logger = custom_logger.setup_logging(__name__)
-
-
-def scrape_html(
-    url,
-    binary_location=None,
-    extension_dir=None,
-    headless2=True,
-    wait_time=10,
-    proxy=None,
-    **kwargs,
-):
-    """
-    Scrape the html of a website.
-
-    Parameters
-    ----------
-    url : str
-        url of the website to scrape
-    binary_location : str (optional)
-        location of the binary for the browser
-    extension_dir : str (optional)
-        location of the extension for the browser
-    headless2 : bool (optional)
-        whether to run the browser in headless mode
-        True by default
-    wait_time : int
-        time in seconds to wait for page load
-    proxy : str (optional)
-        proxy to use for the browser
-    **kwargs : dict (optional)
-        keyword arguments for the options of the driver
-
-    Returns
-    -------
-    scrape_results : dict
-        dictionary with the url and the text of the website
-    """
-    scrape_results = {"url": url, "text": None}
-    if binary_location or config_helper.BROWSER_LOCATION:
-        logger.info("using binary location for browser")
-        binary_location = binary_location or config_helper.BROWSER_LOCATION
-    if extension_dir or config_helper.BROWSER_EXTENSION:
-        logger.info("using extension location for browser")
-        extension_dir = extension_dir or config_helper.BROWSER_EXTENSION
-    with SB(
-        uc=True,
-        headless2=headless2,
-        binary_location=binary_location,
-        extension_dir=extension_dir,
-        proxy=proxy,
-        **kwargs,
-    ) as sb:
-        logger.info("initializing the driver")
-
-        # wsj
-        if url.startswith("https://www.wsj.com/finance"):
-            url = "https://www.wsj.com/finance"
-            sb.driver.uc_open_with_reconnect(url, reconnect_time=wait_time)
-            close_windows(sb, url)
-            try:
-                logger.info("wsj has specific landing page")
-                sb.driver.uc_click("(//p[contains(text(), 'View All')])[1]")
-            except Exception:
-                logger.error("WSJ probably flagged bot: returning None")
-                return scrape_results
-
-            logger.info(f"scraping {sb.get_current_url()}")
-            sb.sleep(wait_time)  # wait for page to load
-            soup = sb.get_beautiful_soup()
-
-            # removing the html tags
-            scrape_text = soup.get_text(separator=" ", strip=True)
-            scrape_text = scrape_text.replace("\xa0", " ").replace("\\", "")
-
-            logger.info("cleaning the text")
-            # Use regex to find everything between "LIVE UPDATES"
-            # and "What to Read Next"
-            pattern = r"LIVE(.*?)— By"
-            match = re.search(pattern, scrape_text, re.DOTALL)
-            try:
-                scrape_text = match.group(1)
-            except AttributeError:
-                logger.info("no match found")
-
-        # TODO: think about adding in https://www.bloomberg.com/ here
-
-        # all other websites
-        else:
-            sb.driver.uc_open_with_reconnect(url, reconnect_time=wait_time)
-            close_windows(sb, url)
-
-            logger.info(f"scraping {url}")
-            soup = sb.get_beautiful_soup()
-            scrape_text = str(soup)
-
-        url = sb.get_current_url()
-        scrape_results = {"url": url, "text": scrape_text}
-
-    return scrape_results
-
-
-def close_windows(sb, url):
-    """
-    Close windows except url.
-
-    Parameters
-    ----------
-    sb : seleniumbase.SB
-        seleniumbase instance
-    url : str
-        url of the website to scrape
-
-    """
-    open_windows = sb.driver.window_handles
-    logger.info(f"close {len(open_windows)-1} open windows")
-    for window in open_windows:
-        sb.driver.switch_to.window(window)
-        if url not in sb.get_current_url():
-            sb.driver.close()
-    open_windows = sb.driver.window_handles
-    sb.driver.switch_to.window(open_windows[0])