flyte 2.0.0b13__py3-none-any.whl → 2.0.0b30__py3-none-any.whl

flyte/__init__.py

@@ -9,15 +9,17 @@ import sys
 from ._build import build
 from ._cache import Cache, CachePolicy, CacheRequest
 from ._context import ctx
+from ._custom_context import custom_context, get_custom_context
 from ._deploy import build_images, deploy
 from ._environment import Environment
 from ._excepthook import custom_excepthook
 from ._group import group
 from ._image import Image
-from ._initialize import init, init_from_config
+from ._initialize import current_domain, init, init_from_config
+from ._logging import logger
 from ._map import map
 from ._pod import PodTemplate
-from ._resources import GPU, TPU, Device, Resources
+from ._resources import AMD_GPU, GPU, HABANA_GAUDI, TPU, Device, DeviceClass, Neuron, Resources
 from ._retry import RetryStrategy
 from ._reusable_environment import ReusePolicy
 from ._run import run, with_runcontext
@@ -25,6 +27,7 @@ from ._secret import Secret, SecretRequest
 from ._task_environment import TaskEnvironment
 from ._timeout import Timeout, TimeoutType
 from ._trace import trace
+from ._trigger import Cron, FixedRate, Trigger, TriggerTime
 from ._version import __version__
 
 sys.excepthook = custom_excepthook
@@ -59,14 +62,20 @@ def version() -> str:
 
 
 __all__ = [
+    "AMD_GPU",
     "GPU",
+    "HABANA_GAUDI",
     "TPU",
     "Cache",
     "CachePolicy",
     "CacheRequest",
+    "Cron",
     "Device",
+    "DeviceClass",
     "Environment",
+    "FixedRate",
     "Image",
+    "Neuron",
     "PodTemplate",
     "Resources",
     "RetryStrategy",
@@ -76,16 +85,23 @@ __all__ = [
     "TaskEnvironment",
     "Timeout",
     "TimeoutType",
+    "Trigger",
+    "TriggerTime",
     "__version__",
     "build",
     "build_images",
     "ctx",
+    "current_domain",
+    "custom_context",
     "deploy",
+    "get_custom_context",
     "group",
     "init",
     "init_from_config",
+    "logger",
     "map",
     "run",
     "trace",
+    "version",
     "with_runcontext",
 ]

flyte/_bin/debug.py

@@ -0,0 +1,38 @@
+import click
+
+
+@click.group()
+def _debug():
+    """Debug commands for Flyte."""
+
+
+@_debug.command("resume")
+@click.option("--pid", "-m", type=int, required=True, help="PID of the vscode server.")
+def resume(pid):
+    """
+    Resume a Flyte task for debugging purposes.
+
+    Args:
+        pid (int): PID of the vscode server.
+    """
+    import os
+    import signal
+
+    print("Terminating server and resuming task.")
+    answer = (
+        input(
+            "This operation will kill the server. All unsaved data will be lost,"
+            " and you will no longer be able to connect to it. Do you really want to terminate? (Y/N): "
+        )
+        .strip()
+        .upper()
+    )
+    if answer == "Y":
+        os.kill(pid, signal.SIGTERM)
+        print("The server has been terminated and the task has been resumed.")
+    else:
+        print("Operation canceled.")
+
+
+if __name__ == "__main__":
+    _debug()

flyte/_bin/runtime.py

@@ -12,6 +12,9 @@ from typing import Any, List
 
 import click
 
+from flyte._utils.helpers import str2bool
+from flyte.models import PathRewrite
+
 # Todo: work with pvditt to make these the names
 # ACTION_NAME = "_U_ACTION_NAME"
 # RUN_NAME = "_U_RUN_NAME"
@@ -21,14 +24,16 @@ import click
 
 ACTION_NAME = "ACTION_NAME"
 RUN_NAME = "RUN_NAME"
-PROJECT_NAME = "FLYTE_INTERNAL_TASK_PROJECT"
-DOMAIN_NAME = "FLYTE_INTERNAL_TASK_DOMAIN"
+PROJECT_NAME = "FLYTE_INTERNAL_EXECUTION_PROJECT"
+DOMAIN_NAME = "FLYTE_INTERNAL_EXECUTION_DOMAIN"
 ORG_NAME = "_U_ORG_NAME"
 ENDPOINT_OVERRIDE = "_U_EP_OVERRIDE"
+INSECURE_SKIP_VERIFY_OVERRIDE = "_U_INSECURE_SKIP_VERIFY"
 RUN_OUTPUT_BASE_DIR = "_U_RUN_BASE"
+FLYTE_ENABLE_VSCODE_KEY = "_F_E_VS"
 
-# TODO: Remove this after proper auth is implemented
 _UNION_EAGER_API_KEY_ENV_VAR = "_UNION_EAGER_API_KEY"
+_F_PATH_REWRITE = "_F_PATH_REWRITE"
 
 
 @click.group()
@@ -49,6 +54,8 @@ def _pass_through():
 @click.option("--project", envvar=PROJECT_NAME, required=False)
 @click.option("--domain", envvar=DOMAIN_NAME, required=False)
 @click.option("--org", envvar=ORG_NAME, required=False)
+@click.option("--debug", envvar=FLYTE_ENABLE_VSCODE_KEY, type=click.BOOL, required=False)
+@click.option("--interactive-mode", type=click.BOOL, required=False)
 @click.option("--image-cache", required=False)
 @click.option("--tgz", required=False)
 @click.option("--pkl", required=False)
@@ -59,12 +66,16 @@ def _pass_through():
     type=click.UNPROCESSED,
     nargs=-1,
 )
+@click.pass_context
 def main(
+    ctx: click.Context,
     run_name: str,
     name: str,
     project: str,
     domain: str,
     org: str,
+    debug: bool,
+    interactive_mode: bool,
     image_cache: str,
     version: str,
     inputs: str,
@@ -87,6 +98,7 @@ def main(
     import flyte
     import flyte._utils as utils
     import flyte.errors
+    import flyte.storage as storage
     from flyte._initialize import init
     from flyte._internal.controllers import create_controller
     from flyte._internal.imagebuild.image_builder import ImageCache
@@ -109,6 +121,13 @@ def main(
     if name.startswith("{{"):
         name = os.getenv("ACTION_NAME", "")
 
+    logger.warning(f"Flyte runtime started for action {name} with run name {run_name}")
+
+    if debug and name == "a0":
+        from flyte._debug.vscode import _start_vscode_server
+
+        asyncio.run(_start_vscode_server(ctx))
+
     # Figure out how to connect
     # This detection of api key is a hack for now.
     controller_kwargs: dict[str, Any] = {"insecure": False}
@@ -122,19 +141,40 @@ def main(
             controller_kwargs["insecure"] = True
         logger.debug(f"Using controller endpoint: {ep} with kwargs: {controller_kwargs}")
 
-    bundle = CodeBundle(tgz=tgz, pkl=pkl, destination=dest, computed_version=version)
-    init(org=org, project=project, domain=domain, **controller_kwargs)
+    # Check for insecure_skip_verify override (e.g. for self-signed certs)
+    insecure_skip_verify_str = os.getenv(INSECURE_SKIP_VERIFY_OVERRIDE, "")
+    if str2bool(insecure_skip_verify_str):
+        controller_kwargs["insecure_skip_verify"] = True
+        logger.info("SSL certificate verification disabled (insecure_skip_verify=True)")
+
+    bundle = None
+    if tgz or pkl:
+        bundle = CodeBundle(tgz=tgz, pkl=pkl, destination=dest, computed_version=version)
+    init(org=org, project=project, domain=domain, image_builder="remote", **controller_kwargs)
     # Controller is created with the same kwargs as init, so that it can be used to run tasks
     controller = create_controller(ct="remote", **controller_kwargs)
 
     ic = ImageCache.from_transport(image_cache) if image_cache else None
 
+    path_rewrite_cfg = os.getenv(_F_PATH_REWRITE, None)
+    path_rewrite = None
+    if path_rewrite_cfg:
+        potential_path_rewrite = PathRewrite.from_str(path_rewrite_cfg)
+        if storage.exists_sync(potential_path_rewrite.new_prefix):
+            path_rewrite = potential_path_rewrite
+            logger.info(f"Path rewrite configured for {path_rewrite.new_prefix}")
+        else:
+            logger.error(
+                f"Path rewrite failed for path {potential_path_rewrite.new_prefix}, "
+                f"not found, reverting to original path {potential_path_rewrite.old_prefix}"
+            )
+
     # Create a coroutine to load the task and run it
     task_coroutine = load_and_run_task(
         resolver=resolver,
         resolver_args=resolver_args,
         action=ActionID(name=name, run_name=run_name, project=project, domain=domain, org=org),
-        raw_data_path=RawDataPath(path=raw_data_path),
+        raw_data_path=RawDataPath(path=raw_data_path, path_rewrite=path_rewrite),
         checkpoints=Checkpoints(checkpoint_path, prev_checkpoint),
         code_bundle=bundle,
         input_path=inputs,
@@ -143,6 +183,7 @@ def main(
         version=version,
         controller=controller,
         image_cache=ic,
+        interactive_mode=interactive_mode or debug,
     )
     # Create a coroutine to watch for errors
     controller_failure = controller.watch_for_errors()
@@ -151,10 +192,23 @@ def main(
     async def _run_and_stop():
         loop = asyncio.get_event_loop()
         loop.set_exception_handler(flyte.errors.silence_grpc_polling_error)
-        await utils.run_coros(controller_failure, task_coroutine)
-        await controller.stop()
+        try:
+            await utils.run_coros(controller_failure, task_coroutine)
+            await controller.stop()
+        except flyte.errors.RuntimeSystemError as e:
+            logger.error(f"Runtime system error: {e}")
+            from flyte._internal.runtime.convert import convert_from_native_to_error
+            from flyte._internal.runtime.io import upload_error
+
+            logger.error(f"Flyte runtime failed for action {name} with run name {run_name}, error: {e}")
+            err = convert_from_native_to_error(e)
+            path = await upload_error(err.err, outputs_path)
+            logger.error(f"Run {run_name} Action {name} failed with error: {err}. Uploaded error to {path}")
+            await controller.stop()
+            raise
 
     asyncio.run(_run_and_stop())
+    logger.warning(f"Flyte runtime completed for action {name} with run name {run_name}")
 
 
 if __name__ == "__main__":

flyte/_cache/cache.py

@@ -77,14 +77,16 @@ class Cache:
     def __post_init__(self):
         if self.behavior not in get_args(CacheBehavior):
             raise ValueError(f"Invalid cache behavior: {self.behavior}. Must be one of ['auto', 'override', 'disable']")
-        if self.behavior == "disable":
-            return
 
+        # Still setup _ignore_inputs when cache is disabled to prevent _ignored_inputs attribute not found error
         if isinstance(self.ignored_inputs, str):
             self._ignored_inputs = (self.ignored_inputs,)
         else:
             self._ignored_inputs = self.ignored_inputs
 
+        if self.behavior == "disable":
+            return
+
         # Normalize policies so that self._policies is always a list
         if self.policies is None:
             from flyte._cache.defaults import get_default_policies

flyte/_cache/local_cache.py

@@ -0,0 +1,216 @@
+import sqlite3
+from pathlib import Path
+
+try:
+    import aiosqlite
+
+    HAS_AIOSQLITE = True
+except ImportError:
+    HAS_AIOSQLITE = False
+
+from flyteidl2.task import common_pb2
+
+from flyte._internal.runtime import convert
+from flyte._logging import logger
+from flyte.config import auto
+
+DEFAULT_CACHE_DIR = "~/.flyte"
+CACHE_LOCATION = "local-cache/cache.db"
+
+
+class LocalTaskCache(object):
+    """
+    This class implements a persistent store able to cache the result of local task executions.
+    """
+
+    _conn: "aiosqlite.Connection | None" = None
+    _conn_sync: sqlite3.Connection | None = None
+    _initialized: bool = False
+
+    @staticmethod
+    def _get_cache_path() -> str:
+        """Get the cache database path, creating directory if needed."""
+        config = auto()
+        if config.source:
+            cache_dir = config.source.parent
+        else:
+            cache_dir = Path(DEFAULT_CACHE_DIR).expanduser()
+
+        cache_path = cache_dir / CACHE_LOCATION
+        # Ensure the directory exists
+        cache_path.parent.mkdir(parents=True, exist_ok=True)
+        logger.info(f"Use local cache path: {cache_path}")
+        return str(cache_path)
+
+    @staticmethod
+    async def initialize():
+        """Initialize the cache with database connection."""
+        if not LocalTaskCache._initialized:
+            if HAS_AIOSQLITE:
+                await LocalTaskCache._initialize_async()
+            else:
+                LocalTaskCache._initialize_sync()
+
+    @staticmethod
+    async def _initialize_async():
+        """Initialize async cache connection."""
+        db_path = LocalTaskCache._get_cache_path()
+        conn = await aiosqlite.connect(db_path)
+        await conn.execute("""
+            CREATE TABLE IF NOT EXISTS task_cache (
+                key TEXT PRIMARY KEY,
+                value BLOB
+            )
+        """)
+        await conn.commit()
+        LocalTaskCache._conn = conn
+        LocalTaskCache._initialized = True
+
+    @staticmethod
+    def _initialize_sync():
+        """Initialize sync cache connection."""
+        db_path = LocalTaskCache._get_cache_path()
+        conn = sqlite3.connect(db_path)
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS task_cache (
+                key TEXT PRIMARY KEY,
+                value BLOB
+            )
+        """)
+        conn.commit()
+        LocalTaskCache._conn_sync = conn
+        LocalTaskCache._initialized = True
+
+    @staticmethod
+    async def clear():
+        """Clear all cache entries."""
+        if not LocalTaskCache._initialized:
+            await LocalTaskCache.initialize()
+
+        if HAS_AIOSQLITE:
+            await LocalTaskCache._clear_async()
+        else:
+            LocalTaskCache._clear_sync()
+
+    @staticmethod
+    async def _clear_async():
+        """Clear all cache entries (async)."""
+        if LocalTaskCache._conn is None:
+            raise RuntimeError("Cache not properly initialized")
+        await LocalTaskCache._conn.execute("DELETE FROM task_cache")
+        await LocalTaskCache._conn.commit()
+
+    @staticmethod
+    def _clear_sync():
+        """Clear all cache entries (sync)."""
+        if LocalTaskCache._conn_sync is None:
+            raise RuntimeError("Cache not properly initialized")
+        LocalTaskCache._conn_sync.execute("DELETE FROM task_cache")
+        LocalTaskCache._conn_sync.commit()
+
+    @staticmethod
+    async def get(cache_key: str) -> convert.Outputs | None:
+        if not LocalTaskCache._initialized:
+            await LocalTaskCache.initialize()
+
+        if HAS_AIOSQLITE:
+            return await LocalTaskCache._get_async(cache_key)
+        else:
+            return LocalTaskCache._get_sync(cache_key)
+
+    @staticmethod
+    async def _get_async(cache_key: str) -> convert.Outputs | None:
+        """Get cache entry (async)."""
+        if LocalTaskCache._conn is None:
+            raise RuntimeError("Cache not properly initialized")
+
+        async with LocalTaskCache._conn.execute("SELECT value FROM task_cache WHERE key = ?", (cache_key,)) as cursor:
+            row = await cursor.fetchone()
+            if row:
+                outputs_bytes = row[0]
+                outputs = common_pb2.Outputs()
+                outputs.ParseFromString(outputs_bytes)
+                return convert.Outputs(proto_outputs=outputs)
+        return None
+
+    @staticmethod
+    def _get_sync(cache_key: str) -> convert.Outputs | None:
+        """Get cache entry (sync)."""
+        if LocalTaskCache._conn_sync is None:
+            raise RuntimeError("Cache not properly initialized")
+
+        cursor = LocalTaskCache._conn_sync.execute("SELECT value FROM task_cache WHERE key = ?", (cache_key,))
+        row = cursor.fetchone()
+        if row:
+            outputs_bytes = row[0]
+            outputs = common_pb2.Outputs()
+            outputs.ParseFromString(outputs_bytes)
+            return convert.Outputs(proto_outputs=outputs)
+        return None
+
+    @staticmethod
+    async def set(
+        cache_key: str,
+        value: convert.Outputs,
+    ) -> None:
+        if not LocalTaskCache._initialized:
+            await LocalTaskCache.initialize()
+
+        if HAS_AIOSQLITE:
+            await LocalTaskCache._set_async(cache_key, value)
+        else:
+            LocalTaskCache._set_sync(cache_key, value)
+
+    @staticmethod
+    async def _set_async(
+        cache_key: str,
+        value: convert.Outputs,
+    ) -> None:
+        """Set cache entry (async)."""
+        if LocalTaskCache._conn is None:
+            raise RuntimeError("Cache not properly initialized")
+
+        output_bytes = value.proto_outputs.SerializeToString()
+        await LocalTaskCache._conn.execute(
+            "INSERT OR REPLACE INTO task_cache (key, value) VALUES (?, ?)", (cache_key, output_bytes)
+        )
+        await LocalTaskCache._conn.commit()
+
+    @staticmethod
+    def _set_sync(
+        cache_key: str,
+        value: convert.Outputs,
+    ) -> None:
+        """Set cache entry (sync)."""
+        if LocalTaskCache._conn_sync is None:
+            raise RuntimeError("Cache not properly initialized")
+
+        output_bytes = value.proto_outputs.SerializeToString()
+        LocalTaskCache._conn_sync.execute(
+            "INSERT OR REPLACE INTO task_cache (key, value) VALUES (?, ?)", (cache_key, output_bytes)
+        )
+        LocalTaskCache._conn_sync.commit()
+
+    @staticmethod
+    async def close():
+        """Close the database connection."""
+        if HAS_AIOSQLITE:
+            await LocalTaskCache._close_async()
+        else:
+            LocalTaskCache._close_sync()
+
+    @staticmethod
+    async def _close_async():
+        """Close async database connection."""
+        if LocalTaskCache._conn:
+            await LocalTaskCache._conn.close()
+            LocalTaskCache._conn = None
+        LocalTaskCache._initialized = False
+
+    @staticmethod
+    def _close_sync():
+        """Close sync database connection."""
+        if LocalTaskCache._conn_sync:
+            LocalTaskCache._conn_sync.close()
+            LocalTaskCache._conn_sync = None
+        LocalTaskCache._initialized = False

flyte/_code_bundle/_ignore.py

@@ -79,12 +79,19 @@ class StandardIgnore(Ignore):
     by fed with custom ignore patterns from cli."""
 
     def __init__(self, root: Path, patterns: Optional[List[str]] = None):
-        super().__init__(root)
+        super().__init__(root.resolve())
         self.patterns = patterns if patterns else STANDARD_IGNORE_PATTERNS
 
     def _is_ignored(self, path: pathlib.Path) -> bool:
+        # Convert to relative path for pattern matching
+        try:
+            rel_path = path.relative_to(self.root)
+        except ValueError:
+            # If path is not under root, don't ignore it
+            return False
+
         for pattern in self.patterns:
-            if fnmatch(str(path), pattern):
+            if fnmatch(str(rel_path), pattern):
                 return True
         return False
 
@@ -105,9 +112,10 @@ class IgnoreGroup(Ignore):
 
     def list_ignored(self) -> List[str]:
         ignored = []
-        for dir, _, files in self.root.walk():
+        for dir, _, files in os.walk(self.root):
+            dir_path = Path(dir)
             for file in files:
-                abs_path = dir / file
+                abs_path = dir_path / file
                 if self.is_ignored(abs_path):
                     ignored.append(str(abs_path.relative_to(self.root)))
         return ignored

flyte/_code_bundle/_packaging.py

@@ -14,10 +14,9 @@ import typing
 from typing import List, Optional, Tuple, Union
 
 import click
-from rich import print as rich_print
 from rich.tree import Tree
 
-from flyte._logging import logger
+from flyte._logging import _get_console, logger
 
 from ._ignore import Ignore, IgnoreGroup
 from ._utils import CopyFiles, _filehash_update, _pathhash_update, ls_files, tar_strip_file_attributes
@@ -27,21 +26,21 @@ FAST_FILEENDING = ".tar.gz"
 
 
 def print_ls_tree(source: os.PathLike, ls: typing.List[str]):
-    click.secho("Files to be copied for fast registration...", fg="bright_blue")
+    logger.info("Files to be copied for fast registration...")
 
     tree_root = Tree(
-        f":open_file_folder: [link file://{source}]{source} (detected source root)",
+        f"File structure:\n:open_file_folder: {source}",
         guide_style="bold bright_blue",
     )
-    trees = {pathlib.Path(source): tree_root}
-
+    source_path = pathlib.Path(source).resolve()
+    trees = {source_path: tree_root}
     for f in ls:
         fpp = pathlib.Path(f)
         if fpp.parent not in trees:
             # add trees for all intermediate folders
             current = tree_root
-            current_path = pathlib.Path(source)
-            for subdir in fpp.parent.relative_to(source).parts:
+            current_path = source_path  # pathlib.Path(source)
+            for subdir in fpp.parent.relative_to(source_path).parts:
                 current_path = current_path / subdir
                 if current_path not in trees:
                     current = current.add(f"{subdir}", guide_style="bold bright_blue")
@@ -49,7 +48,12 @@ def print_ls_tree(source: os.PathLike, ls: typing.List[str]):
                 else:
                     current = trees[current_path]
         trees[fpp.parent].add(f"{fpp.name}", guide_style="bold bright_blue")
-    rich_print(tree_root)
+
+    console = _get_console()
+    with console.capture() as capture:
+        console.print(tree_root, overflow="ignore", no_wrap=True, crop=False)
+    logger.info(f"Root directory: [link=file://{source}]{source}[/link]")
+    logger.info(capture.get(), extra={"console": console})
 
 
 def _compress_tarball(source: pathlib.Path, output: pathlib.Path) -> None:

flyte/_code_bundle/_utils.py

@@ -156,7 +156,7 @@ def list_all_files(source_path: pathlib.Path, deref_symlinks, ignore_group: Opti
 
     # This is needed to prevent infinite recursion when walking with followlinks
     visited_inodes = set()
-    for root, dirnames, files in source_path.walk(top_down=True, follow_symlinks=deref_symlinks):
+    for root, dirnames, files in os.walk(source_path, topdown=True, followlinks=deref_symlinks):
         dirnames[:] = [d for d in dirnames if d not in EXCLUDE_DIRS]
         if deref_symlinks:
             inode = os.stat(root).st_ino
@@ -167,7 +167,7 @@ def list_all_files(source_path: pathlib.Path, deref_symlinks, ignore_group: Opti
         ff = []
         files.sort()
         for fname in files:
-            abspath = (root / fname).absolute()
+            abspath = (pathlib.Path(root) / fname).absolute()
             # Only consider files that exist (e.g. disregard symlinks that point to non-existent files)
             if not os.path.exists(abspath):
                 logger.info(f"Skipping non-existent file {abspath}")
@@ -193,15 +193,15 @@ def list_all_files(source_path: pathlib.Path, deref_symlinks, ignore_group: Opti
 def _file_is_in_directory(file: str, directory: str) -> bool:
     """Return True if file is in directory and in its children."""
     try:
-        return os.path.commonpath([file, directory]) == directory
-    except ValueError as e:
-        # ValueError is raised by windows if the paths are not from the same drive
-        logger.debug(f"{file} and {directory} are not in the same drive: {e!s}")
+        return pathlib.Path(file).resolve().is_relative_to(pathlib.Path(directory).resolve())
+    except OSError as e:
+        # OSError can be raised if paths cannot be resolved (permissions, broken symlinks, etc.)
+        logger.debug(f"Failed to resolve paths for {file} and {directory}: {e!s}")
         return False
 
 
 def list_imported_modules_as_files(source_path: str, modules: List[ModuleType]) -> List[str]:
-    """Copies modules into destination that are in modules. The module files are copied only if:
+    """Lists the files of modules that have been loaded.  The files are only included if:
 
     1. Not a site-packages. These are installed packages and not user files.
     2. Not in the sys.base_prefix or sys.prefix. These are also installed and not user files.
@@ -211,7 +211,7 @@ def list_imported_modules_as_files(source_path: str, modules: List[ModuleType])
     import flyte
     from flyte._utils.lazy_module import is_imported
 
-    files = []
+    files = set()
     flyte_root = os.path.dirname(flyte.__file__)
 
     # These directories contain installed packages or modules from the Python standard library.
@@ -240,11 +240,19 @@ def list_imported_modules_as_files(source_path: str, modules: List[ModuleType])
 
         if not _file_is_in_directory(mod_file, source_path):
             # Only upload files where the module file in the source directory
+            # print log line for files that have common ancestor with source_path, but not in it.
+            logger.debug(f"{mod_file} is not in {source_path}")
+            continue
+
+        if not pathlib.Path(mod_file).is_file():
+            # Some modules have a __file__ attribute that are relative to the base package. Let's skip these,
+            # can add more rigorous logic to really pull out the correct file location if we need to.
+            logger.debug(f"Skipping {mod_file} from {mod.__name__} because it is not a file")
             continue
 
-        files.append(mod_file)
+        files.add(mod_file)
 
-    return files
+    return list(files)
 
 
 def add_imported_modules_from_source(source_path: str, destination: str, modules: List[ModuleType]):