fosslight-dependency 4.1.19__tar.gz → 4.1.21__tar.gz

{fosslight_dependency-4.1.19 → fosslight_dependency-4.1.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight_dependency
-Version: 4.1.19
+Version: 4.1.21
 Summary: FOSSLight Dependency Scanner
 Home-page: https://github.com/fosslight/fosslight_dependency_scanner
 Download-URL: https://github.com/fosslight/fosslight_dependency_scanner
@@ -51,7 +51,7 @@ In this user guide, you can see how to install the FOSSLight Dependency Scanner
 </thead>
 <tbody>
   <tr>
-    <td rowspan="2">Javascript</td>
+    <td rowspan="3">Javascript</td>
     <td>Npm</td>
     <td>package.json</td>
     <td>O</td>
@@ -64,6 +64,13 @@ In this user guide, you can see how to install the FOSSLight Dependency Scanner
     <td>O</td>
     <td>O</td>
     <td>O</td>
+  </tr>
+    <tr>
+    <td>Yarn</td>
+    <td>package.json</td>
+    <td>O</td>
+    <td>O</td>
+    <td>O</td>
   </tr>
   <tr>
     <td rowspan="2">Java</td>

{fosslight_dependency-4.1.19 → fosslight_dependency-4.1.21}/README.md

@@ -31,7 +31,7 @@ In this user guide, you can see how to install the FOSSLight Dependency Scanner
 </thead>
 <tbody>
   <tr>
-    <td rowspan="2">Javascript</td>
+    <td rowspan="3">Javascript</td>
     <td>Npm</td>
     <td>package.json</td>
     <td>O</td>
@@ -44,6 +44,13 @@ In this user guide, you can see how to install the FOSSLight Dependency Scanner
     <td>O</td>
     <td>O</td>
     <td>O</td>
+  </tr>
+    <tr>
+    <td>Yarn</td>
+    <td>package.json</td>
+    <td>O</td>
+    <td>O</td>
+    <td>O</td>
   </tr>
   <tr>
     <td rowspan="2">Java</td>

{fosslight_dependency-4.1.19 → fosslight_dependency-4.1.21}/setup.py

@@ -35,7 +35,7 @@ if __name__ == "__main__":
 
     setup(
         name=_PACKAEG_NAME,
-        version='4.1.19',
+        version='4.1.21',
         package_dir={"": "src"},
         packages=find_namespace_packages(where='src'),
         description='FOSSLight Dependency Scanner',

{fosslight_dependency-4.1.19 → fosslight_dependency-4.1.21}/src/fosslight_dependency/_analyze_dependency.py

@@ -35,7 +35,7 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
 
     if package_manager_name == const.PYPI:
         package_manager = Pypi(input_dir, output_dir, pip_activate_cmd, pip_deactivate_cmd)
-    elif package_manager_name == const.NPM:
+    elif package_manager_name == const.NPM or package_manager_name == const.YARN:
         package_manager = Npm(input_dir, output_dir)
     elif package_manager_name == const.MAVEN:
         package_manager = Maven(input_dir, output_dir, output_custom_dir)
@@ -92,11 +92,11 @@ def analyze_dependency(package_manager_name, input_dir, output_dir, pip_activate
             package_manager.parse_oss_information_for_pnpm()
             package_dep_item_list.extend(package_manager.dep_items)
     if ret:
-        logger.warning(f"### Complete to analyze: {package_manager_name}")
+        logger.warning(f"### Complete to analyze: {package_manager_name}({input_dir}: {','.join(manifest_file_name)})")
         if package_manager.cover_comment:
             cover_comment = package_manager.cover_comment
     else:
-        logger.error(f"### Fail to analyze: {package_manager_name}")
+        logger.error(f"### Fail to analyze: {package_manager_name}({input_dir}: {','.join(manifest_file_name)})")
 
     del package_manager
 

{fosslight_dependency-4.1.19 → fosslight_dependency-4.1.21}/src/fosslight_dependency/_help.py

@@ -16,6 +16,7 @@ _HELP_MESSAGE_DEPENDENCY = """
         Maven (Java)
         NPM (Node.js)
         PNPM (Node.js)
+        Yarn (Node.js)
         PIP (Python)
         Pub (Dart with flutter)
         Cocoapods (Swift/Obj-C)
@@ -33,7 +34,7 @@ _HELP_MESSAGE_DEPENDENCY = """
             -v\t\t\t\t    Print the version of the script.
             -m <package_manager>\t    Enter the package manager.
                                         \t(npm, maven, gradle, pypi, pub, cocoapods, android, swift, carthage,
-                                        \t go, nuget, helm, unity, cargo, pnpm)
+                                        \t go, nuget, helm, unity, cargo, pnpm, yarn)
             -p <input_path>\t\t    Enter the path where the script will be run.
             -e <exclude_path>\t\t    Enter the path where the analysis will not be performed.
             -o <output_path>\t\t    Output path
@@ -46,6 +47,7 @@ _HELP_MESSAGE_DEPENDENCY = """
             \t\t\t\t\t--graph-path option is required
             --direct\t\t\t    Print the direct/transitive dependency type in comment.
                                 \t\tChoice 'True' or 'False'. (default:True)
+            -r\t\t\t\t    Recursive mode. Scan all subdirectories for manifest files.
             --notice\t\t\t    Print the open source license notice text.
 
         Required only for swift, carthage

{fosslight_dependency-4.1.19 → fosslight_dependency-4.1.21}/src/fosslight_dependency/_package_manager.py

@@ -108,6 +108,7 @@ class PackageManager:
                     cmd_gradle = "./gradlew"
             else:
                 ret_task = False
+                self.set_direct_dependencies(False)
                 logger.warning('No gradlew file exists (Skip to find dependencies relationship.).')
                 if ret_plugin:
                     logger.warning('Also it cannot run android-dependency-scanning plugin.')
@@ -153,6 +154,7 @@ class PackageManager:
                     shutil.move(module_gradle_backup, module_build_gradle)
         if os.path.isfile(self.input_file_name):
             logger.info(f'Found {self.input_file_name}, skip to run plugin.')
+            self.set_direct_dependencies(False)
             ret_task = True
         return ret_task
 

{fosslight_dependency-4.1.19 → fosslight_dependency-4.1.21}/src/fosslight_dependency/constant.py

@@ -25,6 +25,7 @@ HELM = 'helm'
 UNITY = 'unity'
 CARGO = 'cargo'
 PNPM = 'pnpm'
+YARN = 'yarn'
 
 # Supported package name and manifest file
 SUPPORT_PACKAE = {

{fosslight_dependency-4.1.19 → fosslight_dependency-4.1.21}/src/fosslight_dependency/package_manager/Npm.py

@@ -31,12 +31,12 @@ class Npm(PackageManager):
         super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
 
     def __del__(self):
-        if os.path.isfile(self.input_file_name):
-            os.remove(self.input_file_name)
+        if os.path.isfile(os.path.join(self.input_dir, self.input_file_name)):
+            os.remove(os.path.join(self.input_dir, self.input_file_name))
         if self.flag_tmp_node_modules:
-            shutil.rmtree(node_modules, ignore_errors=True)
-        if os.path.exists(self.tmp_custom_json):
-            os.remove(self.tmp_custom_json)
+            shutil.rmtree(os.path.join(self.input_dir, node_modules), ignore_errors=True)
+        if os.path.exists(os.path.join(self.input_dir, self.tmp_custom_json)):
+            os.remove(os.path.join(self.input_dir, self.tmp_custom_json))
 
     def run_plugin(self):
         ret = self.start_license_checker()
@@ -53,8 +53,14 @@ class Npm(PackageManager):
             self.flag_tmp_node_modules = True
             cmd_ret = subprocess.call(npm_install_cmd, shell=True)
             if cmd_ret != 0:
-                logger.error(f"{npm_install_cmd} returns an error")
-                return False
+                logger.warning(f"{npm_install_cmd} returns an error. Trying yarn as fallback...")
+                yarn_install_cmd = 'yarn install --production --ignore-scripts'
+                cmd_ret = subprocess.call(yarn_install_cmd, shell=True)
+                if cmd_ret != 0:
+                    logger.error(f"Both {npm_install_cmd} and {yarn_install_cmd} failed")
+                    return False
+                else:
+                    logger.info(f"Successfully executed {yarn_install_cmd}")
 
         # customized json file for obtaining specific items with license-checker
         self.make_custom_json(self.tmp_custom_json)
@@ -122,9 +128,9 @@ class Npm(PackageManager):
                 if len(rel_json) < 1:
                     ret = False
                 else:
-                    self.package_name = f'{rel_json[_name]}({rel_json[_version]})'
+                    self.package_name = f'{rel_json[_name]}({rel_json.get(_version, "")})'
                     if _dependencies in rel_json:
-                        self.parse_rel_dependencies(rel_json[_name], rel_json[_version], rel_json[_dependencies])
+                        self.parse_rel_dependencies(rel_json[_name], rel_json.get(_version, ""), rel_json[_dependencies])
             except Exception as e:
                 ret = False
                 err_msg = e
@@ -178,7 +184,7 @@ class Npm(PackageManager):
 
             oss_item.download_location = f"{self.dn_url}{oss_init_name}/v/{oss_item.version}"
             dn_loc = f"{self.dn_url}{oss_init_name}"
-            dep_item.purl = get_url_to_purl(dn_loc, self.package_manager_name)
+            dep_item.purl = get_url_to_purl(oss_item.download_location, self.package_manager_name)
             purl_dict[f'{oss_init_name}({oss_item.version})'] = dep_item.purl
             if d[_repository]:
                 dn_loc = d[_repository]

{fosslight_dependency-4.1.19 → fosslight_dependency-4.1.21}/src/fosslight_dependency/package_manager/Pypi.py

@@ -13,7 +13,7 @@ import re
 import fosslight_util.constant as constant
 import fosslight_dependency.constant as const
 from fosslight_dependency._package_manager import PackageManager
-from fosslight_dependency._package_manager import check_license_name, get_url_to_purl
+from fosslight_dependency._package_manager import get_url_to_purl, check_license_name
 from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
 from fosslight_util.oss_item import OssItem
 
@@ -26,7 +26,6 @@ class Pypi(PackageManager):
     dn_url = 'https://pypi.org/project/'
     venv_tmp_dir = 'venv_osc_dep_tmp'
     tmp_file_name = "tmp_pip_license_output.json"
-    tmp_pip_license_info_file_name = "tmp_pip_license_info_output.json"
     tmp_deptree_file = "tmp_pipdeptree.json"
     pip_activate_cmd = ''
     pip_deactivate_cmd = ''
@@ -41,9 +40,6 @@ class Pypi(PackageManager):
         if os.path.isfile(self.tmp_file_name):
             os.remove(self.tmp_file_name)
 
-        if os.path.isfile(self.tmp_pip_license_info_file_name):
-            os.remove(self.tmp_pip_license_info_file_name)
-
         shutil.rmtree(self.venv_tmp_dir, ignore_errors=True)
 
         if os.path.isfile(self.tmp_deptree_file):
@@ -73,7 +69,7 @@ class Pypi(PackageManager):
             ret = self.create_virtualenv()
 
         if ret:
-            ret = self.start_pip_licenses()
+            ret = self.start_pip_inspect()
 
         return ret
 
@@ -103,7 +99,7 @@ class Pypi(PackageManager):
             activate_cmd = os.path.join(self.venv_tmp_dir, "Scripts", "activate.bat")
             cmd_separator = "&"
         else:
-            create_venv_cmd = f"python3 -m venv {self.venv_tmp_dir}"
+            create_venv_cmd = f"virtualenv -p python3 {self.venv_tmp_dir}"
             activate_cmd = ". " + os.path.join(venv_path, "bin", "activate")
             cmd_separator = ";"
 
@@ -117,11 +113,12 @@ class Pypi(PackageManager):
             return False
 
         deactivate_cmd = "deactivate"
+        pip_upgrade_cmd = "pip install --upgrade pip"
 
         self.set_pip_activate_cmd(activate_cmd)
         self.set_pip_deactivate_cmd(deactivate_cmd)
 
-        cmd_list = [create_venv_cmd, activate_cmd, install_cmd, deactivate_cmd]
+        cmd_list = [create_venv_cmd, activate_cmd, install_cmd, pip_upgrade_cmd, deactivate_cmd]
         cmd = cmd_separator.join(cmd_list)
 
         try:
@@ -139,9 +136,9 @@ class Pypi(PackageManager):
             try:
                 if (not ret) and (self.platform != const.WINDOWS):
                     ret = True
-                    create_venv_cmd = f"virtualenv -p python3 {self.venv_tmp_dir}"
+                    create_venv_cmd = f"python3 -m venv {self.venv_tmp_dir}"
 
-                    cmd_list = [create_venv_cmd, activate_cmd, install_cmd, deactivate_cmd]
+                    cmd_list = [create_venv_cmd, activate_cmd, install_cmd, pip_upgrade_cmd, deactivate_cmd]
                     cmd = cmd_separator.join(cmd_list)
                     cmd_ret = subprocess.run(cmd, shell=True, stderr=subprocess.PIPE)
                     if cmd_ret.returncode != 0:
@@ -160,14 +157,9 @@ class Pypi(PackageManager):
 
         return ret
 
-    def start_pip_licenses(self):
+    def start_pip_inspect(self):
         ret = True
-        pip_licenses = 'pip-licenses'
-        prettytable = 'prettytable'
-        wcwidth = 'wcwidth'
         pipdeptree = 'pipdeptree'
-        pip_licenses_default_options = ' --from=mixed --with-url --format=json --with-license-file'
-        pip_licenses_system_option = ' --with-system -p '
         tmp_pip_list = "tmp_list.txt"
         python_cmd = "python -m"
 
@@ -193,11 +185,21 @@ class Pypi(PackageManager):
         command_list = [activate_command, pip_list_command, deactivate_command]
         command = command_separator.join(command_list)
 
+        exists_pipdeptree = False
         try:
             cmd_ret = subprocess.call(command, shell=True)
             if cmd_ret != 0:
                 ret = False
                 err_msg = f"cmd ret code({cmd_ret})"
+            else:
+                if os.path.isfile(tmp_pip_list):
+                    with open(tmp_pip_list, 'r', encoding='utf-8') as pip_list_file:
+                        for pip_list in pip_list_file.readlines():
+                            pip_list_name = pip_list.split('==')[0]
+                            if pip_list_name == pipdeptree:
+                                exists_pipdeptree = True
+                                break
+                    os.remove(tmp_pip_list)
         except Exception as e:
             ret = False
             err_msg = str(e)
@@ -206,63 +208,11 @@ class Pypi(PackageManager):
                 logger.error(f"Failed to freeze dependencies ({command}): {err_msg})")
                 return False
 
-        exists_pip_licenses = False
-        exists_prettytable = False
-        exists_wcwidth = False
-        pip_license_pkg_list = []
-        uninstall_pkg_list = []
-        exists_pipdeptree = False
-
-        if os.path.isfile(tmp_pip_list):
-            try:
-                with open(tmp_pip_list, 'r', encoding='utf-8') as pip_list_file:
-                    for pip_list in pip_list_file.readlines():
-                        pip_list_name = pip_list.split('==')[0]
-                        if pip_list_name == pip_licenses:
-                            exists_pip_licenses = True
-                        if pip_list_name == prettytable:
-                            exists_prettytable = True
-                        if pip_list_name == wcwidth:
-                            exists_wcwidth = True
-                        if pip_list_name == pipdeptree:
-                            exists_pipdeptree = True
-                os.remove(tmp_pip_list)
-            except Exception as e:
-                logger.error(f"Failed to read freezed package list file: {e}")
-                return False
-        if exists_pip_licenses:
-            pip_license_pkg_list.append(pip_licenses)
-        else:
-            uninstall_pkg_list.append(pip_licenses)
-        if exists_prettytable:
-            pip_license_pkg_list.append(prettytable)
-        else:
-            uninstall_pkg_list.append(prettytable)
-        if exists_wcwidth:
-            pip_license_pkg_list.append(wcwidth)
-        else:
-            uninstall_pkg_list.append(wcwidth)
-
         command_list = []
         command_list.append(activate_command)
-        if not exists_pip_licenses:
-            install_pip_command = f"{python_cmd} pip install {pip_licenses}"
-            command_list.append(install_pip_command)
-
-        pip_licenses_command = f"{pip_licenses}{pip_licenses_default_options} > {self.tmp_file_name}"
-        command_list.append(pip_licenses_command)
-
-        if len(pip_license_pkg_list) != 0:
-            pip_licenses_info_command = f"{pip_licenses}{pip_licenses_default_options}{pip_licenses_system_option}"
-            pip_licenses_info_command += " ".join(pip_license_pkg_list)
-
-            pip_licenses_info_command += f" > {self.tmp_pip_license_info_file_name}"
-            command_list.append(pip_licenses_info_command)
 
-        if len(uninstall_pkg_list) > 0:
-            uninstall_pip_command = f"{python_cmd} pip uninstall -y "
-            uninstall_pip_command += ' '.join(uninstall_pkg_list)
-            command_list.append(uninstall_pip_command)
+        pip_inspect_command = f"{python_cmd} pip inspect > {self.tmp_file_name}"
+        command_list.append(pip_inspect_command)
 
         if not exists_pipdeptree:
             install_deptree_command = f"{python_cmd} pip install {pipdeptree}"
@@ -270,30 +220,37 @@ class Pypi(PackageManager):
             uninstall_deptree_command = f"{python_cmd} pip uninstall -y {pipdeptree}"
         pipdeptree_command = f"{pipdeptree} --json-tree -e 'pipdeptree,pip,wheel,setuptools' > {self.tmp_deptree_file}"
         command_list.append(pipdeptree_command)
-        command_list.append(uninstall_deptree_command)
+
+        if not exists_pipdeptree:
+            command_list.append(uninstall_deptree_command)
+
         command_list.append(deactivate_command)
         command = command_separator.join(command_list)
 
         try:
             cmd_ret = subprocess.call(command, shell=True)
             if cmd_ret == 0:
-                self.append_input_package_list_file(self.tmp_file_name)
-                with open(self.tmp_file_name, 'r', encoding='utf-8') as json_f:
-                    json_data = json.load(json_f)
-                    for d in json_data:
-                        self.total_dep_list.append(re.sub(r"[-_.]+", "-", d['Name']).lower())
-                if len(pip_license_pkg_list) != 0:
-                    self.append_input_package_list_file(self.tmp_pip_license_info_file_name)
-                    with open(self.tmp_pip_license_info_file_name, 'r', encoding='utf-8') as json_f:
-                        json_data = json.load(json_f)
-                        for d in json_data:
-                            self.total_dep_list.append(re.sub(r"[-_.]+", "-", d['Name']).lower())
+                if os.path.exists(self.tmp_file_name):
+                    self.append_input_package_list_file(self.tmp_file_name)
+
+                    with open(self.tmp_file_name, 'r', encoding='utf-8') as json_f:
+                        inspect_data = json.load(json_f)
+                        for package in inspect_data.get('installed', []):
+                            metadata = package.get('metadata', {})
+                            package_name = metadata.get('name', '')
+                            if package_name:
+                                if package_name in ['pip', 'setuptools']:
+                                    continue
+                                self.total_dep_list.append(re.sub(r"[-_.]+", "-", package_name).lower())
+                else:
+                    logger.error(f"pip inspect output file not found: {self.tmp_file_name}")
+                    ret = False
             else:
                 logger.error(f"Failed to run command: {command}")
                 ret = False
         except Exception as e:
             ret = False
-            logger.error(f"Failed to install/uninstall pypi packages: {e}")
+            logger.error(f"Failed to get package information using pip inspect: {e}")
 
         return ret
 
@@ -302,25 +259,70 @@ class Pypi(PackageManager):
         try:
             oss_init_name = ''
             with open(f_name, 'r', encoding='utf-8') as json_file:
-                json_data = json.load(json_file)
+                inspect_data = json.load(json_file)
 
-            for d in json_data:
+            for package in inspect_data.get('installed', []):
                 dep_item = DependencyItem()
                 oss_item = OssItem()
-                oss_init_name = d['Name']
+                metadata = package.get('metadata', {})
+                if not metadata:
+                    continue
+
+                oss_init_name = metadata.get('name', '')
                 oss_init_name = re.sub(r"[-_.]+", "-", oss_init_name).lower()
+                if oss_init_name not in self.total_dep_list:
+                    continue
                 oss_item.name = f"{self.package_manager_name}:{oss_init_name}"
-                license_name = check_UNKNOWN(d['License'])
-                oss_item.homepage = check_UNKNOWN(d['URL'])
-                oss_item.version = d['Version']
+                oss_item.version = metadata.get('version', '')
+
+                # license_expression > license > classifier
+                license_info = check_UNKNOWN(metadata.get('license_expression', ''))
+                if not license_info:
+                    license_info = metadata.get('license', '')
+                    if '\n' in license_info:
+                        license_info = check_UNKNOWN(check_license_name(license_info))
+                if not license_info:
+                    classifiers = metadata.get('classifier', [])
+                    license_classifiers = [c for c in classifiers if c.startswith('License ::')]
+                    if license_classifiers:
+                        license_info_l = []
+                        for license_classifier in license_classifiers:
+                            if license_classifier.startswith('License :: OSI Approved ::'):
+                                license_info_l.append(license_classifier.split('::')[-1].strip())
+                                break
+                        license_info = ','.join(license_info_l)
+                license_name = check_UNKNOWN(license_info)
+                if license_name:
+                    license_name = license_name.replace(';', ',')
+                oss_item.license = license_name
+
                 oss_item.download_location = f"{self.dn_url}{oss_init_name}/{oss_item.version}"
+
+                # project_url 'source' > download_url > project_url 'homepage' > homepage
+                homepage_url = check_UNKNOWN(metadata.get('home_page', ''))
+                download_url = check_UNKNOWN(metadata.get('download_url', ''))
+                project_urls = check_UNKNOWN(metadata.get('project_url', []))
+                if project_urls:
+                    priority_order = ['source', 'repository', 'github', 'code', 'source code', 'homepage']
+                    for priority in priority_order:
+                        for url_entry in project_urls:
+                            url_entry_lower = url_entry.lower()
+                            if url_entry_lower.startswith(priority):
+                                download_url = url_entry.split(', ')[-1]
+                                break
+                        if download_url:
+                            break
+                oss_item.homepage = download_url or homepage_url or f"{self.dn_url}{oss_init_name}"
+
                 dep_item.purl = get_url_to_purl(oss_item.download_location, self.package_manager_name)
                 purl_dict[f'{oss_init_name}({oss_item.version})'] = dep_item.purl
-                if license_name is not None:
-                    license_name = license_name.replace(';', ',')
-                else:
-                    license_name = check_license_name(d['LicenseFile'], True)
-                oss_item.license = license_name
+
+                direct_url = package.get('direct_url', {})
+                if direct_url:
+                    oss_item.download_location = direct_url.get('url', '')
+                    oss_item.homepage = oss_item.download_location
+                if not package.get('installer', ''):
+                    oss_item.download_location = oss_item.homepage
 
                 if oss_init_name == self.package_name:
                     oss_item.comment = 'root package'

{fosslight_dependency-4.1.19 → fosslight_dependency-4.1.21}/src/fosslight_dependency/run_dependency_scanner.py

@@ -30,7 +30,7 @@ EXTENDED_HEADER = {_sheet_name: ['ID', 'Package URL', 'OSS Name',
                                        'OSS Version', 'License', 'Download Location',
                                        'Homepage', 'Copyright Text', 'Exclude',
                                        'Comment', 'Depends On']}
-_exclude_dir = ['node_moduels', 'venv']
+_exclude_dir = ['node_modules', 'venv', 'Pods', 'Carthage']
 
 
 def get_terminal_size():
@@ -50,7 +50,7 @@ def paginate_file(file_path):
             input("Press Enter to see the next page...")
 
 
-def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[]):
+def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[], recursive=False):
     ret = True
     if not manifest_file_name:
         for value in const.SUPPORT_PACKAE.values():
@@ -62,9 +62,8 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
     found_manifest_file = []
     suggested_files = []
     for parent, dirs, files in os.walk(input_dir):
-        if len(files) < 1:
-            continue
-        if os.path.basename(parent) in _exclude_dir:
+        parent_parts = parent.split(os.sep)
+        if any(ex_dir in parent_parts for ex_dir in _exclude_dir):
             continue
         if os.path.abspath(parent) in abs_path_to_exclude:
             continue
@@ -75,7 +74,8 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
                    for exclude_path in abs_path_to_exclude):
                 continue
             if file in manifest_file_name:
-                found_manifest_file.append(file)
+                path_with_filename = os.path.join(parent, file)
+                found_manifest_file.append(path_with_filename)
             if file in const.SUGGESTED_PACKAGE.keys():
                 suggested_files.append(os.path.join(parent, file))
         for dir in dirs:
@@ -84,31 +84,32 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
                 if len(manifest_l) > 1:
                     if manifest_l[0] == dir:
                         if os.path.exists(os.path.join(parent, manifest_f)):
-                            found_manifest_file.append(manifest_f)
-        if len(found_manifest_file) > 0:
-            input_dir = parent
-            break
-    found_package_manager = defaultdict(list)
-    for f_idx in found_manifest_file:
+                            found_manifest_file.append(os.path.join(parent, manifest_f))
+        if not recursive:
+            if len(found_manifest_file) > 0:
+                input_dir = parent
+                break
+
+    found_package_manager = defaultdict(lambda: defaultdict(list))
+    for f_with_path in found_manifest_file:
+        f_name = os.path.basename(f_with_path)
+        dir_path = os.path.dirname(f_with_path)
         for key, value in const.SUPPORT_PACKAE.items():
             if isinstance(value, list):
-                for v in value:
-                    if f_idx == v:
-                        if key in found_package_manager.keys():
-                            found_package_manager[key].append(f_idx)
-                        else:
-                            found_package_manager[key] = [f_idx]
+                if f_name in value:
+                    found_package_manager[key][dir_path].append(f_name)
             else:
-                if value == f_idx:
-                    found_package_manager[key] = [f_idx]
+                if f_name == value:
+                    found_package_manager[key][dir_path].append(f_name)
+    found_package_manager = {k: dict(v) for k, v in found_package_manager.items()}
 
     # both npm and pnpm are detected, remove npm.
-    if 'npm' in found_package_manager and 'pnpm' in found_package_manager:
+    if 'npm' in found_package_manager.keys() and 'pnpm' in found_package_manager.keys():
         del found_package_manager['npm']
     if len(found_package_manager) >= 1:
-        manifest_file_w_path = [os.path.join(input_dir, file) for pkg, files in found_package_manager.items() for file in files]
-        logger.info(f"Found the manifest file({','.join(manifest_file_w_path)}) automatically.")
-        logger.warning(f"### Set Package Manager = {', '.join(found_package_manager.keys())}")
+        log_lines = ["\nDetected Manifest Files automatically"]
+        log_lines = print_package_info(found_package_manager, log_lines)
+        logger.info('\n'.join(log_lines))
     else:
         ret = False
         logger.info("Cannot find the manifest file.")
@@ -116,10 +117,20 @@ def find_package_manager(input_dir, abs_path_to_exclude=[], manifest_file_name=[
     return ret, found_package_manager, input_dir, suggested_files
 
 
+def print_package_info(success_pm, log_lines):
+    if success_pm:
+        for pm, dir_dict in success_pm.items():
+            log_lines.append(f"- {pm}:")
+            for path, files in dir_dict.items():
+                file_list = ', '.join(files)
+                log_lines.append(f"  {path}: {file_list}")
+    return log_lines
+
+
 def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='', pip_activate_cmd='',
                            pip_deactivate_cmd='', output_custom_dir='', app_name=const.default_app_name,
                            github_token='', formats=[], direct=True, path_to_exclude=[], graph_path='',
-                           graph_size=(600, 600)):
+                           graph_size=(600, 600), recursive=False):
     global logger
 
     ret = True
@@ -196,6 +207,9 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='',
     autodetect = True
     found_package_manager = {}
     if package_manager:
+        scan_item.set_cover_comment(f"Manual detect mode (-m {package_manager})")
+        if package_manager == const.YARN:
+            package_manager = const.NPM
         autodetect = False
         support_packagemanager = list(const.SUPPORT_PACKAE.keys())
 
@@ -210,16 +224,14 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='',
             manifest_file_name.extend(value)
         else:
             manifest_file_name.append(value)
-        scan_item.set_cover_comment(f"Manual detect mode (-m {package_manager})")
     else:
         manifest_file_name = []
 
     try:
         ret, found_package_manager, input_dir, suggested_files = find_package_manager(input_dir,
                                                                                       abs_path_to_exclude,
-                                                                                      manifest_file_name)
-        if ret:
-            os.chdir(input_dir)
+                                                                                      manifest_file_name,
+                                                                                      recursive)
     except Exception as e:
         if autodetect:
             logger.error(f'Fail to find package manager: {e}')
@@ -228,7 +240,7 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='',
         if not ret:
             if not autodetect:
                 logger.info('Try to analyze dependency without manifest file. (Manual mode)')
-                found_package_manager[package_manager] = []
+                found_package_manager[package_manager] = {}
             else:
                 ret = False
                 if suggested_files:
@@ -243,40 +255,64 @@ def run_dependency_scanner(package_manager='', input_dir='', output_dir_file='',
                 else:
                     scan_item.set_cover_comment("No Package manager detected.")
 
-    pass_key = 'PASS'
-    success_pm = []
-    fail_pm = []
+    pass_key = ['PASS']
+    success_pm = defaultdict(lambda: defaultdict(list))
+    fail_pm = defaultdict(lambda: defaultdict(list))
     cover_comment = ''
-    for pm, manifest_file_name in found_package_manager.items():
-        if manifest_file_name == pass_key:
-            continue
-        ret, package_dep_item_list, cover_comment = analyze_dependency(pm, input_dir, output_path,
-                                                                       pip_activate_cmd, pip_deactivate_cmd,
-                                                                       output_custom_dir, app_name, github_token,
-                                                                       manifest_file_name, direct)
-        if ret:
-            success_pm.append(f"{pm} ({', '.join(manifest_file_name)})")
-            scan_item.append_file_items(package_dep_item_list)
-            if pm == const.GRADLE:
-                if const.ANDROID in found_package_manager.keys():
-                    found_package_manager[const.ANDROID] = pass_key
-                    if f"{const.ANDROID} ({', '.join(manifest_file_name)})" in fail_pm:
-                        fail_pm.remove(f"{const.ANDROID} ({', '.join(manifest_file_name)})")
-            elif pm == const.ANDROID:
-                if const.GRADLE in found_package_manager.keys():
-                    found_package_manager[const.GRADLE] = pass_key
-                    if f"{const.GRADLE} ({', '.join(manifest_file_name)})" in fail_pm:
-                        fail_pm.remove(f"{const.GRADLE} ({', '.join(manifest_file_name)})")
+    for pm, manifest_file_name_list in found_package_manager.items():
+        if not manifest_file_name_list and not autodetect:
+            ret, package_dep_item_list, cover_comment = analyze_dependency(pm, input_dir, output_path,
+                                                                           pip_activate_cmd, pip_deactivate_cmd,
+                                                                           output_custom_dir, app_name, github_token,
+                                                                           [], direct)
+            if ret:
+                success_pm[pm][input_dir].extend(['manual mode (-m option)'])
+                scan_item.append_file_items(package_dep_item_list)
+            else:
+                fail_pm[pm][input_dir].extend(['manual mode (-m option)'])
         else:
-            fail_pm.append(f"{pm} ({', '.join(manifest_file_name)})")
+            for manifest_dir, manifest_file_name in manifest_file_name_list.items():
+                input_dir = manifest_dir
+                if manifest_file_name == pass_key:
+                    continue
+                os.chdir(input_dir)
+                ret, package_dep_item_list, cover_comment = analyze_dependency(pm, input_dir, output_path,
+                                                                               pip_activate_cmd, pip_deactivate_cmd,
+                                                                               output_custom_dir, app_name, github_token,
+                                                                               manifest_file_name, direct)
+                if ret:
+                    success_pm[pm][input_dir].extend(manifest_file_name)
+                    scan_item.append_file_items(package_dep_item_list)
+
+                    dup_pm = None
+                    if pm == const.GRADLE and const.ANDROID in found_package_manager:
+                        dup_pm = const.ANDROID
+                    elif pm == const.ANDROID and const.GRADLE in found_package_manager:
+                        dup_pm = const.GRADLE
+
+                    if dup_pm:
+                        if dup_pm in fail_pm and input_dir in fail_pm[dup_pm]:
+                            fail_pm[dup_pm].pop(input_dir, None)
+                            if not fail_pm[dup_pm]:
+                                fail_pm.pop(dup_pm, None)
+                        else:
+                            found_package_manager[dup_pm][manifest_dir] = pass_key
+                else:
+                    fail_pm[pm][input_dir].extend(manifest_file_name)
 
+    success_pm = {k: dict(v) for k, v in success_pm.items()}
+    fail_pm = {k: dict(v) for k, v in fail_pm.items()}
     if len(found_package_manager.keys()) > 0:
         if len(success_pm) > 0:
-            scan_item.set_cover_comment(f"Analyzed Package manager: {', '.join(success_pm)}")
+            log_lines = ["Success to analyze:"]
+            log_lines = print_package_info(success_pm, log_lines)
+            scan_item.set_cover_comment('\n'.join(log_lines))
         if len(fail_pm) > 0:
-            info_msg = 'Check log file(fosslight_log*.txt) ' \
-                       'and https://fosslight.org/fosslight-guide-en/scanner/3_dependency.html#-prerequisite.'
-            scan_item.set_cover_comment(f"Analysis failed Package manager: {', '.join(fail_pm)} ({info_msg})")
+            log_lines = ["Fail to analyze:"]
+            log_lines = print_package_info(fail_pm, log_lines)
+            scan_item.set_cover_comment('\n'.join(log_lines))
+            scan_item.set_cover_comment('Check log file(fosslight_log*.txt) '
+                                        'and https://fosslight.org/fosslight-guide-en/scanner/3_dependency.html#-prerequisite.')
 
     if ret and graph_path:
         graph_path = os.path.abspath(graph_path)
@@ -335,6 +371,7 @@ def main():
     graph_path = ''
     graph_size = (600, 600)
     direct = True
+    recursive = False
 
     parser = argparse.ArgumentParser(add_help=False)
     parser.add_argument('-h', '--help', action='store_true', required=False)
@@ -353,6 +390,7 @@ def main():
     parser.add_argument('--graph-size', nargs=2, type=int, metavar=("WIDTH", "HEIGHT"), required=False)
     parser.add_argument('--direct', choices=('true', 'false'), default='True', required=False)
     parser.add_argument('--notice', action='store_true', required=False)
+    parser.add_argument('-r', '--recursive', action='store_true', required=False)
 
     args = parser.parse_args()
 
@@ -405,10 +443,12 @@ def main():
             paginate_file(source_file)
             shutil.copyfile(source_file, destination_file)
         sys.exit(0)
+    if args.recursive:  # -r option
+        recursive = True
 
     run_dependency_scanner(package_manager, input_dir, output_dir, pip_activate_cmd, pip_deactivate_cmd,
                            output_custom_dir, app_name, github_token, format, direct, path_to_exclude,
-                           graph_path, graph_size)
+                           graph_path, graph_size, recursive)
 
 
 if __name__ == '__main__':

{fosslight_dependency-4.1.19 → fosslight_dependency-4.1.21}/src/fosslight_dependency.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-dependency
-Version: 4.1.19
+Version: 4.1.21
 Summary: FOSSLight Dependency Scanner
 Home-page: https://github.com/fosslight/fosslight_dependency_scanner
 Download-URL: https://github.com/fosslight/fosslight_dependency_scanner
@@ -51,7 +51,7 @@ In this user guide, you can see how to install the FOSSLight Dependency Scanner
 </thead>
 <tbody>
   <tr>
-    <td rowspan="2">Javascript</td>
+    <td rowspan="3">Javascript</td>
     <td>Npm</td>
     <td>package.json</td>
     <td>O</td>
@@ -64,6 +64,13 @@ In this user guide, you can see how to install the FOSSLight Dependency Scanner
     <td>O</td>
     <td>O</td>
     <td>O</td>
+  </tr>
+    <tr>
+    <td>Yarn</td>
+    <td>package.json</td>
+    <td>O</td>
+    <td>O</td>
+    <td>O</td>
   </tr>
   <tr>
     <td rowspan="2">Java</td>