panopticas 0.0.9__py3-none-any.whl

panopticas-0.0.9.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Peter Freiberg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

panopticas-0.0.9.dist-info/METADATA

@@ -0,0 +1,72 @@
+Metadata-Version: 2.2
+Name: panopticas
+Version: 0.0.9
+Summary: The panopticas CLI tool.
+Author-email: Peter Freiberg <peter.freiberg@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2024 Peter Freiberg
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://panopticas.io
+Project-URL: Issues, https://github.com/kospex/panopticas/issues
+Project-URL: Repository, https://github.com/kospex/panopticas.git
+Keywords: analytics,git
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: Click
+Requires-Dist: prettytable
+Requires-Dist: pathspec
+
+# panopticas
+
+Discover insights into the types of data and functions used in your code.
+
+Inspired by tools like [enry](https://github.com/go-enry/go-enry) and [linguist](https://github.com/github-linguist/linguist)
+
+Initial feaures: language detector and metadata identifiers, based on the filename extension, filename details and the shebang line. 
+
+The official documentation can be found at [panopticas.io](https://panopticas.io)
+
+## Installation
+
+> pip install panopticas
+
+## Usage
+
+Change into the directory you want check the file types of and then run
+
+> panopticas assess
+
+To check a single file and get some metadata
+
+> panopticas file FILENAME
+
+## Development 
+
+If you want to check out the [panopitcas repo](https://github.com/kospex/panopticas) and work on bug fixes, use the pip "editable" install to set up the _panopticas_ CLI for your shell using:
+> pip install -e .
+
+
+

panopticas-0.0.9.dist-info/RECORD

@@ -0,0 +1,8 @@
+panopticas.py,sha256=gYg5DNyV9w6OIdoQD-lODZIQQ1zEGajPteoP9loZ1fs,11592
+panopticas_cli.py,sha256=Es4p_l0Tdsf2YXN7HK48iCtLjJ-f8tKXVBGjlefqULw,3472
+panopticas-0.0.9.dist-info/LICENSE,sha256=jxVEmRQitTDmkgtGeuVkMkGS4tqq553njlJnp_PpMbQ,1071
+panopticas-0.0.9.dist-info/METADATA,sha256=0p_6ArQWc2KCO31Uh2ZdYF_PRX-J9Fcn0pS2DeAHMBU,2776
+panopticas-0.0.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+panopticas-0.0.9.dist-info/entry_points.txt,sha256=buvlxCprd7u_QCF4vTI6w75GpENtBa5YkDngKjwGgqY,50
+panopticas-0.0.9.dist-info/top_level.txt,sha256=zUI8atAFe07KkXk9JvE7l8YXZxk5RJRx2Ypc6Eg062E,26
+panopticas-0.0.9.dist-info/RECORD,,

panopticas-0.0.9.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

panopticas-0.0.9.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+panopticas = panopticas_cli:cli

panopticas-0.0.9.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+panopticas
+panopticas_cli

panopticas.py

@@ -0,0 +1,436 @@
+"""
+Analysis functions for Panopticas.
+"""
+import os
+import re
+import pathspec
+
+VERSION = "0.0.8"
+
+EXT_FILETYPES = {
+        '.c': 'C',
+        '.cpp': 'C++',
+        '.cs': 'C#',
+        '.css': 'CSS',
+        '.csv': 'CSV',
+        '.dockerignore': 'Dockerignore',
+        '.gitignore': 'Gitignore',
+        '.gitattributes': 'GitAttributes',
+        '.go': 'Go',
+        '.gif': "GIF",
+        '.h': 'C Header',
+        '.htm': 'HTML',
+        '.html': 'HTML',
+        '.ico': 'ICO',
+        '.ini': 'INI',
+        '.ipynb': 'Jupyter Notebook',
+        '.java': 'Java',
+        '.jpg': 'JPEG',
+        '.jpeg': 'JPEG',
+        '.js': 'JavaScript',
+        '.json': 'JSON',
+        '.jsx': 'JSX',
+        '.kt': 'Kotlin',
+        '.m': 'Objective-C',
+        '.mailmap': 'Mailmap',
+        '.md': 'Markdown',
+        '.nvmrc': "nvmrc",
+        '.php': 'PHP',
+        '.pl': 'Perl',
+        '.pm': 'Perl',
+        '.png': 'PNG',
+        '.py': 'Python',
+        '.r': 'R',
+        '.rb': 'Ruby',
+        '.rs': 'Rust',
+        '.rst': 'ReStructuredText',
+        '.scala': 'Scala',
+        '.sh': 'Shell',
+        '.sql': 'SQL',
+        '.sqlfluff': 'SQLFluff',
+        '.sqlfluffignore': 'SQLFluffIgnore',
+        '.svg': 'SVG',
+        '.swift': 'Swift',
+        '.tf': 'Terraform',
+        '.toml': 'TOML',
+        '.ts': 'TypeScript',
+        '.tsv': 'TSV',
+        '.tsx': 'TSX',
+        '.txt': 'Text',
+        '.vue': 'Vue',
+        '.xml': 'XML',
+        '.yaml': 'YAML',
+        '.yml': 'YAML',
+        # Special cases for files without extensions or .format files
+        "codeowners": "CODEOWNERS",
+        'dockerfile': 'Dockerfile',
+        'makefile': 'Makefile',
+    }
+
+LANGUAGE_BY_BASENAME = {
+    'go.mod': 'go.mod',
+    'go.sum': 'go.sum',
+}
+
+
+def get_fileext(file_path):
+    """ Get the file extension of a file """
+    file_type = None
+
+    #if os.path.isfile(file_path):
+    #    file_type = os.path.splitext(file_path)[1]
+    file_type = os.path.splitext(file_path)[1]
+
+    if file_type:
+        return file_type
+    else:
+        return os.path.basename(file_path)
+
+def get_extension_filetype(file_ext):
+    """ Get the file extension of a file, using an exact match """
+
+    if file_ext:
+        return EXT_FILETYPES.get(file_ext.lower(), None)
+    else:
+        return None
+
+def get_filename_metatypes(file_path):
+    """
+    Return an array of metatypes based on the file_path
+    For example:
+        pyproject.toml will return build, dependencies
+        .github/workflows/python-app.yml will return Github, workflow
+    """
+
+    filename = os.path.basename(file_path).lower()
+    ext = get_fileext(file_path)
+
+    file_no_ext = os.path.splitext(filename)[0]
+
+    tags = []
+
+    if ext == ".pm":
+        # Perl module
+        # Languauge should already be set by the extension
+        tags.append("module")
+
+    if filename == "pyproject.toml":
+        tags.append("build")
+        tags.append("dependencies")
+        tags.append("Python")
+
+    if ".github" in file_path:
+        tags.append("Github")
+        tags.append("Git")
+
+    if filename == '.gitattributes':
+        tags.append("Git")
+
+    if filename == "eslint.config.js":
+        tags.extend(["JavaScript", "linter", "eslint", "config"])
+
+    if filename == ".mailmap":
+        tags.append("Git")
+
+    if is_pip_requirements(filename):
+        #if filename == "requirements.txt":
+        tags.append("pip")
+        tags.append("Python")
+        tags.append("dependencies")
+
+    if filename == '.sqlfluff':
+        tags.append("SQLFluff")
+        tags.append("SQL")
+        tags.append("linter")
+
+    if filename == ".mailmap":
+        tags.append("Git")
+
+    if filename == '.nvmrc':
+        tags.append("Node")
+        tags.append("dependencies")
+
+    # Usually the filename will be uppercase CODEOWNERS for GitHub
+    if filename == "codeowners":
+        tags.append("Git")
+
+    if filename == ".gitignore":
+        tags.append("Git")
+        tags.append("ignore")
+
+    if filename == "dockerfile":
+        tags.append("IaC")
+        tags.append("Docker")
+        tags.append("dependencies")
+
+    if filename == ".dockerignore":
+        tags.append("Docker")
+        tags.append("ignore")
+
+    if filename == "makefile":
+        tags.append("build")
+
+    if ".github/workflows" in file_path:
+        tags.append("workflow")
+
+    if filename == "go.mod":
+        tags.append("Go")
+        tags.append("module")
+        tags.append("dependencies")
+
+    if filename == "go.sum":
+        tags.append("Go")
+        tags.append("dependencies")
+        # TODO - check what we actually want to call hashes etc
+        # maybe integrity-checks
+        tags.append("checksum")
+
+    if filename == ".sqlfluffignore":
+        tags.append("SQLFluff")
+        tags.append("ignore")
+
+    if filename == "codefresh.yml":
+        tags.append("pipeline")
+        tags.append("Codefresh")
+
+    if filename == ".travis.yml":
+            tags.append("pipeline")
+            tags.append("TravisCI")
+
+    if filename in ( "package.json", "package-lock.json"):
+        tags.append("npm")
+        tags.append("dependencies")
+
+    if filename == "pom.xml":
+        tags.append("maven")
+        tags.append("build")
+        tags.append("dependencies")
+
+    if filename == "build.gradle":
+        tags.append("gradle")
+        tags.append("build")
+        tags.append("dependencies")
+
+    if file_no_ext == "license":
+        tags.append("license")
+
+    return tags
+
+def check_shebang(file_path):
+    """ Check if a file has a shebang """
+    try:
+
+        with open(file_path) as file:
+            first_line = file.readline()
+            if first_line and first_line.startswith("#!"):
+                return first_line.strip()
+            else:
+                return None
+
+    except FileNotFoundError:
+        # TODO - better logging instead of print
+        #print(f"File {file_path} not found")
+        return None
+    except UnicodeDecodeError:
+        # TODO - log this exception
+        return None
+
+
+def get_shebang_language(shebang):
+    """ Return the language of a shebang """
+    lang = extract_shebang_language(shebang)
+    return lang
+
+def load_gitignore_patterns(directory):
+    """
+    Load gitignore patterns from a directory
+    """
+    gitignore_path = os.path.join(directory, '.gitignore')
+
+    if os.path.exists(gitignore_path):
+
+        with open(gitignore_path, 'r', encoding='utf-8') as file:
+            patterns = file.read().splitlines()
+            # Add .git to the patterns
+            patterns.append('.git')
+            # These seem to be ignore by git
+            patterns.append('.jekyll-cache')
+            patterns.append('.ruff_cache')
+            patterns.append('.DS_Store')
+
+        return pathspec.PathSpec.from_lines('gitwildmatch', patterns)
+    return None
+
+def identify_files(directory):
+    """
+    Identify files in a directory.
+    Returns a dict of the relative path filenams to their file_type
+    """
+
+    gitignore_spec = load_gitignore_patterns(directory)
+
+    file_paths = {}
+
+    for root, _, files in os.walk(directory):
+        for file in files:
+            full_path = os.path.join(root, file)
+
+            relative_path = os.path.relpath(full_path, directory)
+            if gitignore_spec and gitignore_spec.match_file(relative_path):
+                continue
+
+            ftype = get_language(full_path)
+
+            if directory == ".":
+                full_path = full_path.removeprefix("./")
+
+            # TODO - see if we want to add a parameter to return the full path
+            #file_paths[fulle_path] = ftype
+            file_paths[relative_path] = ftype
+
+    return file_paths
+
+def find_files(directory,all_files=None):
+    """
+    Find all files in a directory, honoring the gitignore patterns.
+    If all_files = True, then find everything.
+    Returns a list of the relative path filenames
+    """
+
+    gitignore_spec = load_gitignore_patterns(directory)
+    if all_files:
+        gitignore_spec = None
+
+    file_paths = []
+
+    for root, _, files in os.walk(directory):
+        for file in files:
+            full_path = os.path.join(root, file)
+
+            relative_path = os.path.relpath(full_path, directory)
+            if gitignore_spec and gitignore_spec.match_file(relative_path):
+                # Ignore and continue
+                continue
+
+            if directory == ".":
+                full_path = full_path.removeprefix("./")
+
+            file_paths.append(relative_path)
+
+    return file_paths
+
+def extract_shebang_language(shebang: str) -> str:
+    """
+    Take a string like
+    #!/usr/bin/env python3
+    and return the language (python3)
+    """
+    parts = shebang.split()
+    # Check if the shebang string starts with '#!' and has at least two parts
+    # Check for #!/usr/bin/env python3 pattern
+    if "#!" in parts[0] and len(parts) > 1:
+
+        # Split the second part by '/' and check if it contains 'env'
+        if 'env' in parts[0]:
+            interpreter = parts[-1]
+            #return parts[-1]  # The interpreter is the last part
+            if interpreter.startswith("python"):
+                return "Python"
+            else:
+                return interpreter
+
+    # Check for the following pattern
+    # #!/usr/local/bin/perl style pattern
+    else:
+        return shebang.split('/')[-1]  # Otherwise, the interpreter is the last
+
+    return None
+
+def get_language_edge_cases(file_path):
+    """
+    Handle edge cases where certain filenames are special file types
+    Seems to be a go thing ...
+    """
+    basename = os.path.basename(file_path)
+
+    if basename:
+        return LANGUAGE_BY_BASENAME.get(basename.lower())
+    else:
+        return None
+
+def get_language(file_path):
+    """ Return the language of a file """
+    ext = get_fileext(file_path)
+    lang = None
+
+    lang_by_basename = get_language_edge_cases(file_path)
+    if lang_by_basename:
+        return lang_by_basename
+
+    if ext:
+        lang = get_extension_filetype(ext)
+
+    if lang:
+        return lang
+
+    shebang = check_shebang(file_path)
+
+    if shebang:
+        lang = get_shebang_language(shebang)
+
+    return lang
+
+#def basename_check(file_path):
+#    """
+#    Return a guessed type based on the basename
+#    """
+
+#    return None
+
+def extract_urls(text):
+    """
+    Find all HTTP/S URLs from a given string and return a list of URLs found.
+
+    """
+    url_pattern = re.compile(r'https?://[^\s\"\'\)]+')
+    urls = re.findall(url_pattern, text)
+    return urls
+
+def extract_urls_from_file(file_path):
+    """
+    Extract URLs from a given file.
+
+    Args:
+        file_path (str): File to extract URLs from
+
+    Returns:
+        list: List of URLs found in the file
+
+    Raises:
+        FileNotFoundError: If the specified file does not exist
+        UnicodeDecodeError: If the file cannot be decoded as UTF-8
+    """
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    try:
+        with open(file_path, 'r', encoding='utf-8') as file:
+            content = file.read()
+            return extract_urls(content)
+    except UnicodeDecodeError as e:
+        raise UnicodeDecodeError(f"Unable to decode file {file_path} as UTF-8: {str(e)}")
+
+def is_pip_requirements(filename: str) -> bool:
+    """
+    Returns True if `filename` matches typical pip requirements filenames.
+    Examples of matching filenames:
+      - requirements.txt
+      - requirements-dev.txt
+      - requirements-dev.in
+      - requirements.in
+      - requirements.dev.txt
+    """
+
+    PATTERN = re.compile(r'^requirements([-._a-zA-Z0-9]*)\.(txt|in)$')
+
+    return bool(PATTERN.match(filename))

panopticas_cli.py

@@ -0,0 +1,111 @@
+# panopticas.py
+import click
+from prettytable import PrettyTable
+import panopticas as ft
+
+VERSION = "0.0.9"
+
+@click.group()
+def cli():
+    """Panopticas is a tool for identifying file types, code and git repositories.
+
+    In future, it will be possible identify external dependencies
+    (e.g. URLs, cloud providers)
+
+    For documentation on how commands run `panopticas COMMAND --help`.
+
+    See also https://panopticas.io/
+
+    """
+
+@cli.command("assess")
+@click.option('-unknown', is_flag=True, default=False, help="Show only files with an unknown language type.")
+@click.argument('directory', required=False, type=click.Path(exists=True))
+def assess(directory,unknown):
+    """Assess a directory."""
+    click.echo()
+    if directory:
+        click.echo(f'Assessing directory: {directory}')
+    else:
+        click.echo('Assessing current directory.')
+        directory = "."
+    files = ft.identify_files(directory)
+    click.echo(f'Found {len(files)} files.\n')
+    table = PrettyTable()
+    table.field_names = ["File", "Language", "Meta"]
+    table.align["File"] = "l"
+    table.align["Language"] = "l"
+    table.align["Meta"] = "l"
+
+    for file, file_type in files.items():
+        meta = ft.get_filename_metatypes(file) if ft.get_filename_metatypes(file) else ""
+        if meta:
+            meta = ", ".join(meta)
+        # with "unknown", we only include files with unknown language types (e.g. None)
+        if unknown:
+            if file_type is None:
+                #print(f"File: {file} is of unknown language type")
+                table.add_row([file, file_type, meta])
+        else:
+            # Default is we add it to the table
+            table.add_row([file, file_type, meta])
+
+    print(table,"\n")
+
+@cli.command("file")
+@click.argument('file', required=True,type=click.Path(exists=True))
+def identify(file):
+    """Assess a filetype."""
+    click.echo(f'\nAssessing filetype for file {file}')
+    click.echo()
+    table = PrettyTable()
+    table.field_names = ["Method", "Result"]
+    table.align["Method"] = "l"
+    table.align["Result"] = "l"
+
+    table.add_row(["File extenion", ft.get_fileext(file)])
+    table.add_row(["File type", ft.get_extension_filetype(ft.get_fileext(file))])
+    shebang = ft.check_shebang(file)
+    table.add_row(["Shebang", shebang])
+    table.add_row(["Shebang Language", ft.extract_shebang_language(shebang) if shebang else None])
+    table.add_row(["Meta", ft.get_filename_metatypes(file)])
+
+    urls = []
+
+    urls = ft.extract_urls_from_file(file)
+    table.add_row(["URLs", '\n'.join(urls)])
+
+    print(table)
+    print()
+
+@cli.command("urls")
+@click.option('-all-files', is_flag=True, default=False, help="Show all files, no gitignore.")
+@click.argument('directory', required=True, type=click.Path(exists=True))
+def find_urls(directory,all_files):
+    """
+    Find and show urls for all files in a given directory.
+    """
+    files = ft.find_files(directory,all_files=all_files)
+
+    table = PrettyTable()
+    table.field_names = ["Filename", "URLs"]
+    table.align["Filename"] = "l"
+    table.align["URLs"] = "l"
+
+    # ft.find_files(directory)
+    for f in files:
+        urls = ft.extract_urls_from_file(f)
+        table.add_row([f, '\n'.join(urls)])
+
+    print(table)
+    print()
+
+
+@cli.command("version")
+def version():
+    """Print the version of Panopticas."""
+    click.echo(f"Panopticas CLI version {VERSION}")
+    click.echo(f"Panopticas v{ft.VERSION}")
+
+if __name__ == '__main__':
+    cli()