checkmate5 4.0.67__py3-none-any.whl

checkmate/__init__.py

@@ -0,0 +1,21 @@
+# The MIT License (MIT)
+# 
+# Copyright (c) 2014 Andreas Dewes
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

checkmate/__main__.py

@@ -0,0 +1,25 @@
+# The MIT License (MIT)
+# 
+# Copyright (c) 2014 Andreas Dewes
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import sys
+
+if __name__ == '__main__':
+    print((sys.argv))

checkmate/contrib/__init__.py

@@ -0,0 +1,21 @@
+# The MIT License (MIT)
+# 
+# Copyright (c) 2014 Andreas Dewes
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

checkmate/contrib/plugins/all/gptanalyzer/analyzer.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+
+
+from checkmate.lib.analysis.base import BaseAnalyzer
+
+import logging
+import os
+import tempfile
+import json
+import subprocess
+import re
+
+
+logger = logging.getLogger(__name__)
+
+
+class GptAnalyzer(BaseAnalyzer):
+
+    def __init__(self, *args, **kwargs):
+        super(GptAnalyzer, self).__init__(*args, **kwargs)
+
+    def summarize(self, items):
+        pass
+
+    def analyze(self, file_revision):
+        issues = []
+        tmpdir = "/tmp/"+file_revision.project.pk
+
+        if not os.path.exists(os.path.dirname(tmpdir+"/"+file_revision.path)):
+            try:
+                os.makedirs(os.path.dirname(tmpdir+"/"+file_revision.path))
+            except OSError as exc:  # Guard against race condition
+                if exc.errno != errno.EEXIST:
+                    raise
+        
+        #result = subprocess.check_output(["rsync -r . "+tmpdir+" --exclude .git"],shell=True).strip()
+                                        
+        f = open(tmpdir+"/"+file_revision.path, "wb")
+
+        result = {}
+        try:
+            with f:
+                try:
+                  f.write(file_revision.get_file_content())
+                except UnicodeDecodeError:
+                  pass
+            os.chdir(tmpdir)
+            os.environ["PATH"] = "/root/.go/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/:/usr/local/go/bin/"
+
+            try:
+                result = subprocess.check_output(["/root/bin/ptpt",
+                                                  "run",
+                                                  "scr",
+                                                  f.name],
+                                                  stderr=subprocess.DEVNULL).strip()
+            except subprocess.CalledProcessError as e:
+                if e.returncode == 2:
+                    result = e.output
+                elif e.returncode == 1:
+                    result = e.output
+                    pass
+                else:
+                    result = []
+
+            try:
+                  json_result = json.loads(result)
+            except ValueError:
+                  json_result = []
+                  pass
+            try:
+              for issue in json_result:
+                  value = issue['line']
+
+                  location = (((value,None),
+                             (value,None)),)
+
+                  string = issue["finding"]
+                  string = string.replace("'","")
+                  string = string.replace("`","")
+                  string = string.replace("\"","")
+                  string = string.strip()
+                  string = re.sub('[^A-Za-z0-9 ]+', '', string)
+
+                  issues.append({
+                            'code': "I001",
+                            'location': location,
+                            'data': string,
+                            'file': file_revision.path,
+                            'line': value,
+                            'fingerprint': self.get_fingerprint_from_code(file_revision, location, extra_data=string)
+                        })
+
+            except:
+                pass
+
+        finally:
+            pass
+        return {'issues': issues}
+

checkmate/contrib/plugins/all/gptanalyzer/issues_data.py

@@ -0,0 +1,6 @@
+issues_data = {   'I001': {   'categories': [],
+                'description': '%(issue.data)s',
+                'file': '%(issue.file)s',
+                'line': '%(issue.line)s',
+                'severity': 3,
+                'title': 'Openai'}}

checkmate/contrib/plugins/all/gptanalyzer/setup.py

@@ -0,0 +1,13 @@
+from .analyzer import GptAnalyzer
+from .issues_data import issues_data
+
+analyzers = {
+    'gptanalyzer':
+        {
+            'name': 'gptanalyzer',
+            'title': 'gptanalyzer',
+            'class': GptAnalyzer,
+            'language': 'all',
+            'issues_data': issues_data,
+        },
+}

checkmate/contrib/plugins/cve/text4shell/analyzer.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+
+
+from checkmate.lib.analysis.base import BaseAnalyzer
+
+import logging
+import os
+import tempfile
+import json
+import subprocess
+
+logger = logging.getLogger(__name__)
+
+
+class Text4shellAnalyzer(BaseAnalyzer):
+
+    def __init__(self, *args, **kwargs):
+        super(Text4shellAnalyzer, self).__init__(*args, **kwargs)
+
+    def summarize(self, items):
+        pass
+
+    def analyze(self, file_revision):
+        issues = []
+        tmpdir = "/tmp/"+file_revision.project.pk
+        f = open(tmpdir+"/"+file_revision.path, "wb")
+        try:
+            with f:
+                f.write(file_revision.get_file_content())
+            try:
+                result = subprocess.check_output(["python3","/root/text4shell-ce/scan_commons_text_versions.py",
+                                                  f.name,
+                                                  "-quiet"]
+                                                  )
+            except subprocess.CalledProcessError as e:
+                pass
+
+            try:
+                json_result = json.loads(result)
+            except ValueError:
+                json_result = {}
+                pass
+
+            try:
+                line = "1"
+                line = int(line)
+                location = (((line, line),
+                             (line, None)),)
+
+                issues.append({
+                    'code': "I001",
+                    'location': location,
+                    'data': json_result["I001"],
+                    'file': file_revision.path,
+                    'line': line,
+                    'fingerprint': self.get_fingerprint_from_code(file_revision, location, extra_data=json_result["I001"])
+                })
+
+            except KeyError:
+                pass
+
+        finally:
+            os.unlink(f.name)
+        return {'issues': issues}

checkmate/contrib/plugins/cve/text4shell/issues_data.py

@@ -0,0 +1,8 @@
+issues_data = {
+    "I001": {
+        "title": "Text4Shell",
+        "description": "%(issue.data)s",
+        "severity": 3,
+        "categories": []
+    }
+}

checkmate/contrib/plugins/cve/text4shell/setup.py

@@ -0,0 +1,13 @@
+from .analyzer import Text4shellAnalyzer
+from .issues_data import issues_data
+
+analyzers = {
+    'text4shell':
+        {
+            'name': 'text4shell',
+            'title': 'text4shell',
+            'class': Text4shellAnalyzer,
+            'language': 'cve',
+            'issues_data': issues_data,
+        },
+}

checkmate/contrib/plugins/git/commands/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from .init import Command as InitCommand
+from .analyze import Command as AnalyzeCommand
+from .diff import Command as DiffCommand
+from .update_stats import Command as UpdateStatsCommand

checkmate/contrib/plugins/git/commands/analyze.py

@@ -0,0 +1,364 @@
+# -*- coding: utf-8 -*-
+
+
+from checkmate.management.commands.analyze import Command as AnalyzeCommand
+from .base import Command as BaseCommand
+from ..lib.repository import group_snapshots_by_date, get_first_date_for_group
+from ..models import GitBranch
+from checkmate.contrib.plugins.git.models import GitSnapshot
+from checkmate.lib.models import Diff, Snapshot
+from checkmate.helpers.hashing import Hasher
+from checkmate.helpers.settings import update
+from checkmate.lib.models import (Snapshot,
+                                  Project,
+                                  BaseDocument,
+                                  FileRevision,
+                                  Diff,
+                                  IssueOccurrence)
+
+import time
+import datetime
+import traceback
+import logging
+
+from checkmate.lib.code import CodeEnvironment
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand, AnalyzeCommand):
+
+    options = AnalyzeCommand.options + [
+        {
+            'name': '--n',
+            'action': 'store',
+            'dest': 'n',
+            'type': int,
+            'default': 1,
+            'help': 'The number of snapshots to analyze.'
+        },
+        {
+            'name': '--offset',
+            'action': 'store',
+            'dest': 'offset',
+            'type': int,
+            'default': 0,
+            'help': 'The offset from the latest snapshot.'
+        },
+        {
+            'name': '--shas',
+            'action': 'store',
+            'dest': 'shas',
+            'type': str,
+            'default': '',
+            'help': 'A comma-separated list of commit to analyze, identified by their SHAs.'
+        },
+        {
+            'name': '--branch',
+            'action': 'store',
+            'dest': 'branch',
+            'type': str,
+            'default': 'master',
+            'help': 'The branch to analyze.'
+        },
+        {
+            'name': '--from',
+            'action': 'store',
+            'dest': 'from',
+            'type': str,
+            'default': '',
+            'help': 'The date for which to perform the analysis.'
+        },
+        {
+            'name': '--type',
+            'action': 'store',
+            'dest': 'type',
+            'type': str,
+            'default': 'latest',
+            'help': 'The type of analysis (latest, monthly, weekly, daily).'
+        },
+    ]
+
+    def analyze_grouped_snapshots(self, branch, group, grouped_snapshots):
+        for key, snapshots in list(grouped_snapshots.items()):
+            last_snp, first_snp = snapshots[0], snapshots[-1]
+            self.analyze_and_generate_diffs(branch,
+                                            [first_snp, last_snp],
+                                            [{'snapshot_a': first_snp.sha,
+                                              'snapshot_b': last_snp.sha}])
+
+    def analyze_and_generate_diffs(self, branch, snapshots, diff_list):
+
+        analyzed_snapshots = {}
+        new_analyzed_snapshots = {}
+
+        if 'project_settings' in self.opts:
+            project_settings = self.opts['project_settings']
+        else:
+            project_settings = self.project.settings
+
+        # we look for a .checkmate.yml file in the main branch of the project
+        # if we find it, we either replace the project settings, update them
+        # or do nothing, depending on the desired configuration.
+        try:
+            git_settings = self.project.git.get_settings()
+            if git_settings is not None:
+                if git_settings.get('overwrite_project_settings', False):
+                    project_settings = git_settings
+                elif not project_settings.get('ignore_git_settings', False):
+                    project_settings.update(git_settings)
+        except ValueError:
+            logger.warning("Error when loading checkmate settings from git...")
+            git_settings = {}
+
+        code_environment = CodeEnvironment(self.project,
+                                           global_settings=self.settings,
+                                           project_settings=project_settings)
+
+        code_environment.env['branch'] = branch
+        code_environment.env['project'] = self.project
+
+        for git_snapshot in snapshots:
+            try:
+                query = {'sha': git_snapshot.sha,
+                         'project': self.project}
+                git_snapshot = self.backend.get(
+                    GitSnapshot, query, include=('snapshot',))
+                snapshot = git_snapshot.snapshot
+                # we check if the snapshot is analyzed and if the analysis configuration matches
+                if snapshot.analyzed and snapshot.configuration == self.project.configuration:
+                    analyze_snapshot = False
+                else:
+                    analyze_snapshot = True
+                    snapshot.configuration = self.project.configuration
+            except GitSnapshot.DoesNotExist:
+                analyze_snapshot = True
+                snapshot = Snapshot()
+                snapshot.configuration = self.project.configuration
+            except GitSnapshot.MultipleDocumentsReturned:
+                logger.error("Multiple snapshots returned, deleting...")
+                with self.backend.transaction():
+                    self.backend.filter(GitSnapshot, query).delete()
+                analyze_snapshot = True
+            if analyze_snapshot:
+
+                try:
+                    file_revisions = self.project.git.get_file_revisions(
+                        git_snapshot.sha)
+                except:
+                    logger.error("Cannot fetch file revisions for snapshot %s in project %s" % (
+                        git_snapshot.sha, self.project.pk))
+                    continue
+
+
+            try:     
+                file_revisions = self.project.git.get_file_revisions(
+                        git_snapshot.sha)
+            except:
+                logger.error("Cannot fetch file revisions for snapshot %s in project %s" % (
+                        git_snapshot.sha, self.project.pk))
+                continue
+
+
+
+            code_environment.file_revisions = file_revisions
+            git_snapshot.snapshot = code_environment.analyze(
+                  file_revisions, save_if_empty=True, snapshot=snapshot)
+
+            new_analyzed_snapshots[git_snapshot.sha] = git_snapshot
+
+            with self.backend.transaction():
+                    git_snapshot.snapshot.hash = git_snapshot.sha
+                    self.backend.save(git_snapshot.snapshot)
+                    git_snapshot.project = self.project
+                    self.backend.save(git_snapshot)
+
+            analyzed_snapshots[git_snapshot.sha] = git_snapshot
+
+        snapshot_pairs = []
+        for diff_params in diff_list:
+            try:
+                snapshot_pairs.append(
+                    [analyzed_snapshots[diff_params['snapshot_a']], analyzed_snapshots[diff_params['snapshot_b']]])
+            except KeyError:
+                continue
+
+        diffs = []
+
+        for git_snapshot_a, git_snapshot_b in snapshot_pairs:
+            diff = None
+            try:
+                query = {'snapshot_a': git_snapshot_a.snapshot,
+                         'snapshot_b': git_snapshot_b.snapshot}
+                diff = self.backend.get(Diff, query)
+            except Diff.DoesNotExist:
+                logger.debug("Generating a diff between snapshots %s and %s" % (git_snapshot_a.sha,
+                                                                                git_snapshot_b.sha))
+            # if the configuration of the diff does not match the project configuration, we regenerate it
+            if diff is None or diff.configuration != self.project.configuration:
+                if diff is not None:
+                    diff.configuration = self.project.configuration
+                diff, diff_file_revisions, diff_issue_occurrences = code_environment.diff_snapshots(
+                    git_snapshot_a.snapshot,
+                    git_snapshot_b.snapshot,
+                    save=True,
+                    diff=diff)
+
+            diffs.append(diff)
+
+        return analyzed_snapshots, diffs
+
+    def update_branch(self, branch_name, head_snapshot=None):
+
+        logger.debug("Updating info for branch %s" % branch_name)
+
+        hasher = Hasher()
+        hasher.add(branch_name)
+
+        branch = GitBranch({'project': self.project,
+                            'name': branch_name,
+                            'hash': hasher.digest.hexdigest(),
+                            'snapshots': []})
+
+        try:
+            branch = self.backend.get(GitBranch,
+                                      {'project.pk': self.project.pk, 'name': branch_name})
+        except GitBranch.DoesNotExist:
+            logger.debug("Creating branch....")
+            # we save the new branch instead...
+            with self.backend.transaction():
+                self.backend.save(branch)
+
+        if head_snapshot:
+            if branch.get('head_snapshot'):
+                if head_snapshot.committer_date_ts > branch.head_snapshot.eager.committer_date_ts:
+                    branch.last_analyzed_snapshot = branch.head_snapshot
+                    branch.head_snapshot = head_snapshot
+            else:
+                branch.head_snapshot = head_snapshot
+            with self.backend.transaction():
+                self.backend.update(
+                    branch, ['last_analyzed_snapshot', 'head_snapshot'])
+
+    def run(self):
+        # to do: replace this with actual configuration information (in the future)
+        hasher = Hasher()
+        hasher.add("bars")
+        self.project.configuration = hasher.digest.hexdigest()
+        with self.backend.transaction():
+            self.backend.update(self.project, ['configuration'])
+        if not 'branch' in self.opts:
+            branch_name = self.project.git.get_default_branch()
+        else:
+            branch_name = self.opts.get('branch')
+
+        if self.opts['type'] in ('monthly', 'weekly', 'daily'):
+
+            if not branch_name:
+                raise ValueError(
+                    "Branch cannot be None for %s analysis!" % self.opts['type'])
+
+            logger.info("Analyzing %d %s commits in branch %s (offset: %d)" %
+                        (self.opts['n'], self.opts['type'], branch_name, self.opts['offset']))
+
+            if self.opts['from']:
+                if isinstance(self.opts['from'], str):
+                    try:
+                        from_date = datetime.datetime.strptime(
+                            self.opts['from'], "%Y-%m-%d")
+                    except ValueError:
+                        raise self.CommandException(
+                            "Invalid date format or value for `from` parameter (use YYYY-mm-dd)")
+            else:
+                try:
+                    from_date = self.project.git.get_snapshots(branch=branch_name, limit=1)[0]\
+                        .committer_date + datetime.timedelta(days=1)
+                except IndexError:
+                    logger.info("No snapshots found in branch %s" %
+                                branch_name)
+                    return
+
+            dt = get_first_date_for_group(from_date, self.opts['type'], self.opts['n'] +
+                                          self.opts['offset'])
+
+            snapshots = self.project.git.get_snapshots(branch=branch_name,
+                                                       since=dt.ctime())[:-self.opts['offset']
+                                                                         if self.opts['offset'] != 0 else None]
+
+            grouped_snapshots = group_snapshots_by_date(
+                snapshots, self.opts['type'])
+
+            snapshots_to_analyze = []
+            diffs_to_generate = []
+            for key, snapshots in list(grouped_snapshots.items()):
+                last_snp, first_snp = snapshots[0], snapshots[-1]
+                snapshots_to_analyze.append(first_snp)
+                snapshots_to_analyze.append(last_snp)
+                diffs_to_generate.append(
+                    {'snapshot_a': first_snp.sha, 'snapshot_b': last_snp.sha})
+
+        else:
+            if self.opts['shas']:
+                logger.debug("Analyzing %d snapshots" %
+                             len(self.opts['shas'].split(",")))
+                if isinstance(self.opts['shas'], str):
+                    shas = self.opts['shas'].split(",")
+                else:
+                    shas = self.opts['shas']
+                snapshots_to_analyze = self.project.git.get_snapshots(
+                    shas=shas)
+            else:
+                if branch_name is None:
+                    logger.error("No branch specified!")
+                    return
+
+                # we perform a sequential analysis
+                branch = None
+                try:
+                    branch = self.backend.get(GitBranch,
+                                              {'project.pk': self.project.pk,
+                                               'name': branch_name})
+                except GitBranch.DoesNotExist:
+                    pass
+
+                logger.info("Analyzing the %d most recent commits in branch %s (offset: %d)" %
+                            (self.opts['n'], branch_name, self.opts['offset']))
+
+                if not self.opts['n']:
+                    snapshots_to_analyze = self.project.git.get_snapshots(
+                        branch=branch_name)
+                else:
+                    snapshots_to_analyze = self.project.git.get_snapshots(branch=branch_name,
+                                                                          limit=self.opts['n'], offset=self.opts['offset'])
+
+                def timestamp(dt):
+                    return (dt-datetime.datetime(1970, 1, 1)).total_seconds()
+
+                diffs_to_generate = [{'snapshot_a': snapshots_to_analyze[i]['sha'],
+                                      'snapshot_b':snapshots_to_analyze[i+1]['sha']}
+                                     for i in range(len(snapshots_to_analyze)-1)]
+
+                if branch and branch.get('last_analyzed_snapshot', None):
+                    last_analyzed_snapshot = branch.last_analyzed_snapshot.eager
+                    if last_analyzed_snapshot and snapshots_to_analyze and last_analyzed_snapshot['sha'] != snapshots_to_analyze[-1]['sha']:
+                        snapshots_to_analyze.append(last_analyzed_snapshot)
+                        last_diff = {
+                            'snapshot_a': last_analyzed_snapshot['sha'],
+                            'snapshot_b': snapshots_to_analyze[-1]['sha']
+                        }
+                        diffs_to_generate.append(last_diff)
+
+            if 'diffs' in self.opts and self.opts['diffs'] is not None:
+                diffs_to_generate = self.opts['diffs']
+            try:
+                analyzed_snapshots, diffs = self.analyze_and_generate_diffs(branch_name,
+                                                                            snapshots_to_analyze, diffs_to_generate)
+                head_snapshot = None
+                if analyzed_snapshots:
+                    head_snapshot = sorted(
+                        list(analyzed_snapshots.values()), key=lambda snp: snp['committer_date_ts'])[-1]
+                    if branch_name:
+                        self.update_branch(
+                            branch_name, head_snapshot=head_snapshot)
+            except KeyboardInterrupt:
+                raise