crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/proxy/server.py

@@ -6,6 +6,7 @@ Look at `index.js` to see how the functionality is invoked.
 from urllib.parse import unquote_plus
 import argparse
 import dataclasses
+import importlib_resources as resources
 import json
 import os
 import sys
@@ -29,9 +30,9 @@ from helm.common.perspective_api_request import PerspectiveAPIRequest
 from helm.common.moderations_api_request import ModerationAPIRequest
 from helm.common.tokenization_request import TokenizationRequest, DecodeRequest
 from helm.proxy.services.service import CACHE_DIR
-from .accounts import Account
-from .services.server_service import ServerService
-from .query import Query
+from helm.proxy.accounts import Account
+from helm.proxy.services.server_service import ServerService
+from helm.proxy.query import Query
 
 try:
     import gunicorn  # noqa
@@ -86,7 +87,7 @@ def handle_root():
 
 @app.get("/static/<filename:path>")
 def handle_static_filename(filename):
-    resp = bottle.static_file(filename, root=os.path.join(os.path.dirname(__file__), "static"))
+    resp = bottle.static_file(filename, root=app.config["helm.staticpath"])
     resp.add_header("Cache-Control", "no-store, must-revalidate ")
     return resp
 
@@ -284,6 +285,12 @@ def main():
         ensure_directory_exists(sqlite_cache_path)
         cache_backend_config = SqliteCacheBackendConfig(sqlite_cache_path)
 
+    static_package_name = "helm.proxy.static"
+    resource_path = resources.files(static_package_name).joinpath("index.html")
+    with resources.as_file(resource_path) as resource_filename:
+        static_path = str(resource_filename.parent)
+    app.config["helm.staticpath"] = static_path
+
     service = ServerService(base_path=args.base_path, cache_backend_config=cache_backend_config)
 
     gunicorn_args = {

helm/proxy/services/remote_service.py

@@ -24,7 +24,7 @@ from helm.common.request import Request, RequestResult
 from dacite import from_dict
 from helm.proxy.accounts import Account
 from helm.proxy.query import Query, QueryResult
-from .service import Service, GeneralInfo
+from helm.proxy.services.service import Service, GeneralInfo
 
 
 class RemoteServiceError(Exception):

helm/proxy/services/server_service.py

@@ -1,8 +1,8 @@
-import dataclasses
 import os
 import signal
-from typing import List, Optional
+from typing import List
 
+from helm.common.local_context import LocalContext
 from helm.common.cache import CacheConfig
 from helm.common.cache_backend_config import CacheBackendConfig, BlackHoleCacheBackendConfig
 from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
@@ -11,7 +11,6 @@ from helm.common.moderations_api_request import ModerationAPIRequest, Moderation
 from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
 from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
 from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
-from helm.common.general import ensure_directory_exists, parse_hocon, get_credentials
 from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
 from helm.common.tokenization_request import (
     TokenizationRequest,
@@ -22,27 +21,13 @@ from helm.common.tokenization_request import (
 from helm.common.request import Request, RequestResult
 from helm.common.hierarchical_logger import hlog
 from helm.proxy.accounts import Accounts, Account
-from helm.clients.auto_client import AutoClient
-from helm.clients.moderation_api_client import ModerationAPIClient
-from helm.clients.image_generation.nudity_check_client import NudityCheckClient
-from helm.clients.gcs_client import GCSClient
-from helm.clients.clip_score_client import CLIPScoreClient
-from helm.clients.toxicity_classifier_client import ToxicityClassifierClient
-from helm.proxy.example_queries import example_queries
-from helm.benchmark.model_metadata_registry import ALL_MODELS_METADATA
 from helm.benchmark.model_deployment_registry import get_model_deployment_host_organization
 from helm.proxy.query import Query, QueryResult
-from helm.proxy.retry import retry_request
 from helm.proxy.token_counters.auto_token_counter import AutoTokenCounter
-from helm.tokenizers.auto_tokenizer import AutoTokenizer
-from .service import (
+from helm.proxy.services.service import (
     Service,
-    CACHE_DIR,
     ACCOUNTS_FILE,
     GeneralInfo,
-    VERSION,
-    expand_environments,
-    synthesize_request,
 )
 
 
@@ -57,43 +42,17 @@ class ServerService(Service):
         root_mode: bool = False,
         cache_backend_config: CacheBackendConfig = BlackHoleCacheBackendConfig(),
     ):
-        ensure_directory_exists(base_path)
-        client_file_storage_path = os.path.join(base_path, CACHE_DIR)
-        ensure_directory_exists(client_file_storage_path)
-
-        credentials = get_credentials(base_path)
         accounts_path = os.path.join(base_path, ACCOUNTS_FILE)
 
-        self.cache_backend_config = cache_backend_config
-        self.client = AutoClient(credentials, client_file_storage_path, cache_backend_config)
-        self.tokenizer = AutoTokenizer(credentials, cache_backend_config)
-        self.token_counter = AutoTokenCounter(self.tokenizer)
+        self.context = LocalContext(base_path, cache_backend_config)
+        self.token_counter = AutoTokenCounter(self.context.tokenizer)
         self.accounts = Accounts(accounts_path, root_mode=root_mode)
 
-        # Lazily instantiate the following clients
-        self.moderation_api_client: Optional[ModerationAPIClient] = None
-        self.toxicity_classifier_client: Optional[ToxicityClassifierClient] = None
-        self.perspective_api_client: Optional[ToxicityClassifierClient] = None
-        self.nudity_check_client: Optional[NudityCheckClient] = None
-        self.clip_score_client: Optional[CLIPScoreClient] = None
-        self.gcs_client: Optional[GCSClient] = None
-
     def get_general_info(self) -> GeneralInfo:
-        # Can't send release_dates in ModelMetadata bacause dates cannot be round-tripped to and from JSON easily.
-        # TODO(#2158): Either fix this or delete get_general_info.
-        all_models = [dataclasses.replace(model_metadata, release_date=None) for model_metadata in ALL_MODELS_METADATA]
-        return GeneralInfo(version=VERSION, example_queries=example_queries, all_models=all_models)
+        return self.context.get_general_info()
 
     def expand_query(self, query: Query) -> QueryResult:
-        """Turn the `query` into requests."""
-        prompt = query.prompt
-        settings = query.settings
-        environments = parse_hocon(query.environments)
-        requests = []
-        for environment in expand_environments(environments):
-            request = synthesize_request(prompt, settings, environment)
-            requests.append(request)
-        return QueryResult(requests=requests)
+        return self.context.expand_query(query)
 
     def _get_model_group_for_model_deployment(self, model_deployment: str) -> str:
         if model_deployment.startswith("openai/"):
@@ -105,7 +64,11 @@ class ServerService(Service):
                 return "gpt4"
             elif model_deployment.startswith("openai/gpt-3"):
                 return "gpt3"
-            elif model_deployment.startswith("openai/o1"):
+            elif (
+                model_deployment.startswith("openai/o1")
+                or model_deployment.startswith("openai/o3")
+                or model_deployment.startswith("openai/o4")
+            ):
                 return "o1"
             else:
                 return "openai"
@@ -126,7 +89,7 @@ class ServerService(Service):
         self.accounts.check_can_use(auth.api_key, model_group)
 
         # Use!
-        request_result: RequestResult = self.client.make_request(request)
+        request_result: RequestResult = self.context.make_request(request)
 
         # Only deduct if not cached
         if not request_result.cached:
@@ -139,66 +102,39 @@ class ServerService(Service):
     def tokenize(self, auth: Authentication, request: TokenizationRequest) -> TokenizationRequestResult:
         """Tokenize via an API."""
         self.accounts.authenticate(auth)
-        return self.tokenizer.tokenize(request)
+        return self.context.tokenize(request)
 
     def decode(self, auth: Authentication, request: DecodeRequest) -> DecodeRequestResult:
         """Decodes to text."""
         self.accounts.authenticate(auth)
-        return self.tokenizer.decode(request)
+        return self.context.decode(request)
 
     def upload(self, auth: Authentication, request: FileUploadRequest) -> FileUploadResult:
         """Uploads a file to external storage."""
         self.accounts.authenticate(auth)
-
-        if not self.gcs_client:
-            self.gcs_client = self.client.get_gcs_client()
-
-        assert self.gcs_client
-        return self.gcs_client.upload(request)
+        return self.context.upload(request)
 
     def check_nudity(self, auth: Authentication, request: NudityCheckRequest) -> NudityCheckResult:
         """Check for nudity."""
         self.accounts.authenticate(auth)
-
-        if not self.nudity_check_client:
-            self.nudity_check_client = self.client.get_nudity_check_client()
-
-        assert self.nudity_check_client
-        return self.nudity_check_client.check_nudity(request)
+        return self.context.check_nudity(request)
 
     def compute_clip_score(self, auth: Authentication, request: CLIPScoreRequest) -> CLIPScoreResult:
         """Computes CLIPScore for a given caption and image."""
         self.accounts.authenticate(auth)
-
-        if not self.clip_score_client:
-            self.clip_score_client = self.client.get_clip_score_client()
-
-        assert self.clip_score_client
-        return self.clip_score_client.compute_score(request)
+        return self.context.compute_clip_score(request)
 
     def get_toxicity_scores(self, auth: Authentication, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
-        @retry_request
-        def get_toxicity_scores_with_retry(request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
-            if not self.toxicity_classifier_client:
-                self.toxicity_classifier_client = self.client.get_toxicity_classifier_client()
-            return self.toxicity_classifier_client.get_toxicity_scores(request)
-
         self.accounts.authenticate(auth)
-        return get_toxicity_scores_with_retry(request)
+        return self.context.get_toxicity_scores(request)
 
     def get_moderation_results(self, auth: Authentication, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
-        @retry_request
-        def get_moderation_results_with_retry(request: ModerationAPIRequest) -> ModerationAPIRequestResult:
-            if not self.moderation_api_client:
-                self.moderation_api_client = self.client.get_moderation_api_client()
-            return self.moderation_api_client.get_moderation_results(request)
-
         self.accounts.authenticate(auth)
-        return get_moderation_results_with_retry(request)
+        return self.context.get_moderation_results(request)
 
     def make_critique_request(self, auth: Authentication, request: CritiqueRequest) -> CritiqueRequestResult:
         self.accounts.authenticate(auth)
-        return self.client.get_critique_client().make_critique_request(request)
+        return self.context.make_critique_request(request)
 
     def create_account(self, auth: Authentication) -> Account:
         """Creates a new account."""
@@ -233,4 +169,4 @@ class ServerService(Service):
         hlog("Done.")
 
     def get_cache_config(self, shard_name: str) -> CacheConfig:
-        return self.cache_backend_config.get_cache_config(shard_name)
+        return self.context.get_cache_config(shard_name)

helm/proxy/services/test_remote_service.py

@@ -18,8 +18,8 @@ from helm.common.authentication import Authentication
 from helm.common.request import Request, RequestResult
 from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult
 from helm.proxy.accounts import Account, set_default_quotas
-from .remote_service import RemoteService
-from .service import ACCOUNTS_FILE
+from helm.proxy.services.remote_service import RemoteService
+from helm.proxy.services.service import ACCOUNTS_FILE
 
 
 @dataclass(frozen=True)

helm/proxy/services/test_service.py

@@ -8,7 +8,7 @@ from helm.common.authentication import Authentication
 from helm.common.request import Request
 from helm.proxy.accounts import AuthenticationError, Accounts
 from helm.proxy.query import Query
-from .server_service import ServerService
+from helm.proxy.services.server_service import ServerService
 
 
 def get_authentication():

helm/proxy/static/general.js

@@ -0,0 +1,122 @@
+function assert(condition, message) {
+  if (!condition) {
+    throw message || "Assertion failed";
+  }
+}
+
+function encodeUrlParams(params) {
+  let s = "";
+  for (let k in params) {
+    if (params[k] != null) {
+      s += (s === "" ? "?" : "&") + k + "=" + encodeURIComponent(params[k]);
+    }
+  }
+  return s;
+}
+
+function decodeUrlParams(str) {
+  const params = {};
+  if (str === "") return params;
+  const items = str.substring(1).split(/&/);
+  for (let i = 0; i < items.length; i++) {
+    const pair = items[i].split(/=/);
+    params[pair[0]] = decodeURIComponent(pair[1]);
+  }
+  return params;
+}
+
+function updateBrowserLocation(params) {
+  // Update the address bar
+  window.history.pushState(
+    {},
+    "",
+    window.location.pathname + encodeUrlParams(params),
+  );
+}
+
+function createCookie(key, value, days) {
+  let expires = "";
+  if (days) {
+    const date = new Date();
+    date.setTime(date.getTime() + days * 24 * 60 * 60 * 1000);
+    expires = "; expires=" + date.toUTCString();
+  }
+  document.cookie = key + "=" + value + expires + "; path=/";
+}
+
+function readCookie(key) {
+  let tokens = document.cookie.split(";");
+  for (let i = 0; i < tokens.length; i++) {
+    const [k, v] = tokens[i].trim().split("=", 2);
+    if (key === k) return v;
+  }
+  return null;
+}
+
+function eraseCookie(key) {
+  createCookie(key, "", -1);
+}
+
+function renderTimestamp(timestamp) {
+  if (!timestamp) return null;
+  const d = new Date(timestamp * 1000);
+  return d.toLocaleString();
+}
+
+function renderDict(data) {
+  return JSON.stringify(data).substring(0, 10000);
+}
+
+function loadScript(src, onload, onerror) {
+  // Using jquery doesn't work, so do it in with our bare hands.
+  const s = document.createElement("script");
+  s.src = src;
+  s.onload = onload;
+  s.onerror = onerror;
+  document.head.appendChild(s);
+}
+
+function getRandomString() {
+  const vocab =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+  let text = "";
+  for (let i = 0; i < 6; i++)
+    text += vocab.charAt(Math.floor(Math.random() * vocab.length));
+  return text;
+}
+
+function round(x, n) {
+  const base = Math.pow(10, n);
+  return Math.round(x * base) / base;
+}
+
+function multilineHtml(s) {
+  return s.replace(/\n/g, "<br>");
+}
+
+function renderError(e) {
+  return $("<div>").addClass("alert alert-danger").append(multilineHtml(e));
+}
+
+function helpIcon(help, link) {
+  // Show a ?
+  return $("<a>", { href: link, target: "blank_", class: "help-icon" }).append(
+    $("<img>", { src: "info-icon.png", width: 15, title: help }),
+  );
+}
+
+const markdownConverter = new showdown.Converter({ optionKey: "value" });
+function renderMarkdown(markdown) {
+  return markdown && markdownConverter.makeHtml(markdown);
+}
+
+function refreshHashLocation() {
+  // If we request a hash location (URL contains #foo), the problem is #foo
+  // might not exist (since it's generated).  Call this function to jump to the
+  // hash location once all the anchors are generated.
+  if (location.hash) {
+    const hash = location.hash;
+    location.hash = "";
+    location.hash = hash;
+  }
+}

helm/proxy/static/help.html

@@ -0,0 +1,99 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<title>CRFM Models API</title>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+		<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/css/bootstrap.min.css" integrity="sha384-/Y6pD6FV/Vv2HJnA6t+vslU6fwYXjCFtcEpHbNJ0lyAFsXTsjBbfaDjzALeQsN6M" crossorigin="anonymous">
+		<link rel="stylesheet" type="text/css" href="index.css">
+	</head>
+
+	<body>
+		<div class="container-fluid">
+      <nav class="navbar navbar-expand-sm navbar-light bg-faded">
+        <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#nav-content" aria-controls="nav-content" aria-expanded="false" aria-label="Toggle navigation">
+          <span class="navbar-toggler-icon"></span>
+        </button>
+
+        <div class="collapse navbar-collapse" id="nav-content">
+          <ul class="navbar-nav">
+            <li class="nav-item"><a class="nav-link" href="index.html">Query</a></li>
+            <li class="nav-item"><a class="nav-link active" href="help.html">Help</a></li>
+            <li class="nav-item" id="loginInfo"></li>
+          </ul>
+        </div>
+      </nav>
+
+      <div class="row">
+        <div class="col-sm-12">
+          <p>
+            This web interface and API allows you to experiment with multiple large language models using a unified interface.
+            To log in, you need an API key.
+          </p>
+          <h2>Query</h2>
+          <ul>
+            <li>In the web interface, you can enter a <b>query</b>, which consists of the following components:
+              <ul>
+                <li><b>prompt</b>, which is what text we want to feed into the language model.  The prompt can have variables (e.g., <tt>${name}</tt>) which are filled in later.</li>
+                <li><b>settings</b>, which configures how we're going to call the backend API (HOCON format):
+                  <ul>
+                    <li><tt>model</tt>: which model to query; options are:
+                      <div id="help-models" />
+                    </li>
+                    <li><tt>temperature</tt>: a non-negative number determining amount of stochasticity (e.g., <tt>1</tt> is sampling from the model, <tt>0</tt> is returning the maximum probability output)</li>
+                    <li><tt>num_completions</tt>: number of completions (sequences, independent sampled) to return</li>
+                    <li><tt>top_k_per_token</tt>: number of candidates per token position in each completion</li>
+                    <li><tt>max_tokens</tt>: maximum number of tokens before generation stops</li>
+                    <li><tt>stop_sequences</tt>: list of strings that will stop generation (e.g., <tt>'.'</tt> or <tt>'\n'</tt>)</li>
+                    <li><tt>echo_prompt</tt>: Whether to include the prompt as a prefix of the completion</li>
+                    <li><tt>top_p</tt>: An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.</li>
+                    <li><tt>presence_penalty</tt>: Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. (OpenAI only)</li>
+                    <li><tt>frequency_penalty</tt>: Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. (OpenAI only)</li>
+                  </ul>
+                  Settings can also have variables in them (e.g., <tt>temperature</tt>).
+                </li>
+                <li><b>environments</b>, which specifies for each variable, a list of values (HOCON format).
+              </ul>
+            </li>
+            <li>
+              When the query is submitted,
+              we consider all possible assignments of values to variables.
+              For example:
+              <ul>
+                <li>environments has <tt>name: [Boston, New York]</tt> and <tt>temperature: [0, 1]</tt></li>
+                <li>prompt is <tt>${name} is a</tt></li>
+                <li>settings is <tt>temperature: ${temperature}</tt></li>
+              </ul>
+              This gives rise to 4 <b>requests</b>:
+              <ul>
+                <li>prompt: <tt>Boston is a</tt>, temperature: <tt>0</tt></li>
+                <li>prompt: <tt>Boston is a</tt>, temperature: <tt>1</tt></li>
+                <li>prompt: <tt>New York is a</tt>, temperature: <tt>0</tt></li>
+                <li>prompt: <tt>New York is a</tt>, temperature: <tt>1</tt></li>
+              </ul>
+            </li>
+          </ul>
+
+          <h2><a name="quotas">Quotas</a></h2>
+          <p>
+            For each model group (e.g., <tt>gpt3</tt>) and time granularity
+            (e.g., <tt>daily</tt>, <tt>monthly</tt>, <tt>total</tt>), you are
+            given a quota of a certain number of tokens.
+            Once you go over that number, you won't be able to use the API.
+            However, note that requests that have already been made (by you or
+            anyone) that are cached are not counted towards your quota.
+            For example, if your daily quota for <tt>gpt3</tt> is <tt>10000</tt>,
+            that means each day, you will get 10000 tokens.
+          </p>
+        </div>
+      </div>
+		</div>
+
+		<script src="https://code.jquery.com/jquery-3.2.1.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.11.0/umd/popper.min.js" integrity="sha384-b/U6ypiBEHpOf/4+1nzFpr53nxSS+GLCkfwBdFNTxtclqqenISfwAzpKaMNFNmj4" crossorigin="anonymous"></script>
+		<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/js/bootstrap.min.js" integrity="sha384-h0AbiXch4ZDo7tp9hKZ4TsHbi047NrKGLO3SEJAg45jXxnGIfYzk4Si90RDIqNm1" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.tablesorter/2.29.0/js/jquery.tablesorter.min.js"></script>
+		<script src="general.js"></script>
+		<script src="index.js"></script>
+	</body>
+</html>

helm/proxy/static/index.css

@@ -0,0 +1,57 @@
+.warning {
+  color: red;
+}
+
+.active {
+  background-color: #f0f0f0;
+}
+
+.examples-block {
+  margin-left: 120px;
+  margin-top: 10px;
+  margin-bottom: 10px;
+}
+
+.request {
+  background-color: #f0f0f0;
+  margin-top: 20px;
+  margin-bottom: 20px;
+}
+
+.completion {
+  border: solid 1px;
+  border-color: #c0c0c0;
+  padding: 3px;
+  margin-top: 5px;
+  margin-bottom: 5px;
+}
+
+.completion:hover {
+  background-color: #e8e8e8;
+}
+
+.metadata {
+  float: right;
+  font-style: italic;
+}
+
+.token:hover {
+  background-color: lightgreen;
+}
+
+.block {
+  margin: 15px;
+  padding: 15px;
+  border: solid 1px;
+  border-color: lightgray;
+}
+
+td {
+  text-align: left;
+  vertical-align: top;
+}
+
+.help-icon {
+  margin-left: 3px;
+  margin-right: 3px;
+}

helm/proxy/static/index.html

@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<title>CRFM Models API</title>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+		<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/css/bootstrap.min.css" integrity="sha384-/Y6pD6FV/Vv2HJnA6t+vslU6fwYXjCFtcEpHbNJ0lyAFsXTsjBbfaDjzALeQsN6M" crossorigin="anonymous">
+		<link rel="stylesheet" type="text/css" href="index.css">
+	</head>
+
+	<body>
+		<div class="container-fluid">
+      <nav class="navbar navbar-expand-sm navbar-light bg-faded">
+        <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#nav-content" aria-controls="nav-content" aria-expanded="false" aria-label="Toggle navigation">
+          <span class="navbar-toggler-icon"></span>
+        </button>
+
+        <div class="collapse navbar-collapse" id="nav-content">
+          <ul class="navbar-nav">
+            <li class="nav-item"><a class="nav-link active" href="index.html">Query</a></li>
+            <li class="nav-item"><a class="nav-link" href="help.html">Help</a></li>
+            <li class="nav-item" id="loginInfo"></li>
+          </ul>
+        </div>
+      </nav>
+
+      <div class="row">
+        <div class="col-sm-12" id="main">
+        </div>
+      </div>
+		</div>
+
+		<script src="https://code.jquery.com/jquery-3.2.1.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.11.0/umd/popper.min.js" integrity="sha384-b/U6ypiBEHpOf/4+1nzFpr53nxSS+GLCkfwBdFNTxtclqqenISfwAzpKaMNFNmj4" crossorigin="anonymous"></script>
+		<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/js/bootstrap.min.js" integrity="sha384-h0AbiXch4ZDo7tp9hKZ4TsHbi047NrKGLO3SEJAg45jXxnGIfYzk4Si90RDIqNm1" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.tablesorter/2.29.0/js/jquery.tablesorter.min.js"></script>
+		<script src="general.js"></script>
+		<script src="index.js"></script>
+	</body>
+</html>