PyPI - waterfall - Versions diffs - 0.2.0__tar.gz → 0.2.1__tar.gz - Mend

waterfall 0.2.0tar.gz → 0.2.1tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (20) hide show

waterfall-0.2.1/.vscode/settings.json ADDED Viewed

@@ -0,0 +1,5 @@
+{
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda",
+    "python-envs.pythonProjects": []
+}

{waterfall-0.2.0 → waterfall-0.2.1}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: waterfall
-Version: 0.2.0
+Version: 0.2.1
 Summary: Scalable Framework for Robust Text Watermarking and Provenance for LLMs
 Project-URL: Homepage, https://github.com/aoi3142/Waterfall
 Project-URL: Issues, https://github.com/aoi3142/Waterfall/issues

{waterfall-0.2.0 → waterfall-0.2.1}/pyproject.toml RENAMED Viewed

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 [project]
 name = "waterfall"
-version = "0.2.0"
+version = "0.2.1"
 authors = [
   { name = "Xinyuan Niu", email="aperture@outlook.sg" }
 ]

waterfall-0.2.1/test_vllm.ipynb ADDED Viewed

@@ -0,0 +1,697 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "df7c520a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "# os.environ[\"VLLM_USE_V1\"] = \"0\"\n",
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "79d9fbdb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from vllm import LLM, SamplingParams\n",
+    "from vllm.inputs import TokensPrompt\n",
+    "import torch\n",
+    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
+    "from transformers.generation.beam_search import *\n",
+    "from transformers.modeling_outputs import CausalLMOutputWithPast\n",
+    "from waterfall.WatermarkerBase import Watermarker\n",
+    "\n",
+    "import types\n",
+    "import inspect\n",
+    "from copy import deepcopy\n",
+    "from timeit import timeit"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "6ff6b383",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Cache size: 84538\n"
+     ]
+    }
+   ],
+   "source": [
+    "base_model = 'meta-llama/Meta-Llama-3.1-8B-Instruct'\n",
+    "tokenizer = AutoTokenizer.from_pretrained(base_model)\n",
+    "tokenizer.pad_token = tokenizer.eos_token\n",
+    "tokenizer.padding_side='left'\n",
+    "watermarker = Watermarker(tokenizer)\n",
+    "logits_processor = watermarker.logits_processor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "b6e4d263",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "max_tokens = max_new_tokens = 10"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "03c09dff",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO 08-15 11:03:46 __init__.py:207] Automatically detected platform cuda.\n",
+      "INFO 08-15 11:04:07 config.py:549] This model supports multiple tasks: {'reward', 'embed', 'classify', 'generate', 'score'}. Defaulting to 'generate'.\n",
+      "INFO 08-15 11:04:07 llm_engine.py:234] Initializing a V0 LLM engine (v0.7.3) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=60, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='xgrammar'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=False, chunked_prefill_enabled=False, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={\"splitting_ops\":[],\"compile_sizes\":[],\"cudagraph_capture_sizes\":[256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],\"max_capture_size\":256}, use_cached_outputs=False, \n",
+      "INFO 08-15 11:04:12 cuda.py:229] Using Flash Attention backend.\n",
+      "INFO 08-15 11:04:20 model_runner.py:1110] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct...\n",
+      "INFO 08-15 11:04:23 weight_utils.py:254] Using model weights format ['*.safetensors']\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7f3d860bd5c24cbe8695345e9f2d9625",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading safetensors checkpoint shards:   0% Completed | 0/4 [00:00<?, ?it/s]\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO 08-15 11:04:28 model_runner.py:1115] Loading model weights took 14.9888 GB\n",
+      "INFO 08-15 11:04:29 worker.py:267] Memory profiling takes 0.44 seconds\n",
+      "INFO 08-15 11:04:29 worker.py:267] the current vLLM instance can use total_gpu_memory (79.10GiB) x gpu_memory_utilization (0.60) = 47.46GiB\n",
+      "INFO 08-15 11:04:29 worker.py:267] model weights take 14.99GiB; non_torch_memory takes 0.14GiB; PyTorch activation peak memory takes 0.24GiB; the rest of the memory reserved for KV Cache is 32.09GiB.\n",
+      "INFO 08-15 11:04:29 executor_base.py:111] # cuda blocks: 16429, # CPU blocks: 2048\n",
+      "INFO 08-15 11:04:29 executor_base.py:116] Maximum concurrency for 60 tokens per request: 4381.07x\n",
+      "INFO 08-15 11:04:32 model_runner.py:1434] Capturing cudagraphs for decoding. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI. If out-of-memory error occurs during cudagraph capture, consider decreasing `gpu_memory_utilization` or switching to eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Capturing CUDA graph shapes: 100%|██████████| 35/35 [00:23<00:00,  1.52it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO 08-15 11:04:55 model_runner.py:1562] Graph capturing finished in 23 secs, took 0.32 GiB\n",
+      "INFO 08-15 11:04:55 llm_engine.py:436] init engine (profile, create kv cache, warmup model) took 26.46 seconds\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "vllm = LLM(base_model,\n",
+    "            tokenizer=base_model, \n",
+    "            gpu_memory_utilization=0.6, \n",
+    "            max_model_len=50 + max_tokens,\n",
+    "            )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "18f02f1e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4f33376b6f9f40a09b52211aafdd89b8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    del model\n",
+    "    torch.cuda.empty_cache()\n",
+    "except:\n",
+    "    pass\n",
+    "model = AutoModelForCausalLM.from_pretrained(base_model, torch_dtype=torch.bfloat16, device_map=\"auto\")\n",
+    "model.eval()\n",
+    "\n",
+    "# Save the original forward\n",
+    "model._original_forward = model.forward"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "fad72592",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1df8f593be4445ef8144ea33f04ece48",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "test = AutoModelForCausalLM.from_pretrained(base_model, torch_dtype=torch.bfloat16, device_map=\"meta\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ff23e4a2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# def forward(\n",
+    "#     self,\n",
+    "#     input_ids: Optional[torch.LongTensor] = None,\n",
+    "#     attention_mask: Optional[torch.Tensor] = None,\n",
+    "#     position_ids: Optional[torch.LongTensor] = None,\n",
+    "#     past_key_values: Optional[Cache] = None,\n",
+    "#     inputs_embeds: Optional[torch.FloatTensor] = None,\n",
+    "#     labels: Optional[torch.LongTensor] = None,\n",
+    "#     use_cache: Optional[bool] = None,\n",
+    "#     cache_position: Optional[torch.LongTensor] = None,\n",
+    "#     logits_to_keep: Union[int, torch.Tensor] = 0,\n",
+    "#     **kwargs: Unpack[TransformersKwargs],\n",
+    "# ) -> CausalLMOutputWithPast:\n",
+    "#     r\"\"\"\n",
+    "#     Example:\n",
+    "\n",
+    "#     ```python\n",
+    "#     >>> from transformers import AutoTokenizer, LlamaForCausalLM\n",
+    "\n",
+    "#     >>> model = LlamaForCausalLM.from_pretrained(\"meta-llama/Llama-2-7b-hf\")\n",
+    "#     >>> tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-hf\")\n",
+    "\n",
+    "#     >>> prompt = \"Hey, are you conscious? Can you talk to me?\"\n",
+    "#     >>> inputs = tokenizer(prompt, return_tensors=\"pt\")\n",
+    "\n",
+    "#     >>> # Generate\n",
+    "#     >>> generate_ids = model.generate(inputs.input_ids, max_length=30)\n",
+    "#     >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]\n",
+    "#     \"Hey, are you conscious? Can you talk to me?\\nI'm not conscious, but I can talk to you.\"\n",
+    "#     ```\"\"\"\n",
+    "#     outputs: BaseModelOutputWithPast = self.model(\n",
+    "#         input_ids=input_ids,\n",
+    "#         attention_mask=attention_mask,\n",
+    "#         position_ids=position_ids,\n",
+    "#         past_key_values=past_key_values,\n",
+    "#         inputs_embeds=inputs_embeds,\n",
+    "#         use_cache=use_cache,\n",
+    "#         cache_position=cache_position,\n",
+    "#         **kwargs,\n",
+    "#     )\n",
+    "\n",
+    "#     hidden_states = outputs.last_hidden_state\n",
+    "#     # Only compute necessary logits, and do not upcast them to float if we are not computing the loss\n",
+    "#     slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep\n",
+    "#     logits = self.lm_head(hidden_states[:, slice_indices, :])\n",
+    "\n",
+    "#     loss = None\n",
+    "#     if labels is not None:\n",
+    "#         loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)\n",
+    "\n",
+    "#     return CausalLMOutputWithPast(\n",
+    "#         loss=loss,\n",
+    "#         logits=logits,\n",
+    "#         past_key_values=outputs.past_key_values,\n",
+    "#         hidden_states=outputs.hidden_states,\n",
+    "#         attentions=outputs.attentions,\n",
+    "#     )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "820f4c9c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt_fn = lambda content: tokenizer.apply_chat_template([\n",
+    "    {\"role\": \"user\", \"content\": content},\n",
+    "], tokenize = False, add_generation_prompt = True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "3813b71e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "inputs = [prompt_fn(\"What is the meaning of life?\"), prompt_fn(\"When was Obama born?\")]\n",
+    "tokd_input = tokenizer(inputs, return_tensors=\"pt\", add_special_tokens=False, padding=True).to(model.device)\n",
+    "num_beam_groups = 2\n",
+    "beams_per_group = 3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cfd0c000",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'input_ids': tensor([[128000, 128006,   9125, 128007,    271,  38766,   1303,  33025,   2696,\n",
+       "             25,   6790,    220,   2366,     18,    198,  15724,   2696,     25,\n",
+       "            220,   1627,  10263,    220,   2366,     19,    271, 128009, 128006,\n",
+       "            882, 128007,    271,   3923,    374,    279,   7438,    315,   2324,\n",
+       "             30, 128009, 128006,  78191, 128007,    271],\n",
+       "        [128009, 128009, 128000, 128006,   9125, 128007,    271,  38766,   1303,\n",
+       "          33025,   2696,     25,   6790,    220,   2366,     18,    198,  15724,\n",
+       "           2696,     25,    220,   1627,  10263,    220,   2366,     19,    271,\n",
+       "         128009, 128006,    882, 128007,    271,   4599,    574,   7250,   9405,\n",
+       "             30, 128009, 128006,  78191, 128007,    271]], device='cuda:0'), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
+       "        [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]],\n",
+       "       device='cuda:0')}"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokd_input"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "57bfea89",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Group Beam Search is scheduled to be moved to a `custom_generate` repository in v4.55.0. To prevent loss of backward compatibility, add `trust_remote_code=True` to your `generate` call.\n"
+     ]
+    }
+   ],
+   "source": [
+    "out = model.generate(\n",
+    "                **tokd_input,\n",
+    "                max_new_tokens=1,\n",
+    "                logits_processor=[logits_processor],\n",
+    "                pad_token_id=tokenizer.eos_token_id,\n",
+    "                do_sample = False, temperature=None, top_p=None,\n",
+    "                num_beams = num_beam_groups * beams_per_group,\n",
+    "                num_beam_groups = num_beam_groups,\n",
+    "                num_return_sequences = num_beam_groups * beams_per_group,\n",
+    "                diversity_penalty = 0.5, use_cache = False\n",
+    "                )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "0811859f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "N = test.lm_head.out_features\n",
+    "\n",
+    "beam_search_params = SamplingParams(logprobs=20,\n",
+    "                                    max_tokens=1,\n",
+    "                                    temperature=1.0,\n",
+    "                                    detokenize=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "d424a6a9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "input_ids = tokd_input[\"input_ids\"]\n",
+    "prompts_batch = [TokensPrompt(prompt_token_ids=i) for i in input_ids]\n",
+    "output = vllm.generate(prompts_batch,\n",
+    "                                   sampling_params=beam_search_params,\n",
+    "                                   use_tqdm=False)\n",
+    "logits = []\n",
+    "size = (N,)\n",
+    "for o in output:\n",
+    "    o = o.outputs[0].logprobs[0]\n",
+    "    tensor = torch.full(size, float('-inf'))\n",
+    "\n",
+    "    # Set values from the dictionary\n",
+    "    if o:  # only if dictionary is not empty\n",
+    "        indices = torch.tensor(list(o.keys()))\n",
+    "        values = torch.tensor([i.logprob for i in o.values()])\n",
+    "        tensor[indices] = values\n",
+    "\n",
+    "    logits.append(tensor)\n",
+    "logits = torch.stack(logits).unsqueeze(1)\n",
+    "output_dict = CausalLMOutputWithPast(\n",
+    "        loss=None,\n",
+    "        logits=logits,\n",
+    "        past_key_values=None,\n",
+    "        hidden_states=None,\n",
+    "        attentions=None,\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d2df69fb",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Using patched forward:\n",
+      "Patched forward signature: {'position_ids', 'past_key_values', 'kwargs', 'cache_position', 'attention_mask', 'use_cache', 'logits_to_keep', 'labels', 'inputs_embeds', 'input_ids'}\n",
+      "791 {791: Logprob(logprob=-0.020645949989557266, rank=1, decoded_token=None), 4054: Logprob(logprob=-4.645646095275879, rank=2, decoded_token=None), 32: Logprob(logprob=-5.020646095275879, rank=3, decoded_token=None), 3923: Logprob(logprob=-6.145646095275879, rank=4, decoded_token=None), 30690: Logprob(logprob=-6.770646095275879, rank=5, decoded_token=None), 20397: Logprob(logprob=-7.520646095275879, rank=6, decoded_token=None), 3947: Logprob(logprob=-9.395646095275879, rank=7, decoded_token=None), 8142: Logprob(logprob=-9.833146095275879, rank=8, decoded_token=None), 31140: Logprob(logprob=-9.895646095275879, rank=9, decoded_token=None), 16533: Logprob(logprob=-10.645646095275879, rank=10, decoded_token=None), 32576: Logprob(logprob=-10.833146095275879, rank=11, decoded_token=None), 2028: Logprob(logprob=-10.958146095275879, rank=12, decoded_token=None), 38356: Logprob(logprob=-11.583146095275879, rank=13, decoded_token=None), 578: Logprob(logprob=-11.708146095275879, rank=14, decoded_token=None), 10227: Logprob(logprob=-11.833146095275879, rank=15, decoded_token=None), 93748: Logprob(logprob=-12.083146095275879, rank=16, decoded_token=None), 31437: Logprob(logprob=-12.083146095275879, rank=17, decoded_token=None), 8607: Logprob(logprob=-12.083146095275879, rank=18, decoded_token=None), 46185: Logprob(logprob=-12.208146095275879, rank=19, decoded_token=None), 71251: Logprob(logprob=-12.270646095275879, rank=20, decoded_token=None)}\n",
+      "791 {791: Logprob(logprob=-0.020645949989557266, rank=1, decoded_token=None), 4054: Logprob(logprob=-4.645646095275879, rank=2, decoded_token=None), 32: Logprob(logprob=-5.020646095275879, rank=3, decoded_token=None), 3923: Logprob(logprob=-6.145646095275879, rank=4, decoded_token=None), 30690: Logprob(logprob=-6.770646095275879, rank=5, decoded_token=None), 20397: Logprob(logprob=-7.520646095275879, rank=6, decoded_token=None), 3947: Logprob(logprob=-9.395646095275879, rank=7, decoded_token=None), 8142: Logprob(logprob=-9.833146095275879, rank=8, decoded_token=None), 31140: Logprob(logprob=-9.895646095275879, rank=9, decoded_token=None), 16533: Logprob(logprob=-10.645646095275879, rank=10, decoded_token=None), 32576: Logprob(logprob=-10.833146095275879, rank=11, decoded_token=None), 2028: Logprob(logprob=-10.958146095275879, rank=12, decoded_token=None), 38356: Logprob(logprob=-11.583146095275879, rank=13, decoded_token=None), 578: Logprob(logprob=-11.708146095275879, rank=14, decoded_token=None), 10227: Logprob(logprob=-11.833146095275879, rank=15, decoded_token=None), 93748: Logprob(logprob=-12.083146095275879, rank=16, decoded_token=None), 31437: Logprob(logprob=-12.083146095275879, rank=17, decoded_token=None), 8607: Logprob(logprob=-12.083146095275879, rank=18, decoded_token=None), 46185: Logprob(logprob=-12.208146095275879, rank=19, decoded_token=None), 71251: Logprob(logprob=-12.270646095275879, rank=20, decoded_token=None)}\n",
+      "791 {791: Logprob(logprob=-0.020645949989557266, rank=1, decoded_token=None), 4054: Logprob(logprob=-4.645646095275879, rank=2, decoded_token=None), 32: Logprob(logprob=-5.020646095275879, rank=3, decoded_token=None), 3923: Logprob(logprob=-6.145646095275879, rank=4, decoded_token=None), 30690: Logprob(logprob=-6.770646095275879, rank=5, decoded_token=None), 20397: Logprob(logprob=-7.520646095275879, rank=6, decoded_token=None), 3947: Logprob(logprob=-9.395646095275879, rank=7, decoded_token=None), 8142: Logprob(logprob=-9.833146095275879, rank=8, decoded_token=None), 31140: Logprob(logprob=-9.895646095275879, rank=9, decoded_token=None), 16533: Logprob(logprob=-10.645646095275879, rank=10, decoded_token=None), 32576: Logprob(logprob=-10.833146095275879, rank=11, decoded_token=None), 2028: Logprob(logprob=-10.958146095275879, rank=12, decoded_token=None), 38356: Logprob(logprob=-11.583146095275879, rank=13, decoded_token=None), 578: Logprob(logprob=-11.708146095275879, rank=14, decoded_token=None), 10227: Logprob(logprob=-11.833146095275879, rank=15, decoded_token=None), 93748: Logprob(logprob=-12.083146095275879, rank=16, decoded_token=None), 31437: Logprob(logprob=-12.083146095275879, rank=17, decoded_token=None), 8607: Logprob(logprob=-12.083146095275879, rank=18, decoded_token=None), 46185: Logprob(logprob=-12.208146095275879, rank=19, decoded_token=None), 71251: Logprob(logprob=-12.270646095275879, rank=20, decoded_token=None)}\n",
+      "791 {791: Logprob(logprob=-0.020645949989557266, rank=1, decoded_token=None), 4054: Logprob(logprob=-4.645646095275879, rank=2, decoded_token=None), 32: Logprob(logprob=-5.020646095275879, rank=3, decoded_token=None), 3923: Logprob(logprob=-6.145646095275879, rank=4, decoded_token=None), 30690: Logprob(logprob=-6.770646095275879, rank=5, decoded_token=None), 20397: Logprob(logprob=-7.520646095275879, rank=6, decoded_token=None), 3947: Logprob(logprob=-9.395646095275879, rank=7, decoded_token=None), 8142: Logprob(logprob=-9.833146095275879, rank=8, decoded_token=None), 31140: Logprob(logprob=-9.895646095275879, rank=9, decoded_token=None), 16533: Logprob(logprob=-10.645646095275879, rank=10, decoded_token=None), 32576: Logprob(logprob=-10.833146095275879, rank=11, decoded_token=None), 2028: Logprob(logprob=-10.958146095275879, rank=12, decoded_token=None), 38356: Logprob(logprob=-11.583146095275879, rank=13, decoded_token=None), 578: Logprob(logprob=-11.708146095275879, rank=14, decoded_token=None), 10227: Logprob(logprob=-11.833146095275879, rank=15, decoded_token=None), 93748: Logprob(logprob=-12.083146095275879, rank=16, decoded_token=None), 31437: Logprob(logprob=-12.083146095275879, rank=17, decoded_token=None), 8607: Logprob(logprob=-12.083146095275879, rank=18, decoded_token=None), 46185: Logprob(logprob=-12.208146095275879, rank=19, decoded_token=None), 71251: Logprob(logprob=-12.270646095275879, rank=20, decoded_token=None)}\n",
+      "791 {791: Logprob(logprob=-0.020645949989557266, rank=1, decoded_token=None), 4054: Logprob(logprob=-4.645646095275879, rank=2, decoded_token=None), 32: Logprob(logprob=-5.020646095275879, rank=3, decoded_token=None), 3923: Logprob(logprob=-6.145646095275879, rank=4, decoded_token=None), 30690: Logprob(logprob=-6.770646095275879, rank=5, decoded_token=None), 20397: Logprob(logprob=-7.520646095275879, rank=6, decoded_token=None), 3947: Logprob(logprob=-9.395646095275879, rank=7, decoded_token=None), 8142: Logprob(logprob=-9.833146095275879, rank=8, decoded_token=None), 31140: Logprob(logprob=-9.895646095275879, rank=9, decoded_token=None), 16533: Logprob(logprob=-10.645646095275879, rank=10, decoded_token=None), 32576: Logprob(logprob=-10.833146095275879, rank=11, decoded_token=None), 2028: Logprob(logprob=-10.958146095275879, rank=12, decoded_token=None), 38356: Logprob(logprob=-11.583146095275879, rank=13, decoded_token=None), 578: Logprob(logprob=-11.708146095275879, rank=14, decoded_token=None), 10227: Logprob(logprob=-11.833146095275879, rank=15, decoded_token=None), 93748: Logprob(logprob=-12.083146095275879, rank=16, decoded_token=None), 31437: Logprob(logprob=-12.083146095275879, rank=17, decoded_token=None), 8607: Logprob(logprob=-12.083146095275879, rank=18, decoded_token=None), 46185: Logprob(logprob=-12.208146095275879, rank=19, decoded_token=None), 71251: Logprob(logprob=-12.270646095275879, rank=20, decoded_token=None)}\n",
+      "791 {791: Logprob(logprob=-0.020645949989557266, rank=1, decoded_token=None), 4054: Logprob(logprob=-4.645646095275879, rank=2, decoded_token=None), 32: Logprob(logprob=-5.020646095275879, rank=3, decoded_token=None), 3923: Logprob(logprob=-6.145646095275879, rank=4, decoded_token=None), 30690: Logprob(logprob=-6.770646095275879, rank=5, decoded_token=None), 20397: Logprob(logprob=-7.520646095275879, rank=6, decoded_token=None), 3947: Logprob(logprob=-9.395646095275879, rank=7, decoded_token=None), 8142: Logprob(logprob=-9.833146095275879, rank=8, decoded_token=None), 31140: Logprob(logprob=-9.895646095275879, rank=9, decoded_token=None), 16533: Logprob(logprob=-10.645646095275879, rank=10, decoded_token=None), 32576: Logprob(logprob=-10.833146095275879, rank=11, decoded_token=None), 2028: Logprob(logprob=-10.958146095275879, rank=12, decoded_token=None), 38356: Logprob(logprob=-11.583146095275879, rank=13, decoded_token=None), 578: Logprob(logprob=-11.708146095275879, rank=14, decoded_token=None), 10227: Logprob(logprob=-11.833146095275879, rank=15, decoded_token=None), 93748: Logprob(logprob=-12.083146095275879, rank=16, decoded_token=None), 31437: Logprob(logprob=-12.083146095275879, rank=17, decoded_token=None), 8607: Logprob(logprob=-12.083146095275879, rank=18, decoded_token=None), 46185: Logprob(logprob=-12.208146095275879, rank=19, decoded_token=None), 71251: Logprob(logprob=-12.270646095275879, rank=20, decoded_token=None)}\n",
+      "3511 {3511: Logprob(logprob=-0.023903636261820793, rank=1, decoded_token=None), 31945: Logprob(logprob=-3.8989036083221436, rank=2, decoded_token=None), 27229: Logprob(logprob=-6.523903846740723, rank=3, decoded_token=None), 791: Logprob(logprob=-7.648903846740723, rank=4, decoded_token=None), 24448: Logprob(logprob=-8.148903846740723, rank=5, decoded_token=None), 2078: Logprob(logprob=-8.523903846740723, rank=6, decoded_token=None), 1271: Logprob(logprob=-8.773903846740723, rank=7, decoded_token=None), 52: Logprob(logprob=-9.023903846740723, rank=8, decoded_token=None), 45437: Logprob(logprob=-9.148903846740723, rank=9, decoded_token=None), 1548: Logprob(logprob=-9.273903846740723, rank=10, decoded_token=None), 40: Logprob(logprob=-9.273903846740723, rank=11, decoded_token=None), 35627: Logprob(logprob=-10.023903846740723, rank=12, decoded_token=None), 23175: Logprob(logprob=-10.148903846740723, rank=13, decoded_token=None), 14704: Logprob(logprob=-10.398903846740723, rank=14, decoded_token=None), 32: Logprob(logprob=-10.648903846740723, rank=15, decoded_token=None), 2308: Logprob(logprob=-10.773903846740723, rank=16, decoded_token=None), 34165: Logprob(logprob=-10.898903846740723, rank=17, decoded_token=None), 644: Logprob(logprob=-10.898903846740723, rank=18, decoded_token=None), 1966: Logprob(logprob=-11.023903846740723, rank=19, decoded_token=None), 18735: Logprob(logprob=-11.273903846740723, rank=20, decoded_token=None)}\n",
+      "3511 {3511: Logprob(logprob=-0.023903636261820793, rank=1, decoded_token=None), 31945: Logprob(logprob=-3.8989036083221436, rank=2, decoded_token=None), 27229: Logprob(logprob=-6.523903846740723, rank=3, decoded_token=None), 791: Logprob(logprob=-7.648903846740723, rank=4, decoded_token=None), 24448: Logprob(logprob=-8.148903846740723, rank=5, decoded_token=None), 2078: Logprob(logprob=-8.523903846740723, rank=6, decoded_token=None), 1271: Logprob(logprob=-8.773903846740723, rank=7, decoded_token=None), 52: Logprob(logprob=-9.023903846740723, rank=8, decoded_token=None), 45437: Logprob(logprob=-9.148903846740723, rank=9, decoded_token=None), 1548: Logprob(logprob=-9.273903846740723, rank=10, decoded_token=None), 40: Logprob(logprob=-9.273903846740723, rank=11, decoded_token=None), 35627: Logprob(logprob=-10.023903846740723, rank=12, decoded_token=None), 23175: Logprob(logprob=-10.148903846740723, rank=13, decoded_token=None), 14704: Logprob(logprob=-10.398903846740723, rank=14, decoded_token=None), 32: Logprob(logprob=-10.648903846740723, rank=15, decoded_token=None), 2308: Logprob(logprob=-10.773903846740723, rank=16, decoded_token=None), 34165: Logprob(logprob=-10.898903846740723, rank=17, decoded_token=None), 644: Logprob(logprob=-10.898903846740723, rank=18, decoded_token=None), 1966: Logprob(logprob=-11.023903846740723, rank=19, decoded_token=None), 18735: Logprob(logprob=-11.273903846740723, rank=20, decoded_token=None)}\n",
+      "3511 {3511: Logprob(logprob=-0.023903636261820793, rank=1, decoded_token=None), 31945: Logprob(logprob=-3.8989036083221436, rank=2, decoded_token=None), 27229: Logprob(logprob=-6.523903846740723, rank=3, decoded_token=None), 791: Logprob(logprob=-7.648903846740723, rank=4, decoded_token=None), 24448: Logprob(logprob=-8.148903846740723, rank=5, decoded_token=None), 2078: Logprob(logprob=-8.523903846740723, rank=6, decoded_token=None), 1271: Logprob(logprob=-8.773903846740723, rank=7, decoded_token=None), 52: Logprob(logprob=-9.023903846740723, rank=8, decoded_token=None), 45437: Logprob(logprob=-9.148903846740723, rank=9, decoded_token=None), 1548: Logprob(logprob=-9.273903846740723, rank=10, decoded_token=None), 40: Logprob(logprob=-9.273903846740723, rank=11, decoded_token=None), 35627: Logprob(logprob=-10.023903846740723, rank=12, decoded_token=None), 23175: Logprob(logprob=-10.148903846740723, rank=13, decoded_token=None), 14704: Logprob(logprob=-10.398903846740723, rank=14, decoded_token=None), 32: Logprob(logprob=-10.648903846740723, rank=15, decoded_token=None), 2308: Logprob(logprob=-10.773903846740723, rank=16, decoded_token=None), 34165: Logprob(logprob=-10.898903846740723, rank=17, decoded_token=None), 644: Logprob(logprob=-10.898903846740723, rank=18, decoded_token=None), 1966: Logprob(logprob=-11.023903846740723, rank=19, decoded_token=None), 18735: Logprob(logprob=-11.273903846740723, rank=20, decoded_token=None)}\n",
+      "3511 {3511: Logprob(logprob=-0.023903636261820793, rank=1, decoded_token=None), 31945: Logprob(logprob=-3.8989036083221436, rank=2, decoded_token=None), 27229: Logprob(logprob=-6.523903846740723, rank=3, decoded_token=None), 791: Logprob(logprob=-7.648903846740723, rank=4, decoded_token=None), 24448: Logprob(logprob=-8.148903846740723, rank=5, decoded_token=None), 2078: Logprob(logprob=-8.523903846740723, rank=6, decoded_token=None), 1271: Logprob(logprob=-8.773903846740723, rank=7, decoded_token=None), 52: Logprob(logprob=-9.023903846740723, rank=8, decoded_token=None), 45437: Logprob(logprob=-9.148903846740723, rank=9, decoded_token=None), 1548: Logprob(logprob=-9.273903846740723, rank=10, decoded_token=None), 40: Logprob(logprob=-9.273903846740723, rank=11, decoded_token=None), 35627: Logprob(logprob=-10.023903846740723, rank=12, decoded_token=None), 23175: Logprob(logprob=-10.148903846740723, rank=13, decoded_token=None), 14704: Logprob(logprob=-10.398903846740723, rank=14, decoded_token=None), 32: Logprob(logprob=-10.648903846740723, rank=15, decoded_token=None), 2308: Logprob(logprob=-10.773903846740723, rank=16, decoded_token=None), 34165: Logprob(logprob=-10.898903846740723, rank=17, decoded_token=None), 644: Logprob(logprob=-10.898903846740723, rank=18, decoded_token=None), 1966: Logprob(logprob=-11.023903846740723, rank=19, decoded_token=None), 18735: Logprob(logprob=-11.273903846740723, rank=20, decoded_token=None)}\n",
+      "3511 {3511: Logprob(logprob=-0.023903636261820793, rank=1, decoded_token=None), 31945: Logprob(logprob=-3.8989036083221436, rank=2, decoded_token=None), 27229: Logprob(logprob=-6.523903846740723, rank=3, decoded_token=None), 791: Logprob(logprob=-7.648903846740723, rank=4, decoded_token=None), 24448: Logprob(logprob=-8.148903846740723, rank=5, decoded_token=None), 2078: Logprob(logprob=-8.523903846740723, rank=6, decoded_token=None), 1271: Logprob(logprob=-8.773903846740723, rank=7, decoded_token=None), 52: Logprob(logprob=-9.023903846740723, rank=8, decoded_token=None), 45437: Logprob(logprob=-9.148903846740723, rank=9, decoded_token=None), 1548: Logprob(logprob=-9.273903846740723, rank=10, decoded_token=None), 40: Logprob(logprob=-9.273903846740723, rank=11, decoded_token=None), 35627: Logprob(logprob=-10.023903846740723, rank=12, decoded_token=None), 23175: Logprob(logprob=-10.148903846740723, rank=13, decoded_token=None), 14704: Logprob(logprob=-10.398903846740723, rank=14, decoded_token=None), 32: Logprob(logprob=-10.648903846740723, rank=15, decoded_token=None), 2308: Logprob(logprob=-10.773903846740723, rank=16, decoded_token=None), 34165: Logprob(logprob=-10.898903846740723, rank=17, decoded_token=None), 644: Logprob(logprob=-10.898903846740723, rank=18, decoded_token=None), 1966: Logprob(logprob=-11.023903846740723, rank=19, decoded_token=None), 18735: Logprob(logprob=-11.273903846740723, rank=20, decoded_token=None)}\n",
+      "3511 {3511: Logprob(logprob=-0.023903636261820793, rank=1, decoded_token=None), 31945: Logprob(logprob=-3.8989036083221436, rank=2, decoded_token=None), 27229: Logprob(logprob=-6.523903846740723, rank=3, decoded_token=None), 791: Logprob(logprob=-7.648903846740723, rank=4, decoded_token=None), 24448: Logprob(logprob=-8.148903846740723, rank=5, decoded_token=None), 2078: Logprob(logprob=-8.523903846740723, rank=6, decoded_token=None), 1271: Logprob(logprob=-8.773903846740723, rank=7, decoded_token=None), 52: Logprob(logprob=-9.023903846740723, rank=8, decoded_token=None), 45437: Logprob(logprob=-9.148903846740723, rank=9, decoded_token=None), 1548: Logprob(logprob=-9.273903846740723, rank=10, decoded_token=None), 40: Logprob(logprob=-9.273903846740723, rank=11, decoded_token=None), 35627: Logprob(logprob=-10.023903846740723, rank=12, decoded_token=None), 23175: Logprob(logprob=-10.148903846740723, rank=13, decoded_token=None), 14704: Logprob(logprob=-10.398903846740723, rank=14, decoded_token=None), 32: Logprob(logprob=-10.648903846740723, rank=15, decoded_token=None), 2308: Logprob(logprob=-10.773903846740723, rank=16, decoded_token=None), 34165: Logprob(logprob=-10.898903846740723, rank=17, decoded_token=None), 644: Logprob(logprob=-10.898903846740723, rank=18, decoded_token=None), 1966: Logprob(logprob=-11.023903846740723, rank=19, decoded_token=None), 18735: Logprob(logprob=-11.273903846740723, rank=20, decoded_token=None)}\n",
+      "3488 {3488: Logprob(logprob=-0.2706400752067566, rank=1, decoded_token=None), 7438: Logprob(logprob=-1.5206401348114014, rank=2, decoded_token=None), 7434: Logprob(logprob=-4.520639896392822, rank=3, decoded_token=None), 4325: Logprob(logprob=-5.895639896392822, rank=4, decoded_token=None), 330: Logprob(logprob=-6.145639896392822, rank=5, decoded_token=None), 4320: Logprob(logprob=-7.020639896392822, rank=6, decoded_token=None), 17139: Logprob(logprob=-7.145639896392822, rank=7, decoded_token=None), 2271: Logprob(logprob=-8.02064037322998, rank=8, decoded_token=None), 364: Logprob(logprob=-8.39564037322998, rank=9, decoded_token=None), 2778: Logprob(logprob=-9.39564037322998, rank=10, decoded_token=None), 35825: Logprob(logprob=-9.70814037322998, rank=11, decoded_token=None), 14209: Logprob(logprob=-9.95814037322998, rank=12, decoded_token=None), 49203: Logprob(logprob=-10.20814037322998, rank=13, decoded_token=None), 23347: Logprob(logprob=-10.39564037322998, rank=14, decoded_token=None), 17571: Logprob(logprob=-10.58314037322998, rank=15, decoded_token=None), 4860: Logprob(logprob=-10.64564037322998, rank=16, decoded_token=None), 3319: Logprob(logprob=-10.83314037322998, rank=17, decoded_token=None), 4751: Logprob(logprob=-10.89564037322998, rank=18, decoded_token=None), 16225: Logprob(logprob=-10.89564037322998, rank=19, decoded_token=None), 7419: Logprob(logprob=-10.95814037322998, rank=20, decoded_token=None)}\n",
+      "315 {315: Logprob(logprob=-3.1709168979432434e-05, rank=1, decoded_token=None), 3488: Logprob(logprob=-10.375031471252441, rank=2, decoded_token=None), 220: Logprob(logprob=-15.750031471252441, rank=3, decoded_token=None), 279: Logprob(logprob=-16.062532424926758, rank=4, decoded_token=None), 5046: Logprob(logprob=-16.375032424926758, rank=5, decoded_token=None), 1073: Logprob(logprob=-16.937532424926758, rank=6, decoded_token=None), 596: Logprob(logprob=-16.937532424926758, rank=7, decoded_token=None), 35825: Logprob(logprob=-17.437532424926758, rank=8, decoded_token=None), 16225: Logprob(logprob=-17.812532424926758, rank=9, decoded_token=None), 4860: Logprob(logprob=-18.375032424926758, rank=10, decoded_token=None), 3659: Logprob(logprob=-18.437532424926758, rank=11, decoded_token=None), 832: Logprob(logprob=-18.687532424926758, rank=12, decoded_token=None), 1455: Logprob(logprob=-19.000032424926758, rank=13, decoded_token=None), 67739: Logprob(logprob=-19.062532424926758, rank=14, decoded_token=None), 3083: Logprob(logprob=-19.187532424926758, rank=15, decoded_token=None), 430: Logprob(logprob=-19.250032424926758, rank=16, decoded_token=None), 60835: Logprob(logprob=-19.375032424926758, rank=17, decoded_token=None), 43146: Logprob(logprob=-19.437532424926758, rank=18, decoded_token=None), 13189: Logprob(logprob=-19.562532424926758, rank=19, decoded_token=None), 311: Logprob(logprob=-19.625032424926758, rank=20, decoded_token=None)}\n",
+      "3488 {3488: Logprob(logprob=-0.0018615075387060642, rank=1, decoded_token=None), 28254: Logprob(logprob=-7.126861572265625, rank=2, decoded_token=None), 11670: Logprob(logprob=-8.001861572265625, rank=3, decoded_token=None), 58105: Logprob(logprob=-8.126861572265625, rank=4, decoded_token=None), 23490: Logprob(logprob=-9.439361572265625, rank=5, decoded_token=None), 41903: Logprob(logprob=-9.501861572265625, rank=6, decoded_token=None), 6485: Logprob(logprob=-9.689361572265625, rank=7, decoded_token=None), 16188: Logprob(logprob=-10.314361572265625, rank=8, decoded_token=None), 74718: Logprob(logprob=-10.626861572265625, rank=9, decoded_token=None), 16225: Logprob(logprob=-10.626861572265625, rank=10, decoded_token=None), 17693: Logprob(logprob=-11.314361572265625, rank=11, decoded_token=None), 4860: Logprob(logprob=-11.314361572265625, rank=12, decoded_token=None), 57697: Logprob(logprob=-11.751861572265625, rank=13, decoded_token=None), 4325: Logprob(logprob=-11.876861572265625, rank=14, decoded_token=None), 32846: Logprob(logprob=-11.939361572265625, rank=15, decoded_token=None), 66278: Logprob(logprob=-11.939361572265625, rank=16, decoded_token=None), 7998: Logprob(logprob=-12.064361572265625, rank=17, decoded_token=None), 8712: Logprob(logprob=-12.189361572265625, rank=18, decoded_token=None), 73835: Logprob(logprob=-12.376861572265625, rank=19, decoded_token=None), 2271: Logprob(logprob=-12.439361572265625, rank=20, decoded_token=None)}\n",
+      "3488 {3488: Logprob(logprob=-0.2706400752067566, rank=1, decoded_token=None), 7438: Logprob(logprob=-1.5206401348114014, rank=2, decoded_token=None), 7434: Logprob(logprob=-4.520639896392822, rank=3, decoded_token=None), 4325: Logprob(logprob=-5.895639896392822, rank=4, decoded_token=None), 330: Logprob(logprob=-6.145639896392822, rank=5, decoded_token=None), 4320: Logprob(logprob=-7.020639896392822, rank=6, decoded_token=None), 17139: Logprob(logprob=-7.145639896392822, rank=7, decoded_token=None), 2271: Logprob(logprob=-8.02064037322998, rank=8, decoded_token=None), 364: Logprob(logprob=-8.39564037322998, rank=9, decoded_token=None), 2778: Logprob(logprob=-9.39564037322998, rank=10, decoded_token=None), 35825: Logprob(logprob=-9.70814037322998, rank=11, decoded_token=None), 14209: Logprob(logprob=-9.95814037322998, rank=12, decoded_token=None), 49203: Logprob(logprob=-10.20814037322998, rank=13, decoded_token=None), 23347: Logprob(logprob=-10.39564037322998, rank=14, decoded_token=None), 17571: Logprob(logprob=-10.58314037322998, rank=15, decoded_token=None), 4860: Logprob(logprob=-10.64564037322998, rank=16, decoded_token=None), 3319: Logprob(logprob=-10.83314037322998, rank=17, decoded_token=None), 4751: Logprob(logprob=-10.89564037322998, rank=18, decoded_token=None), 16225: Logprob(logprob=-10.89564037322998, rank=19, decoded_token=None), 7419: Logprob(logprob=-10.95814037322998, rank=20, decoded_token=None)}\n",
+      "315 {315: Logprob(logprob=-3.1709168979432434e-05, rank=1, decoded_token=None), 3488: Logprob(logprob=-10.375031471252441, rank=2, decoded_token=None), 220: Logprob(logprob=-15.750031471252441, rank=3, decoded_token=None), 279: Logprob(logprob=-16.062532424926758, rank=4, decoded_token=None), 5046: Logprob(logprob=-16.375032424926758, rank=5, decoded_token=None), 1073: Logprob(logprob=-16.937532424926758, rank=6, decoded_token=None), 596: Logprob(logprob=-16.937532424926758, rank=7, decoded_token=None), 35825: Logprob(logprob=-17.437532424926758, rank=8, decoded_token=None), 16225: Logprob(logprob=-17.812532424926758, rank=9, decoded_token=None), 4860: Logprob(logprob=-18.375032424926758, rank=10, decoded_token=None), 3659: Logprob(logprob=-18.437532424926758, rank=11, decoded_token=None), 832: Logprob(logprob=-18.687532424926758, rank=12, decoded_token=None), 1455: Logprob(logprob=-19.000032424926758, rank=13, decoded_token=None), 67739: Logprob(logprob=-19.062532424926758, rank=14, decoded_token=None), 3083: Logprob(logprob=-19.187532424926758, rank=15, decoded_token=None), 430: Logprob(logprob=-19.250032424926758, rank=16, decoded_token=None), 60835: Logprob(logprob=-19.375032424926758, rank=17, decoded_token=None), 43146: Logprob(logprob=-19.437532424926758, rank=18, decoded_token=None), 13189: Logprob(logprob=-19.562532424926758, rank=19, decoded_token=None), 311: Logprob(logprob=-19.625032424926758, rank=20, decoded_token=None)}\n",
+      "3488 {3488: Logprob(logprob=-0.0018615075387060642, rank=1, decoded_token=None), 28254: Logprob(logprob=-7.126861572265625, rank=2, decoded_token=None), 11670: Logprob(logprob=-8.001861572265625, rank=3, decoded_token=None), 58105: Logprob(logprob=-8.126861572265625, rank=4, decoded_token=None), 23490: Logprob(logprob=-9.439361572265625, rank=5, decoded_token=None), 41903: Logprob(logprob=-9.501861572265625, rank=6, decoded_token=None), 6485: Logprob(logprob=-9.689361572265625, rank=7, decoded_token=None), 16188: Logprob(logprob=-10.314361572265625, rank=8, decoded_token=None), 74718: Logprob(logprob=-10.626861572265625, rank=9, decoded_token=None), 16225: Logprob(logprob=-10.626861572265625, rank=10, decoded_token=None), 17693: Logprob(logprob=-11.314361572265625, rank=11, decoded_token=None), 4860: Logprob(logprob=-11.314361572265625, rank=12, decoded_token=None), 57697: Logprob(logprob=-11.751861572265625, rank=13, decoded_token=None), 4325: Logprob(logprob=-11.876861572265625, rank=14, decoded_token=None), 32846: Logprob(logprob=-11.939361572265625, rank=15, decoded_token=None), 66278: Logprob(logprob=-11.939361572265625, rank=16, decoded_token=None), 7998: Logprob(logprob=-12.064361572265625, rank=17, decoded_token=None), 8712: Logprob(logprob=-12.189361572265625, rank=18, decoded_token=None), 73835: Logprob(logprob=-12.376861572265625, rank=19, decoded_token=None), 2271: Logprob(logprob=-12.439361572265625, rank=20, decoded_token=None)}\n",
+      "474 {474: Logprob(logprob=-0.0003861635341309011, rank=1, decoded_token=None), 587: Logprob(logprob=-7.8753862380981445, rank=2, decoded_token=None), 894: Logprob(logprob=-12.625386238098145, rank=3, decoded_token=None), 377: Logprob(logprob=-13.875386238098145, rank=4, decoded_token=None), 613: Logprob(logprob=-14.125386238098145, rank=5, decoded_token=None), 4122: Logprob(logprob=-15.000386238098145, rank=6, decoded_token=None), 13637: Logprob(logprob=-15.125386238098145, rank=7, decoded_token=None), 25062: Logprob(logprob=-15.625386238098145, rank=8, decoded_token=None), 582: Logprob(logprob=-15.750386238098145, rank=9, decoded_token=None), 5827: Logprob(logprob=-16.875385284423828, rank=10, decoded_token=None), 37406: Logprob(logprob=-17.000385284423828, rank=11, decoded_token=None), 7977: Logprob(logprob=-17.250385284423828, rank=12, decoded_token=None), 10956: Logprob(logprob=-17.375385284423828, rank=13, decoded_token=None), 81057: Logprob(logprob=-17.500385284423828, rank=14, decoded_token=None), 77527: Logprob(logprob=-17.625385284423828, rank=15, decoded_token=None), 56659: Logprob(logprob=-18.125385284423828, rank=16, decoded_token=None), 1412: Logprob(logprob=-18.125385284423828, rank=17, decoded_token=None), 263: Logprob(logprob=-18.437885284423828, rank=18, decoded_token=None), 566: Logprob(logprob=-18.562885284423828, rank=19, decoded_token=None), 3520: Logprob(logprob=-18.750385284423828, rank=20, decoded_token=None)}\n",
+      "549 {549: Logprob(logprob=-1.0475294589996338, rank=2, decoded_token=None), 2326: Logprob(logprob=-0.5475294589996338, rank=1, decoded_token=None), 4900: Logprob(logprob=-3.297529458999634, rank=3, decoded_token=None), 3723: Logprob(logprob=-3.422529458999634, rank=4, decoded_token=None), 4872: Logprob(logprob=-7.547529220581055, rank=5, decoded_token=None), 3778: Logprob(logprob=-8.297529220581055, rank=6, decoded_token=None), 7427: Logprob(logprob=-8.547529220581055, rank=7, decoded_token=None), 33440: Logprob(logprob=-9.047529220581055, rank=8, decoded_token=None), 220: Logprob(logprob=-9.547529220581055, rank=9, decoded_token=None), 35681: Logprob(logprob=-10.547529220581055, rank=10, decoded_token=None), 4073: Logprob(logprob=-12.047529220581055, rank=11, decoded_token=None), 86481: Logprob(logprob=-12.297529220581055, rank=12, decoded_token=None), 603: Logprob(logprob=-12.422529220581055, rank=13, decoded_token=None), 62602: Logprob(logprob=-12.672529220581055, rank=14, decoded_token=None), 2078: Logprob(logprob=-13.422529220581055, rank=15, decoded_token=None), 97038: Logprob(logprob=-13.797529220581055, rank=16, decoded_token=None), 29292: Logprob(logprob=-13.797529220581055, rank=17, decoded_token=None), 4203: Logprob(logprob=-13.922529220581055, rank=18, decoded_token=None), 85610: Logprob(logprob=-14.047529220581055, rank=19, decoded_token=None), 12: Logprob(logprob=-14.422529220581055, rank=20, decoded_token=None)}\n",
+      "24448 {24448: Logprob(logprob=-0.04389546066522598, rank=1, decoded_token=None), 7250: Logprob(logprob=-3.1688954830169678, rank=2, decoded_token=None), 4821: Logprob(logprob=-8.043895721435547, rank=3, decoded_token=None), 220: Logprob(logprob=-8.293895721435547, rank=4, decoded_token=None), 32817: Logprob(logprob=-8.418895721435547, rank=5, decoded_token=None), 315: Logprob(logprob=-9.668895721435547, rank=6, decoded_token=None), 3703: Logprob(logprob=-11.918895721435547, rank=7, decoded_token=None), 13142: Logprob(logprob=-12.043895721435547, rank=8, decoded_token=None), 38180: Logprob(logprob=-12.543895721435547, rank=9, decoded_token=None), 37081: Logprob(logprob=-12.543895721435547, rank=10, decoded_token=None), 92169: Logprob(logprob=-12.668895721435547, rank=11, decoded_token=None), 9641: Logprob(logprob=-12.793895721435547, rank=12, decoded_token=None), 45287: Logprob(logprob=-12.918895721435547, rank=13, decoded_token=None), 31858: Logprob(logprob=-13.168895721435547, rank=14, decoded_token=None), 94420: Logprob(logprob=-13.418895721435547, rank=15, decoded_token=None), 4761: Logprob(logprob=-13.668895721435547, rank=16, decoded_token=None), 16150: Logprob(logprob=-13.793895721435547, rank=17, decoded_token=None), 3511: Logprob(logprob=-13.918895721435547, rank=18, decoded_token=None), 45437: Logprob(logprob=-13.918895721435547, rank=19, decoded_token=None), 320: Logprob(logprob=-14.418895721435547, rank=20, decoded_token=None)}\n",
+      "474 {474: Logprob(logprob=-0.0003861635341309011, rank=1, decoded_token=None), 587: Logprob(logprob=-7.8753862380981445, rank=2, decoded_token=None), 894: Logprob(logprob=-12.625386238098145, rank=3, decoded_token=None), 377: Logprob(logprob=-13.875386238098145, rank=4, decoded_token=None), 613: Logprob(logprob=-14.125386238098145, rank=5, decoded_token=None), 4122: Logprob(logprob=-15.000386238098145, rank=6, decoded_token=None), 13637: Logprob(logprob=-15.125386238098145, rank=7, decoded_token=None), 25062: Logprob(logprob=-15.625386238098145, rank=8, decoded_token=None), 582: Logprob(logprob=-15.750386238098145, rank=9, decoded_token=None), 5827: Logprob(logprob=-16.875385284423828, rank=10, decoded_token=None), 37406: Logprob(logprob=-17.000385284423828, rank=11, decoded_token=None), 7977: Logprob(logprob=-17.250385284423828, rank=12, decoded_token=None), 10956: Logprob(logprob=-17.375385284423828, rank=13, decoded_token=None), 81057: Logprob(logprob=-17.500385284423828, rank=14, decoded_token=None), 77527: Logprob(logprob=-17.625385284423828, rank=15, decoded_token=None), 56659: Logprob(logprob=-18.125385284423828, rank=16, decoded_token=None), 1412: Logprob(logprob=-18.125385284423828, rank=17, decoded_token=None), 263: Logprob(logprob=-18.437885284423828, rank=18, decoded_token=None), 566: Logprob(logprob=-18.562885284423828, rank=19, decoded_token=None), 3520: Logprob(logprob=-18.750385284423828, rank=20, decoded_token=None)}\n",
+      "549 {549: Logprob(logprob=-1.0475294589996338, rank=2, decoded_token=None), 2326: Logprob(logprob=-0.5475294589996338, rank=1, decoded_token=None), 4900: Logprob(logprob=-3.297529458999634, rank=3, decoded_token=None), 3723: Logprob(logprob=-3.422529458999634, rank=4, decoded_token=None), 4872: Logprob(logprob=-7.547529220581055, rank=5, decoded_token=None), 3778: Logprob(logprob=-8.297529220581055, rank=6, decoded_token=None), 7427: Logprob(logprob=-8.547529220581055, rank=7, decoded_token=None), 33440: Logprob(logprob=-9.047529220581055, rank=8, decoded_token=None), 220: Logprob(logprob=-9.547529220581055, rank=9, decoded_token=None), 35681: Logprob(logprob=-10.547529220581055, rank=10, decoded_token=None), 4073: Logprob(logprob=-12.047529220581055, rank=11, decoded_token=None), 86481: Logprob(logprob=-12.297529220581055, rank=12, decoded_token=None), 603: Logprob(logprob=-12.422529220581055, rank=13, decoded_token=None), 62602: Logprob(logprob=-12.672529220581055, rank=14, decoded_token=None), 2078: Logprob(logprob=-13.422529220581055, rank=15, decoded_token=None), 97038: Logprob(logprob=-13.797529220581055, rank=16, decoded_token=None), 29292: Logprob(logprob=-13.797529220581055, rank=17, decoded_token=None), 4203: Logprob(logprob=-13.922529220581055, rank=18, decoded_token=None), 85610: Logprob(logprob=-14.047529220581055, rank=19, decoded_token=None), 12: Logprob(logprob=-14.422529220581055, rank=20, decoded_token=None)}\n",
+      "24448 {24448: Logprob(logprob=-0.04389546066522598, rank=1, decoded_token=None), 7250: Logprob(logprob=-3.1688954830169678, rank=2, decoded_token=None), 4821: Logprob(logprob=-8.043895721435547, rank=3, decoded_token=None), 220: Logprob(logprob=-8.293895721435547, rank=4, decoded_token=None), 32817: Logprob(logprob=-8.418895721435547, rank=5, decoded_token=None), 315: Logprob(logprob=-9.668895721435547, rank=6, decoded_token=None), 3703: Logprob(logprob=-11.918895721435547, rank=7, decoded_token=None), 13142: Logprob(logprob=-12.043895721435547, rank=8, decoded_token=None), 38180: Logprob(logprob=-12.543895721435547, rank=9, decoded_token=None), 37081: Logprob(logprob=-12.543895721435547, rank=10, decoded_token=None), 92169: Logprob(logprob=-12.668895721435547, rank=11, decoded_token=None), 9641: Logprob(logprob=-12.793895721435547, rank=12, decoded_token=None), 45287: Logprob(logprob=-12.918895721435547, rank=13, decoded_token=None), 31858: Logprob(logprob=-13.168895721435547, rank=14, decoded_token=None), 94420: Logprob(logprob=-13.418895721435547, rank=15, decoded_token=None), 4761: Logprob(logprob=-13.668895721435547, rank=16, decoded_token=None), 16150: Logprob(logprob=-13.793895721435547, rank=17, decoded_token=None), 3511: Logprob(logprob=-13.918895721435547, rank=18, decoded_token=None), 45437: Logprob(logprob=-13.918895721435547, rank=19, decoded_token=None), 320: Logprob(logprob=-14.418895721435547, rank=20, decoded_token=None)}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/niuxy/maplecg_nfs/miniconda3/envs/vllm_waterfall/lib/python3.12/site-packages/transformers/generation/utils.py:2506: UserWarning: You are calling .generate() with the `input_ids` being on a device type different than your model's device. `input_ids` is on cuda, whereas the model is on meta. You may experience unexpected behaviors or slower generation. Please make sure that you have put `input_ids` to the correct device by calling for example input_ids = input_ids.to('meta') before running `.generate()`.\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "315 {315: Logprob(logprob=-0.006171337328851223, rank=1, decoded_token=None), 330: Logprob(logprob=-5.131171226501465, rank=2, decoded_token=None), 364: Logprob(logprob=-8.631171226501465, rank=3, decoded_token=None), 430: Logprob(logprob=-10.131171226501465, rank=4, decoded_token=None), 922: Logprob(logprob=-11.506171226501465, rank=5, decoded_token=None), 11: Logprob(logprob=-12.131171226501465, rank=6, decoded_token=None), 499: Logprob(logprob=-12.631171226501465, rank=7, decoded_token=None), 1054: Logprob(logprob=-13.381171226501465, rank=8, decoded_token=None), 5196: Logprob(logprob=-13.881171226501465, rank=9, decoded_token=None), 374: Logprob(logprob=-14.568671226501465, rank=10, decoded_token=None), 389: Logprob(logprob=-14.943671226501465, rank=11, decoded_token=None), 14932: Logprob(logprob=-15.068671226501465, rank=12, decoded_token=None), 706: Logprob(logprob=-15.256171226501465, rank=13, decoded_token=None), 311: Logprob(logprob=-15.318671226501465, rank=14, decoded_token=None), 37260: Logprob(logprob=-15.568671226501465, rank=15, decoded_token=None), 3451: Logprob(logprob=-15.631171226501465, rank=16, decoded_token=None), 1073: Logprob(logprob=-16.13117218017578, rank=17, decoded_token=None), 9002: Logprob(logprob=-16.44367218017578, rank=18, decoded_token=None), 5084: Logprob(logprob=-16.50617218017578, rank=19, decoded_token=None), 520: Logprob(logprob=-16.56867218017578, rank=20, decoded_token=None)}\n",
+      "315 {315: Logprob(logprob=0.0, rank=1, decoded_token=None), 323: Logprob(logprob=-17.625, rank=2, decoded_token=None), 477: Logprob(logprob=-18.125, rank=3, decoded_token=None), 8838: Logprob(logprob=-18.125, rank=4, decoded_token=None), 5046: Logprob(logprob=-19.375, rank=5, decoded_token=None), 3659: Logprob(logprob=-20.75, rank=6, decoded_token=None), 11: Logprob(logprob=-20.875, rank=7, decoded_token=None), 2324: Logprob(logprob=-22.125, rank=8, decoded_token=None), 220: Logprob(logprob=-22.125, rank=9, decoded_token=None), 311: Logprob(logprob=-22.25, rank=10, decoded_token=None), 3083: Logprob(logprob=-22.25, rank=11, decoded_token=None), 297: Logprob(logprob=-22.25, rank=12, decoded_token=None), 1073: Logprob(logprob=-22.25, rank=13, decoded_token=None), 10440: Logprob(logprob=-22.625, rank=14, decoded_token=None), 33028: Logprob(logprob=-23.25, rank=15, decoded_token=None), 304: Logprob(logprob=-23.6875, rank=16, decoded_token=None), 60835: Logprob(logprob=-23.9375, rank=17, decoded_token=None), 44323: Logprob(logprob=-24.25, rank=18, decoded_token=None), 1161: Logprob(logprob=-24.3125, rank=19, decoded_token=None), 330: Logprob(logprob=-24.625, rank=20, decoded_token=None)}\n",
+      "315 {315: Logprob(logprob=-3.3854863431770355e-05, rank=1, decoded_token=None), 323: Logprob(logprob=-11.375033378601074, rank=2, decoded_token=None), 330: Logprob(logprob=-11.500033378601074, rank=3, decoded_token=None), 3967: Logprob(logprob=-12.000033378601074, rank=4, decoded_token=None), 364: Logprob(logprob=-12.500033378601074, rank=5, decoded_token=None), 430: Logprob(logprob=-14.125033378601074, rank=6, decoded_token=None), 499: Logprob(logprob=-14.375033378601074, rank=7, decoded_token=None), 220: Logprob(logprob=-14.875033378601074, rank=8, decoded_token=None), 374: Logprob(logprob=-15.625033378601074, rank=9, decoded_token=None), 11: Logprob(logprob=-15.750033378601074, rank=10, decoded_token=None), 14932: Logprob(logprob=-16.31253433227539, rank=11, decoded_token=None), 477: Logprob(logprob=-16.43753433227539, rank=12, decoded_token=None), 3629: Logprob(logprob=-16.56253433227539, rank=13, decoded_token=None), 584: Logprob(logprob=-17.12503433227539, rank=14, decoded_token=None), 2663: Logprob(logprob=-17.18753433227539, rank=15, decoded_token=None), 706: Logprob(logprob=-17.37503433227539, rank=16, decoded_token=None), 14183: Logprob(logprob=-18.00003433227539, rank=17, decoded_token=None), 1073: Logprob(logprob=-18.12503433227539, rank=18, decoded_token=None), 279: Logprob(logprob=-18.18753433227539, rank=19, decoded_token=None), 311: Logprob(logprob=-18.31253433227539, rank=20, decoded_token=None)}\n",
+      "315 {315: Logprob(logprob=-0.006171337328851223, rank=1, decoded_token=None), 330: Logprob(logprob=-5.131171226501465, rank=2, decoded_token=None), 364: Logprob(logprob=-8.631171226501465, rank=3, decoded_token=None), 430: Logprob(logprob=-10.131171226501465, rank=4, decoded_token=None), 922: Logprob(logprob=-11.506171226501465, rank=5, decoded_token=None), 11: Logprob(logprob=-12.131171226501465, rank=6, decoded_token=None), 499: Logprob(logprob=-12.631171226501465, rank=7, decoded_token=None), 1054: Logprob(logprob=-13.381171226501465, rank=8, decoded_token=None), 5196: Logprob(logprob=-13.881171226501465, rank=9, decoded_token=None), 374: Logprob(logprob=-14.568671226501465, rank=10, decoded_token=None), 389: Logprob(logprob=-14.943671226501465, rank=11, decoded_token=None), 14932: Logprob(logprob=-15.068671226501465, rank=12, decoded_token=None), 706: Logprob(logprob=-15.256171226501465, rank=13, decoded_token=None), 311: Logprob(logprob=-15.318671226501465, rank=14, decoded_token=None), 37260: Logprob(logprob=-15.568671226501465, rank=15, decoded_token=None), 3451: Logprob(logprob=-15.631171226501465, rank=16, decoded_token=None), 1073: Logprob(logprob=-16.13117218017578, rank=17, decoded_token=None), 9002: Logprob(logprob=-16.44367218017578, rank=18, decoded_token=None), 5084: Logprob(logprob=-16.50617218017578, rank=19, decoded_token=None), 520: Logprob(logprob=-16.56867218017578, rank=20, decoded_token=None)}\n",
+      "315 {315: Logprob(logprob=0.0, rank=1, decoded_token=None), 323: Logprob(logprob=-17.625, rank=2, decoded_token=None), 477: Logprob(logprob=-18.125, rank=3, decoded_token=None), 8838: Logprob(logprob=-18.125, rank=4, decoded_token=None), 5046: Logprob(logprob=-19.375, rank=5, decoded_token=None), 3659: Logprob(logprob=-20.75, rank=6, decoded_token=None), 11: Logprob(logprob=-20.875, rank=7, decoded_token=None), 2324: Logprob(logprob=-22.125, rank=8, decoded_token=None), 220: Logprob(logprob=-22.125, rank=9, decoded_token=None), 311: Logprob(logprob=-22.25, rank=10, decoded_token=None), 3083: Logprob(logprob=-22.25, rank=11, decoded_token=None), 297: Logprob(logprob=-22.25, rank=12, decoded_token=None), 1073: Logprob(logprob=-22.25, rank=13, decoded_token=None), 10440: Logprob(logprob=-22.625, rank=14, decoded_token=None), 33028: Logprob(logprob=-23.25, rank=15, decoded_token=None), 304: Logprob(logprob=-23.6875, rank=16, decoded_token=None), 60835: Logprob(logprob=-23.9375, rank=17, decoded_token=None), 44323: Logprob(logprob=-24.25, rank=18, decoded_token=None), 1161: Logprob(logprob=-24.3125, rank=19, decoded_token=None), 330: Logprob(logprob=-24.625, rank=20, decoded_token=None)}\n",
+      "279 {279: Logprob(logprob=-0.0002455409849062562, rank=1, decoded_token=None), 22706: Logprob(logprob=-8.875245094299316, rank=2, decoded_token=None), 19675: Logprob(logprob=-9.250245094299316, rank=3, decoded_token=None), 2324: Logprob(logprob=-12.250245094299316, rank=4, decoded_token=None), 3925: Logprob(logprob=-12.750245094299316, rank=5, decoded_token=None), 46969: Logprob(logprob=-14.250245094299316, rank=6, decoded_token=None), 8396: Logprob(logprob=-15.437745094299316, rank=7, decoded_token=None), 2854: Logprob(logprob=-15.562745094299316, rank=8, decoded_token=None), 94135: Logprob(logprob=-15.687745094299316, rank=9, decoded_token=None), 43384: Logprob(logprob=-16.062746047973633, rank=10, decoded_token=None), 11104: Logprob(logprob=-16.312746047973633, rank=11, decoded_token=None), 578: Logprob(logprob=-16.625246047973633, rank=12, decoded_token=None), 3432: Logprob(logprob=-16.687746047973633, rank=13, decoded_token=None), 1820: Logprob(logprob=-16.750246047973633, rank=14, decoded_token=None), 36017: Logprob(logprob=-17.187746047973633, rank=15, decoded_token=None), 3247: Logprob(logprob=-17.375246047973633, rank=16, decoded_token=None), 83695: Logprob(logprob=-17.437746047973633, rank=17, decoded_token=None), 11: Logprob(logprob=-17.500246047973633, rank=18, decoded_token=None), 1455: Logprob(logprob=-17.562746047973633, rank=19, decoded_token=None), 1057: Logprob(logprob=-17.625246047973633, rank=20, decoded_token=None)}\n",
+      "7250 {7250: Logprob(logprob=-0.002914468990638852, rank=1, decoded_token=None), 61628: Logprob(logprob=-5.8779144287109375, rank=2, decoded_token=None), 220: Logprob(logprob=-9.877914428710938, rank=3, decoded_token=None), 473: Logprob(logprob=-10.377914428710938, rank=4, decoded_token=None), 45437: Logprob(logprob=-10.752914428710938, rank=5, decoded_token=None), 80081: Logprob(logprob=-13.127914428710938, rank=6, decoded_token=None), 4761: Logprob(logprob=-13.752914428710938, rank=7, decoded_token=None), 24448: Logprob(logprob=-14.502914428710938, rank=8, decoded_token=None), 330: Logprob(logprob=-14.940414428710938, rank=9, decoded_token=None), 364: Logprob(logprob=-14.940414428710938, rank=10, decoded_token=None), 83342: Logprob(logprob=-15.377914428710938, rank=11, decoded_token=None), 4194: Logprob(logprob=-15.565414428710938, rank=12, decoded_token=None), 44273: Logprob(logprob=-15.690414428710938, rank=13, decoded_token=None), 28997: Logprob(logprob=-15.690414428710938, rank=14, decoded_token=None), 1536: Logprob(logprob=-15.877914428710938, rank=15, decoded_token=None), 256: Logprob(logprob=-16.065414428710938, rank=16, decoded_token=None), 11: Logprob(logprob=-16.190414428710938, rank=17, decoded_token=None), 507: Logprob(logprob=-16.315414428710938, rank=18, decoded_token=None), 80798: Logprob(logprob=-16.690414428710938, rank=19, decoded_token=None), 56875: Logprob(logprob=-16.815414428710938, rank=20, decoded_token=None)}\n",
+      "4900 {4900: Logprob(logprob=-0.0263874139636755, rank=1, decoded_token=None), 4872: Logprob(logprob=-3.6513874530792236, rank=2, decoded_token=None), 86481: Logprob(logprob=-10.401387214660645, rank=3, decoded_token=None), 220: Logprob(logprob=-11.151387214660645, rank=4, decoded_token=None), 67088: Logprob(logprob=-11.776387214660645, rank=5, decoded_token=None), 27229: Logprob(logprob=-11.776387214660645, rank=6, decoded_token=None), 4203: Logprob(logprob=-12.026387214660645, rank=7, decoded_token=None), 13: Logprob(logprob=-12.151387214660645, rank=8, decoded_token=None), 42855: Logprob(logprob=-12.401387214660645, rank=9, decoded_token=None), 79739: Logprob(logprob=-12.526387214660645, rank=10, decoded_token=None), 13621: Logprob(logprob=-12.901387214660645, rank=11, decoded_token=None), 79864: Logprob(logprob=-13.651387214660645, rank=12, decoded_token=None), 7808: Logprob(logprob=-14.151387214660645, rank=13, decoded_token=None), 47608: Logprob(logprob=-14.526387214660645, rank=14, decoded_token=None), 24544: Logprob(logprob=-14.526387214660645, rank=15, decoded_token=None), 52485: Logprob(logprob=-14.526387214660645, rank=16, decoded_token=None), 23072: Logprob(logprob=-15.276387214660645, rank=17, decoded_token=None), 11: Logprob(logprob=-15.401387214660645, rank=18, decoded_token=None), 1685: Logprob(logprob=-15.526387214660645, rank=19, decoded_token=None), 24145: Logprob(logprob=-15.838887214660645, rank=20, decoded_token=None)}\n",
+      "815 {815: Logprob(logprob=-0.0032160962000489235, rank=1, decoded_token=None), 13: Logprob(logprob=-6.00321626663208, rank=2, decoded_token=None), 328: Logprob(logprob=-7.37821626663208, rank=3, decoded_token=None), 6354: Logprob(logprob=-9.878215789794922, rank=4, decoded_token=None), 21861: Logprob(logprob=-10.628215789794922, rank=5, decoded_token=None), 516: Logprob(logprob=-10.878215789794922, rank=6, decoded_token=None), 662: Logprob(logprob=-12.378215789794922, rank=7, decoded_token=None), 220: Logprob(logprob=-12.503215789794922, rank=8, decoded_token=None), 497: Logprob(logprob=-13.065715789794922, rank=9, decoded_token=None), 4273: Logprob(logprob=-13.190715789794922, rank=10, decoded_token=None), 11: Logprob(logprob=-13.940715789794922, rank=11, decoded_token=None), 13575: Logprob(logprob=-13.940715789794922, rank=12, decoded_token=None), 1117: Logprob(logprob=-14.065715789794922, rank=13, decoded_token=None), 26922: Logprob(logprob=-14.253215789794922, rank=14, decoded_token=None), 12: Logprob(logprob=-14.628215789794922, rank=15, decoded_token=None), 5415: Logprob(logprob=-14.940715789794922, rank=16, decoded_token=None), 76469: Logprob(logprob=-15.128215789794922, rank=17, decoded_token=None), 24137: Logprob(logprob=-15.565715789794922, rank=18, decoded_token=None), 3238: Logprob(logprob=-15.628215789794922, rank=19, decoded_token=None), 790: Logprob(logprob=-15.628215789794922, rank=20, decoded_token=None)}\n",
+      "7250 {7250: Logprob(logprob=-0.002914468990638852, rank=1, decoded_token=None), 61628: Logprob(logprob=-5.8779144287109375, rank=2, decoded_token=None), 220: Logprob(logprob=-9.877914428710938, rank=3, decoded_token=None), 473: Logprob(logprob=-10.377914428710938, rank=4, decoded_token=None), 45437: Logprob(logprob=-10.752914428710938, rank=5, decoded_token=None), 80081: Logprob(logprob=-13.127914428710938, rank=6, decoded_token=None), 4761: Logprob(logprob=-13.752914428710938, rank=7, decoded_token=None), 24448: Logprob(logprob=-14.502914428710938, rank=8, decoded_token=None), 330: Logprob(logprob=-14.940414428710938, rank=9, decoded_token=None), 364: Logprob(logprob=-14.940414428710938, rank=10, decoded_token=None), 83342: Logprob(logprob=-15.377914428710938, rank=11, decoded_token=None), 4194: Logprob(logprob=-15.565414428710938, rank=12, decoded_token=None), 44273: Logprob(logprob=-15.690414428710938, rank=13, decoded_token=None), 28997: Logprob(logprob=-15.690414428710938, rank=14, decoded_token=None), 1536: Logprob(logprob=-15.877914428710938, rank=15, decoded_token=None), 256: Logprob(logprob=-16.065414428710938, rank=16, decoded_token=None), 11: Logprob(logprob=-16.190414428710938, rank=17, decoded_token=None), 507: Logprob(logprob=-16.315414428710938, rank=18, decoded_token=None), 80798: Logprob(logprob=-16.690414428710938, rank=19, decoded_token=None), 56875: Logprob(logprob=-16.815414428710938, rank=20, decoded_token=None)}\n",
+      "4900 {4900: Logprob(logprob=-0.0263874139636755, rank=1, decoded_token=None), 4872: Logprob(logprob=-3.6513874530792236, rank=2, decoded_token=None), 86481: Logprob(logprob=-10.401387214660645, rank=3, decoded_token=None), 220: Logprob(logprob=-11.151387214660645, rank=4, decoded_token=None), 67088: Logprob(logprob=-11.776387214660645, rank=5, decoded_token=None), 27229: Logprob(logprob=-11.776387214660645, rank=6, decoded_token=None), 4203: Logprob(logprob=-12.026387214660645, rank=7, decoded_token=None), 13: Logprob(logprob=-12.151387214660645, rank=8, decoded_token=None), 42855: Logprob(logprob=-12.401387214660645, rank=9, decoded_token=None), 79739: Logprob(logprob=-12.526387214660645, rank=10, decoded_token=None), 13621: Logprob(logprob=-12.901387214660645, rank=11, decoded_token=None), 79864: Logprob(logprob=-13.651387214660645, rank=12, decoded_token=None), 7808: Logprob(logprob=-14.151387214660645, rank=13, decoded_token=None), 47608: Logprob(logprob=-14.526387214660645, rank=14, decoded_token=None), 24544: Logprob(logprob=-14.526387214660645, rank=15, decoded_token=None), 52485: Logprob(logprob=-14.526387214660645, rank=16, decoded_token=None), 23072: Logprob(logprob=-15.276387214660645, rank=17, decoded_token=None), 11: Logprob(logprob=-15.401387214660645, rank=18, decoded_token=None), 1685: Logprob(logprob=-15.526387214660645, rank=19, decoded_token=None), 24145: Logprob(logprob=-15.838887214660645, rank=20, decoded_token=None)}\n",
+      "815 {815: Logprob(logprob=-0.0032160962000489235, rank=1, decoded_token=None), 13: Logprob(logprob=-6.00321626663208, rank=2, decoded_token=None), 328: Logprob(logprob=-7.37821626663208, rank=3, decoded_token=None), 6354: Logprob(logprob=-9.878215789794922, rank=4, decoded_token=None), 21861: Logprob(logprob=-10.628215789794922, rank=5, decoded_token=None), 516: Logprob(logprob=-10.878215789794922, rank=6, decoded_token=None), 662: Logprob(logprob=-12.378215789794922, rank=7, decoded_token=None), 220: Logprob(logprob=-12.503215789794922, rank=8, decoded_token=None), 497: Logprob(logprob=-13.065715789794922, rank=9, decoded_token=None), 4273: Logprob(logprob=-13.190715789794922, rank=10, decoded_token=None), 11: Logprob(logprob=-13.940715789794922, rank=11, decoded_token=None), 13575: Logprob(logprob=-13.940715789794922, rank=12, decoded_token=None), 1117: Logprob(logprob=-14.065715789794922, rank=13, decoded_token=None), 26922: Logprob(logprob=-14.253215789794922, rank=14, decoded_token=None), 12: Logprob(logprob=-14.628215789794922, rank=15, decoded_token=None), 5415: Logprob(logprob=-14.940715789794922, rank=16, decoded_token=None), 76469: Logprob(logprob=-15.128215789794922, rank=17, decoded_token=None), 24137: Logprob(logprob=-15.565715789794922, rank=18, decoded_token=None), 3238: Logprob(logprob=-15.628215789794922, rank=19, decoded_token=None), 790: Logprob(logprob=-15.628215789794922, rank=20, decoded_token=None)}\n",
+      "279 {279: Logprob(logprob=-0.009556027129292488, rank=1, decoded_token=None), 1148: Logprob(logprob=-4.884555816650391, rank=2, decoded_token=None), 330: Logprob(logprob=-6.259555816650391, rank=3, decoded_token=None), 364: Logprob(logprob=-10.75955581665039, rank=4, decoded_token=None), 3508: Logprob(logprob=-12.13455581665039, rank=5, decoded_token=None), 2324: Logprob(logprob=-12.13455581665039, rank=6, decoded_token=None), 7438: Logprob(logprob=-13.38455581665039, rank=7, decoded_token=None), 1054: Logprob(logprob=-14.38455581665039, rank=8, decoded_token=None), 14209: Logprob(logprob=-14.88455581665039, rank=9, decoded_token=None), 17139: Logprob(logprob=-15.00955581665039, rank=10, decoded_token=None), 9455: Logprob(logprob=-15.13455581665039, rank=11, decoded_token=None), 3249: Logprob(logprob=-15.50955581665039, rank=12, decoded_token=None), 3639: Logprob(logprob=-15.75955581665039, rank=13, decoded_token=None), 220: Logprob(logprob=-15.82205581665039, rank=14, decoded_token=None), 1268: Logprob(logprob=-15.94705581665039, rank=15, decoded_token=None), 1820: Logprob(logprob=-16.69705581665039, rank=16, decoded_token=None), 31437: Logprob(logprob=-17.19705581665039, rank=17, decoded_token=None), 578: Logprob(logprob=-17.50955581665039, rank=18, decoded_token=None), 3451: Logprob(logprob=-17.50955581665039, rank=19, decoded_token=None), 12840: Logprob(logprob=-17.50955581665039, rank=20, decoded_token=None)}\n",
+      "2324 {2324: Logprob(logprob=0.0, rank=1, decoded_token=None), 9601: Logprob(logprob=-17.25, rank=2, decoded_token=None), 14789: Logprob(logprob=-19.25, rank=3, decoded_token=None), 14209: Logprob(logprob=-19.75, rank=4, decoded_token=None), 330: Logprob(logprob=-21.125, rank=5, decoded_token=None), 26928: Logprob(logprob=-21.25, rank=6, decoded_token=None), 83900: Logprob(logprob=-21.75, rank=7, decoded_token=None), 364: Logprob(logprob=-21.75, rank=8, decoded_token=None), 220: Logprob(logprob=-22.25, rank=9, decoded_token=None), 3823: Logprob(logprob=-22.625, rank=10, decoded_token=None), 61094: Logprob(logprob=-22.75, rank=11, decoded_token=None), 10345: Logprob(logprob=-22.875, rank=12, decoded_token=None), 73591: Logprob(logprob=-23.625, rank=13, decoded_token=None), 3974: Logprob(logprob=-23.875, rank=14, decoded_token=None), 279: Logprob(logprob=-24.25, rank=15, decoded_token=None), 49785: Logprob(logprob=-24.625, rank=16, decoded_token=None), 26833: Logprob(logprob=-25.625, rank=17, decoded_token=None), 25994: Logprob(logprob=-25.75, rank=18, decoded_token=None), 107749: Logprob(logprob=-26.375, rank=19, decoded_token=None), 61961: Logprob(logprob=-26.625, rank=20, decoded_token=None)}\n",
+      "279 {279: Logprob(logprob=-0.01450834795832634, rank=1, decoded_token=None), 330: Logprob(logprob=-4.389508247375488, rank=2, decoded_token=None), 7438: Logprob(logprob=-7.514508247375488, rank=3, decoded_token=None), 2324: Logprob(logprob=-7.889508247375488, rank=4, decoded_token=None), 364: Logprob(logprob=-7.889508247375488, rank=5, decoded_token=None), 264: Logprob(logprob=-8.014508247375488, rank=6, decoded_token=None), 9455: Logprob(logprob=-8.014508247375488, rank=7, decoded_token=None), 1148: Logprob(logprob=-10.764508247375488, rank=8, decoded_token=None), 1054: Logprob(logprob=-12.389508247375488, rank=9, decoded_token=None), 459: Logprob(logprob=-12.889508247375488, rank=10, decoded_token=None), 26679: Logprob(logprob=-13.139508247375488, rank=11, decoded_token=None), 220: Logprob(logprob=-13.202008247375488, rank=12, decoded_token=None), 7580: Logprob(logprob=-13.264508247375488, rank=13, decoded_token=None), 14209: Logprob(logprob=-13.702008247375488, rank=14, decoded_token=None), 11125: Logprob(logprob=-14.139508247375488, rank=15, decoded_token=None), 27409: Logprob(logprob=-14.452008247375488, rank=16, decoded_token=None), 3451: Logprob(logprob=-14.639508247375488, rank=17, decoded_token=None), 323: Logprob(logprob=-14.639508247375488, rank=18, decoded_token=None), 15389: Logprob(logprob=-14.639508247375488, rank=19, decoded_token=None), 42687: Logprob(logprob=-14.764508247375488, rank=20, decoded_token=None)}\n",
+      "279 {279: Logprob(logprob=-0.009556027129292488, rank=1, decoded_token=None), 1148: Logprob(logprob=-4.884555816650391, rank=2, decoded_token=None), 330: Logprob(logprob=-6.259555816650391, rank=3, decoded_token=None), 364: Logprob(logprob=-10.75955581665039, rank=4, decoded_token=None), 3508: Logprob(logprob=-12.13455581665039, rank=5, decoded_token=None), 2324: Logprob(logprob=-12.13455581665039, rank=6, decoded_token=None), 7438: Logprob(logprob=-13.38455581665039, rank=7, decoded_token=None), 1054: Logprob(logprob=-14.38455581665039, rank=8, decoded_token=None), 14209: Logprob(logprob=-14.88455581665039, rank=9, decoded_token=None), 17139: Logprob(logprob=-15.00955581665039, rank=10, decoded_token=None), 9455: Logprob(logprob=-15.13455581665039, rank=11, decoded_token=None), 3249: Logprob(logprob=-15.50955581665039, rank=12, decoded_token=None), 3639: Logprob(logprob=-15.75955581665039, rank=13, decoded_token=None), 220: Logprob(logprob=-15.82205581665039, rank=14, decoded_token=None), 1268: Logprob(logprob=-15.94705581665039, rank=15, decoded_token=None), 1820: Logprob(logprob=-16.69705581665039, rank=16, decoded_token=None), 31437: Logprob(logprob=-17.19705581665039, rank=17, decoded_token=None), 578: Logprob(logprob=-17.50955581665039, rank=18, decoded_token=None), 3451: Logprob(logprob=-17.50955581665039, rank=19, decoded_token=None), 12840: Logprob(logprob=-17.50955581665039, rank=20, decoded_token=None)}\n",
+      "2324 {2324: Logprob(logprob=0.0, rank=1, decoded_token=None), 9601: Logprob(logprob=-17.25, rank=2, decoded_token=None), 14789: Logprob(logprob=-19.25, rank=3, decoded_token=None), 14209: Logprob(logprob=-19.75, rank=4, decoded_token=None), 330: Logprob(logprob=-21.125, rank=5, decoded_token=None), 26928: Logprob(logprob=-21.25, rank=6, decoded_token=None), 83900: Logprob(logprob=-21.75, rank=7, decoded_token=None), 364: Logprob(logprob=-21.75, rank=8, decoded_token=None), 220: Logprob(logprob=-22.25, rank=9, decoded_token=None), 3823: Logprob(logprob=-22.625, rank=10, decoded_token=None), 61094: Logprob(logprob=-22.75, rank=11, decoded_token=None), 10345: Logprob(logprob=-22.875, rank=12, decoded_token=None), 73591: Logprob(logprob=-23.625, rank=13, decoded_token=None), 3974: Logprob(logprob=-23.875, rank=14, decoded_token=None), 279: Logprob(logprob=-24.25, rank=15, decoded_token=None), 49785: Logprob(logprob=-24.625, rank=16, decoded_token=None), 26833: Logprob(logprob=-25.625, rank=17, decoded_token=None), 25994: Logprob(logprob=-25.75, rank=18, decoded_token=None), 107749: Logprob(logprob=-26.375, rank=19, decoded_token=None), 61961: Logprob(logprob=-26.625, rank=20, decoded_token=None)}\n",
+      "1455 {1455: Logprob(logprob=-0.009604200720787048, rank=1, decoded_token=None), 24417: Logprob(logprob=-5.009603977203369, rank=2, decoded_token=None), 8706: Logprob(logprob=-6.509603977203369, rank=3, decoded_token=None), 12474: Logprob(logprob=-6.759603977203369, rank=4, decoded_token=None), 1917: Logprob(logprob=-9.509604454040527, rank=5, decoded_token=None), 7928: Logprob(logprob=-10.509604454040527, rank=6, decoded_token=None), 51621: Logprob(logprob=-10.697104454040527, rank=7, decoded_token=None), 17139: Logprob(logprob=-10.884604454040527, rank=8, decoded_token=None), 22807: Logprob(logprob=-10.947104454040527, rank=9, decoded_token=None), 35825: Logprob(logprob=-11.259604454040527, rank=10, decoded_token=None), 3646: Logprob(logprob=-11.697104454040527, rank=11, decoded_token=None), 60406: Logprob(logprob=-11.759604454040527, rank=12, decoded_token=None), 15861: Logprob(logprob=-11.759604454040527, rank=13, decoded_token=None), 4325: Logprob(logprob=-12.072104454040527, rank=14, decoded_token=None), 2324: Logprob(logprob=-12.134604454040527, rank=15, decoded_token=None), 37554: Logprob(logprob=-12.572104454040527, rank=16, decoded_token=None), 7648: Logprob(logprob=-12.822104454040527, rank=17, decoded_token=None), 30758: Logprob(logprob=-12.884604454040527, rank=18, decoded_token=None), 67739: Logprob(logprob=-13.384604454040527, rank=19, decoded_token=None), 28254: Logprob(logprob=-13.822104454040527, rank=20, decoded_token=None)}\n",
+      "11 {11: Logprob(logprob=-0.2046964168548584, rank=1, decoded_token=None), 574: Logprob(logprob=-1.7046964168548584, rank=2, decoded_token=None), 279: Logprob(logprob=-6.3296966552734375, rank=3, decoded_token=None), 596: Logprob(logprob=-6.9546966552734375, rank=4, decoded_token=None), 16014: Logprob(logprob=-7.9546966552734375, rank=5, decoded_token=None), 320: Logprob(logprob=-9.704696655273438, rank=6, decoded_token=None), 220: Logprob(logprob=-9.954696655273438, rank=7, decoded_token=None), 374: Logprob(logprob=-10.329696655273438, rank=8, decoded_token=None), 8105: Logprob(logprob=-10.704696655273438, rank=9, decoded_token=None), 34688: Logprob(logprob=-11.954696655273438, rank=10, decoded_token=None), 1174: Logprob(logprob=-12.204696655273438, rank=11, decoded_token=None), 256: Logprob(logprob=-12.954696655273438, rank=12, decoded_token=None), 63738: Logprob(logprob=-13.079696655273438, rank=13, decoded_token=None), 9405: Logprob(logprob=-13.329696655273438, rank=14, decoded_token=None), 31870: Logprob(logprob=-13.454696655273438, rank=15, decoded_token=None), 13: Logprob(logprob=-13.704696655273438, rank=16, decoded_token=None), 482: Logprob(logprob=-14.142196655273438, rank=17, decoded_token=None), 14767: Logprob(logprob=-14.204696655273438, rank=18, decoded_token=None), 26: Logprob(logprob=-14.454696655273438, rank=19, decoded_token=None), 262: Logprob(logprob=-14.579696655273438, rank=20, decoded_token=None)}\n",
+      "24448 {24448: Logprob(logprob=-0.009981834329664707, rank=1, decoded_token=None), 7250: Logprob(logprob=-5.259981632232666, rank=2, decoded_token=None), 11: Logprob(logprob=-6.259981632232666, rank=3, decoded_token=None), 220: Logprob(logprob=-6.509981632232666, rank=4, decoded_token=None), 4821: Logprob(logprob=-7.134981632232666, rank=5, decoded_token=None), 32817: Logprob(logprob=-7.634981632232666, rank=6, decoded_token=None), 3703: Logprob(logprob=-11.134982109069824, rank=7, decoded_token=None), 31858: Logprob(logprob=-11.634982109069824, rank=8, decoded_token=None), 364: Logprob(logprob=-11.759982109069824, rank=9, decoded_token=None), 94420: Logprob(logprob=-12.009982109069824, rank=10, decoded_token=None), 45287: Logprob(logprob=-12.634982109069824, rank=11, decoded_token=None), 482: Logprob(logprob=-13.009982109069824, rank=12, decoded_token=None), 92169: Logprob(logprob=-13.259982109069824, rank=13, decoded_token=None), 256: Logprob(logprob=-13.259982109069824, rank=14, decoded_token=None), 3511: Logprob(logprob=-13.384982109069824, rank=15, decoded_token=None), 320: Logprob(logprob=-13.509982109069824, rank=16, decoded_token=None), 323: Logprob(logprob=-13.759982109069824, rank=17, decoded_token=None), 9641: Logprob(logprob=-14.009982109069824, rank=18, decoded_token=None), 16150: Logprob(logprob=-14.009982109069824, rank=19, decoded_token=None), 330: Logprob(logprob=-14.259982109069824, rank=20, decoded_token=None)}\n",
+      "13 {13: Logprob(logprob=-0.06436779350042343, rank=1, decoded_token=None), 4900: Logprob(logprob=-2.8143677711486816, rank=2, decoded_token=None), 4872: Logprob(logprob=-6.064367771148682, rank=3, decoded_token=None), 27229: Logprob(logprob=-11.31436824798584, rank=4, decoded_token=None), 79739: Logprob(logprob=-11.43936824798584, rank=5, decoded_token=None), 1087: Logprob(logprob=-11.43936824798584, rank=6, decoded_token=None), 86481: Logprob(logprob=-11.56436824798584, rank=7, decoded_token=None), 885: Logprob(logprob=-12.06436824798584, rank=8, decoded_token=None), 11: Logprob(logprob=-12.18936824798584, rank=9, decoded_token=None), 42855: Logprob(logprob=-12.68936824798584, rank=10, decoded_token=None), 220: Logprob(logprob=-12.93936824798584, rank=11, decoded_token=None), 47608: Logprob(logprob=-13.31436824798584, rank=12, decoded_token=None), 67088: Logprob(logprob=-13.31436824798584, rank=13, decoded_token=None), 2637: Logprob(logprob=-13.43936824798584, rank=14, decoded_token=None), 79864: Logprob(logprob=-13.56436824798584, rank=15, decoded_token=None), 4203: Logprob(logprob=-13.68936824798584, rank=16, decoded_token=None), 52485: Logprob(logprob=-13.75186824798584, rank=17, decoded_token=None), 13621: Logprob(logprob=-14.12686824798584, rank=18, decoded_token=None), 497: Logprob(logprob=-14.25186824798584, rank=19, decoded_token=None), 12898: Logprob(logprob=-15.06436824798584, rank=20, decoded_token=None)}\n",
+      "11 {11: Logprob(logprob=-0.2046964168548584, rank=1, decoded_token=None), 574: Logprob(logprob=-1.7046964168548584, rank=2, decoded_token=None), 279: Logprob(logprob=-6.3296966552734375, rank=3, decoded_token=None), 596: Logprob(logprob=-6.9546966552734375, rank=4, decoded_token=None), 16014: Logprob(logprob=-7.9546966552734375, rank=5, decoded_token=None), 320: Logprob(logprob=-9.704696655273438, rank=6, decoded_token=None), 220: Logprob(logprob=-9.954696655273438, rank=7, decoded_token=None), 374: Logprob(logprob=-10.329696655273438, rank=8, decoded_token=None), 8105: Logprob(logprob=-10.704696655273438, rank=9, decoded_token=None), 34688: Logprob(logprob=-11.954696655273438, rank=10, decoded_token=None), 1174: Logprob(logprob=-12.204696655273438, rank=11, decoded_token=None), 256: Logprob(logprob=-12.954696655273438, rank=12, decoded_token=None), 63738: Logprob(logprob=-13.079696655273438, rank=13, decoded_token=None), 9405: Logprob(logprob=-13.329696655273438, rank=14, decoded_token=None), 31870: Logprob(logprob=-13.454696655273438, rank=15, decoded_token=None), 13: Logprob(logprob=-13.704696655273438, rank=16, decoded_token=None), 482: Logprob(logprob=-14.142196655273438, rank=17, decoded_token=None), 14767: Logprob(logprob=-14.204696655273438, rank=18, decoded_token=None), 26: Logprob(logprob=-14.454696655273438, rank=19, decoded_token=None), 262: Logprob(logprob=-14.579696655273438, rank=20, decoded_token=None)}\n",
+      "24448 {24448: Logprob(logprob=-0.009981834329664707, rank=1, decoded_token=None), 7250: Logprob(logprob=-5.259981632232666, rank=2, decoded_token=None), 11: Logprob(logprob=-6.259981632232666, rank=3, decoded_token=None), 220: Logprob(logprob=-6.509981632232666, rank=4, decoded_token=None), 4821: Logprob(logprob=-7.134981632232666, rank=5, decoded_token=None), 32817: Logprob(logprob=-7.634981632232666, rank=6, decoded_token=None), 3703: Logprob(logprob=-11.134982109069824, rank=7, decoded_token=None), 31858: Logprob(logprob=-11.634982109069824, rank=8, decoded_token=None), 364: Logprob(logprob=-11.759982109069824, rank=9, decoded_token=None), 94420: Logprob(logprob=-12.009982109069824, rank=10, decoded_token=None), 45287: Logprob(logprob=-12.634982109069824, rank=11, decoded_token=None), 482: Logprob(logprob=-13.009982109069824, rank=12, decoded_token=None), 92169: Logprob(logprob=-13.259982109069824, rank=13, decoded_token=None), 256: Logprob(logprob=-13.259982109069824, rank=14, decoded_token=None), 3511: Logprob(logprob=-13.384982109069824, rank=15, decoded_token=None), 320: Logprob(logprob=-13.509982109069824, rank=16, decoded_token=None), 323: Logprob(logprob=-13.759982109069824, rank=17, decoded_token=None), 9641: Logprob(logprob=-14.009982109069824, rank=18, decoded_token=None), 16150: Logprob(logprob=-14.009982109069824, rank=19, decoded_token=None), 330: Logprob(logprob=-14.259982109069824, rank=20, decoded_token=None)}\n",
+      "13 {13: Logprob(logprob=-0.06436779350042343, rank=1, decoded_token=None), 4900: Logprob(logprob=-2.8143677711486816, rank=2, decoded_token=None), 4872: Logprob(logprob=-6.064367771148682, rank=3, decoded_token=None), 27229: Logprob(logprob=-11.31436824798584, rank=4, decoded_token=None), 79739: Logprob(logprob=-11.43936824798584, rank=5, decoded_token=None), 1087: Logprob(logprob=-11.43936824798584, rank=6, decoded_token=None), 86481: Logprob(logprob=-11.56436824798584, rank=7, decoded_token=None), 885: Logprob(logprob=-12.06436824798584, rank=8, decoded_token=None), 11: Logprob(logprob=-12.18936824798584, rank=9, decoded_token=None), 42855: Logprob(logprob=-12.68936824798584, rank=10, decoded_token=None), 220: Logprob(logprob=-12.93936824798584, rank=11, decoded_token=None), 47608: Logprob(logprob=-13.31436824798584, rank=12, decoded_token=None), 67088: Logprob(logprob=-13.31436824798584, rank=13, decoded_token=None), 2637: Logprob(logprob=-13.43936824798584, rank=14, decoded_token=None), 79864: Logprob(logprob=-13.56436824798584, rank=15, decoded_token=None), 4203: Logprob(logprob=-13.68936824798584, rank=16, decoded_token=None), 52485: Logprob(logprob=-13.75186824798584, rank=17, decoded_token=None), 13621: Logprob(logprob=-14.12686824798584, rank=18, decoded_token=None), 497: Logprob(logprob=-14.25186824798584, rank=19, decoded_token=None), 12898: Logprob(logprob=-15.06436824798584, rank=20, decoded_token=None)}\n",
+      "7438 {7438: Logprob(logprob=-0.00019298121333122253, rank=1, decoded_token=None), 330: Logprob(logprob=-8.750192642211914, rank=2, decoded_token=None), 17139: Logprob(logprob=-11.125192642211914, rank=3, decoded_token=None), 7138: Logprob(logprob=-11.250192642211914, rank=4, decoded_token=None), 57865: Logprob(logprob=-13.375192642211914, rank=5, decoded_token=None), 364: Logprob(logprob=-13.375192642211914, rank=6, decoded_token=None), 14209: Logprob(logprob=-13.500192642211914, rank=7, decoded_token=None), 3152: Logprob(logprob=-14.250192642211914, rank=8, decoded_token=None), 49203: Logprob(logprob=-14.375192642211914, rank=9, decoded_token=None), 50800: Logprob(logprob=-15.250192642211914, rank=10, decoded_token=None), 1054: Logprob(logprob=-15.375192642211914, rank=11, decoded_token=None), 6371: Logprob(logprob=-15.750192642211914, rank=12, decoded_token=None), 26431: Logprob(logprob=-16.000192642211914, rank=13, decoded_token=None), 4325: Logprob(logprob=-16.125192642211914, rank=14, decoded_token=None), 23222: Logprob(logprob=-16.125192642211914, rank=15, decoded_token=None), 17051: Logprob(logprob=-16.500192642211914, rank=16, decoded_token=None), 7580: Logprob(logprob=-16.625192642211914, rank=17, decoded_token=None), 84096: Logprob(logprob=-17.312692642211914, rank=18, decoded_token=None), 7419: Logprob(logprob=-17.750192642211914, rank=19, decoded_token=None), 38088: Logprob(logprob=-17.937692642211914, rank=20, decoded_token=None)}\n",
+      "374 {374: Logprob(logprob=-0.006966823246330023, rank=1, decoded_token=None), 706: Logprob(logprob=-5.006966590881348, rank=2, decoded_token=None), 11: Logprob(logprob=-9.006966590881348, rank=3, decoded_token=None), 649: Logprob(logprob=-9.506966590881348, rank=4, decoded_token=None), 0: Logprob(logprob=-10.506966590881348, rank=5, decoded_token=None), 13: Logprob(logprob=-12.006966590881348, rank=6, decoded_token=None), 220: Logprob(logprob=-12.131966590881348, rank=7, decoded_token=None), 8625: Logprob(logprob=-12.506966590881348, rank=8, decoded_token=None), 482: Logprob(logprob=-12.756966590881348, rank=9, decoded_token=None), 6: Logprob(logprob=-12.881966590881348, rank=10, decoded_token=None), 19813: Logprob(logprob=-13.881966590881348, rank=11, decoded_token=None), 320: Logprob(logprob=-14.006966590881348, rank=12, decoded_token=None), 574: Logprob(logprob=-14.381966590881348, rank=13, decoded_token=None), 35327: Logprob(logprob=-14.506966590881348, rank=14, decoded_token=None), 1389: Logprob(logprob=-14.569466590881348, rank=15, decoded_token=None), 1253: Logprob(logprob=-14.631966590881348, rank=16, decoded_token=None), 2643: Logprob(logprob=-15.006966590881348, rank=17, decoded_token=None), 3488: Logprob(logprob=-15.069466590881348, rank=18, decoded_token=None), 1: Logprob(logprob=-15.256966590881348, rank=19, decoded_token=None), 285: Logprob(logprob=-15.569466590881348, rank=20, decoded_token=None)}\n",
+      "330 {330: Logprob(logprob=-1.0593980550765991, rank=2, decoded_token=None), 7438: Logprob(logprob=-0.43439802527427673, rank=1, decoded_token=None), 364: Logprob(logprob=-5.184398174285889, rank=3, decoded_token=None), 1054: Logprob(logprob=-9.93439769744873, rank=4, decoded_token=None), 17139: Logprob(logprob=-12.05939769744873, rank=5, decoded_token=None), 3436: Logprob(logprob=-12.05939769744873, rank=6, decoded_token=None), 7580: Logprob(logprob=-12.30939769744873, rank=7, decoded_token=None), 3451: Logprob(logprob=-12.30939769744873, rank=8, decoded_token=None), 49203: Logprob(logprob=-13.05939769744873, rank=9, decoded_token=None), 57865: Logprob(logprob=-13.68439769744873, rank=10, decoded_token=None), 3152: Logprob(logprob=-13.68439769744873, rank=11, decoded_token=None), 23222: Logprob(logprob=-13.80939769744873, rank=12, decoded_token=None), 1595: Logprob(logprob=-13.80939769744873, rank=13, decoded_token=None), 2778: Logprob(logprob=-14.30939769744873, rank=14, decoded_token=None), 10103: Logprob(logprob=-14.55939769744873, rank=15, decoded_token=None), 7326: Logprob(logprob=-15.55939769744873, rank=16, decoded_token=None), 50800: Logprob(logprob=-15.68439769744873, rank=17, decoded_token=None), 7138: Logprob(logprob=-15.80939769744873, rank=18, decoded_token=None), 220: Logprob(logprob=-16.059398651123047, rank=19, decoded_token=None), 2846: Logprob(logprob=-16.434398651123047, rank=20, decoded_token=None)}\n",
+      "7438 {7438: Logprob(logprob=-0.00019298121333122253, rank=1, decoded_token=None), 330: Logprob(logprob=-8.750192642211914, rank=2, decoded_token=None), 17139: Logprob(logprob=-11.125192642211914, rank=3, decoded_token=None), 7138: Logprob(logprob=-11.250192642211914, rank=4, decoded_token=None), 57865: Logprob(logprob=-13.375192642211914, rank=5, decoded_token=None), 364: Logprob(logprob=-13.375192642211914, rank=6, decoded_token=None), 14209: Logprob(logprob=-13.500192642211914, rank=7, decoded_token=None), 3152: Logprob(logprob=-14.250192642211914, rank=8, decoded_token=None), 49203: Logprob(logprob=-14.375192642211914, rank=9, decoded_token=None), 50800: Logprob(logprob=-15.250192642211914, rank=10, decoded_token=None), 1054: Logprob(logprob=-15.375192642211914, rank=11, decoded_token=None), 6371: Logprob(logprob=-15.750192642211914, rank=12, decoded_token=None), 26431: Logprob(logprob=-16.000192642211914, rank=13, decoded_token=None), 4325: Logprob(logprob=-16.125192642211914, rank=14, decoded_token=None), 23222: Logprob(logprob=-16.125192642211914, rank=15, decoded_token=None), 17051: Logprob(logprob=-16.500192642211914, rank=16, decoded_token=None), 7580: Logprob(logprob=-16.625192642211914, rank=17, decoded_token=None), 84096: Logprob(logprob=-17.312692642211914, rank=18, decoded_token=None), 7419: Logprob(logprob=-17.750192642211914, rank=19, decoded_token=None), 38088: Logprob(logprob=-17.937692642211914, rank=20, decoded_token=None)}\n",
+      "374 {374: Logprob(logprob=-0.006966823246330023, rank=1, decoded_token=None), 706: Logprob(logprob=-5.006966590881348, rank=2, decoded_token=None), 11: Logprob(logprob=-9.006966590881348, rank=3, decoded_token=None), 649: Logprob(logprob=-9.506966590881348, rank=4, decoded_token=None), 0: Logprob(logprob=-10.506966590881348, rank=5, decoded_token=None), 13: Logprob(logprob=-12.006966590881348, rank=6, decoded_token=None), 220: Logprob(logprob=-12.131966590881348, rank=7, decoded_token=None), 8625: Logprob(logprob=-12.506966590881348, rank=8, decoded_token=None), 482: Logprob(logprob=-12.756966590881348, rank=9, decoded_token=None), 6: Logprob(logprob=-12.881966590881348, rank=10, decoded_token=None), 19813: Logprob(logprob=-13.881966590881348, rank=11, decoded_token=None), 320: Logprob(logprob=-14.006966590881348, rank=12, decoded_token=None), 574: Logprob(logprob=-14.381966590881348, rank=13, decoded_token=None), 35327: Logprob(logprob=-14.506966590881348, rank=14, decoded_token=None), 1389: Logprob(logprob=-14.569466590881348, rank=15, decoded_token=None), 1253: Logprob(logprob=-14.631966590881348, rank=16, decoded_token=None), 2643: Logprob(logprob=-15.006966590881348, rank=17, decoded_token=None), 3488: Logprob(logprob=-15.069466590881348, rank=18, decoded_token=None), 1: Logprob(logprob=-15.256966590881348, rank=19, decoded_token=None), 285: Logprob(logprob=-15.569466590881348, rank=20, decoded_token=None)}\n",
+      "28254 {28254: Logprob(logprob=-0.03447406738996506, rank=1, decoded_token=None), 52876: Logprob(logprob=-3.9094741344451904, rank=2, decoded_token=None), 6485: Logprob(logprob=-5.284473896026611, rank=3, decoded_token=None), 16188: Logprob(logprob=-5.284473896026611, rank=4, decoded_token=None), 66278: Logprob(logprob=-6.534473896026611, rank=5, decoded_token=None), 41765: Logprob(logprob=-7.409473896026611, rank=6, decoded_token=None), 59674: Logprob(logprob=-7.909473896026611, rank=7, decoded_token=None), 14154: Logprob(logprob=-8.34697437286377, rank=8, decoded_token=None), 67739: Logprob(logprob=-8.84697437286377, rank=9, decoded_token=None), 58105: Logprob(logprob=-8.84697437286377, rank=10, decoded_token=None), 41903: Logprob(logprob=-8.97197437286377, rank=11, decoded_token=None), 4691: Logprob(logprob=-9.15947437286377, rank=12, decoded_token=None), 5199: Logprob(logprob=-9.78447437286377, rank=13, decoded_token=None), 4325: Logprob(logprob=-9.97197437286377, rank=14, decoded_token=None), 48883: Logprob(logprob=-10.22197437286377, rank=15, decoded_token=None), 57697: Logprob(logprob=-10.34697437286377, rank=16, decoded_token=None), 74252: Logprob(logprob=-10.47197437286377, rank=17, decoded_token=None), 22872: Logprob(logprob=-10.53447437286377, rank=18, decoded_token=None), 66684: Logprob(logprob=-10.65947437286377, rank=19, decoded_token=None), 57216: Logprob(logprob=-10.72197437286377, rank=20, decoded_token=None)}\n",
+      "279 {279: Logprob(logprob=-0.0008725888328626752, rank=1, decoded_token=None), 220: Logprob(logprob=-7.125872611999512, rank=2, decoded_token=None), 4846: Logprob(logprob=-10.125872611999512, rank=3, decoded_token=None), 889: Logprob(logprob=-11.875872611999512, rank=4, decoded_token=None), 1101: Logprob(logprob=-12.375872611999512, rank=5, decoded_token=None), 578: Logprob(logprob=-12.625872611999512, rank=6, decoded_token=None), 264: Logprob(logprob=-13.000872611999512, rank=7, decoded_token=None), 574: Logprob(logprob=-13.125872611999512, rank=8, decoded_token=None), 256: Logprob(logprob=-13.125872611999512, rank=9, decoded_token=None), 5270: Logprob(logprob=-13.250872611999512, rank=10, decoded_token=None), 320: Logprob(logprob=-13.750872611999512, rank=11, decoded_token=None), 459: Logprob(logprob=-14.125872611999512, rank=12, decoded_token=None), 364: Logprob(logprob=-14.125872611999512, rank=13, decoded_token=None), 1694: Logprob(logprob=-14.750872611999512, rank=14, decoded_token=None), 9405: Logprob(logprob=-15.250872611999512, rank=15, decoded_token=None), 720: Logprob(logprob=-15.313372611999512, rank=16, decoded_token=None), 34833: Logprob(logprob=-15.438372611999512, rank=17, decoded_token=None), 3778: Logprob(logprob=-15.438372611999512, rank=18, decoded_token=None), 813: Logprob(logprob=-15.563372611999512, rank=19, decoded_token=None), 33600: Logprob(logprob=-15.625872611999512, rank=20, decoded_token=None)}\n",
+      "9405 {9405: Logprob(logprob=-6.890059739816934e-05, rank=1, decoded_token=None), 220: Logprob(logprob=-10.125068664550781, rank=2, decoded_token=None), 38916: Logprob(logprob=-11.375068664550781, rank=3, decoded_token=None), 13118: Logprob(logprob=-12.500068664550781, rank=4, decoded_token=None), 279: Logprob(logprob=-12.625068664550781, rank=5, decoded_token=None), 7342: Logprob(logprob=-13.125068664550781, rank=6, decoded_token=None), 18307: Logprob(logprob=-13.750068664550781, rank=7, decoded_token=None), 256: Logprob(logprob=-13.875068664550781, rank=8, decoded_token=None), 11: Logprob(logprob=-14.000068664550781, rank=9, decoded_token=None), 16381: Logprob(logprob=-14.125068664550781, rank=10, decoded_token=None), 264: Logprob(logprob=-14.250068664550781, rank=11, decoded_token=None), 459: Logprob(logprob=-14.375068664550781, rank=12, decoded_token=None), 3604: Logprob(logprob=-14.625068664550781, rank=13, decoded_token=None), 40260: Logprob(logprob=-14.625068664550781, rank=14, decoded_token=None), 19073: Logprob(logprob=-14.750068664550781, rank=15, decoded_token=None), 65162: Logprob(logprob=-15.187568664550781, rank=16, decoded_token=None), 18665: Logprob(logprob=-15.750068664550781, rank=17, decoded_token=None), 16689: Logprob(logprob=-16.00006866455078, rank=18, decoded_token=None), 539: Logprob(logprob=-16.18756866455078, rank=19, decoded_token=None), 4872: Logprob(logprob=-16.25006866455078, rank=20, decoded_token=None)}\n",
+      "7250 {7250: Logprob(logprob=-0.000730128725990653, rank=1, decoded_token=None), 61628: Logprob(logprob=-7.625730037689209, rank=2, decoded_token=None), 473: Logprob(logprob=-8.625730514526367, rank=3, decoded_token=None), 220: Logprob(logprob=-10.000730514526367, rank=4, decoded_token=None), 45437: Logprob(logprob=-11.500730514526367, rank=5, decoded_token=None), 4761: Logprob(logprob=-12.625730514526367, rank=6, decoded_token=None), 80081: Logprob(logprob=-13.625730514526367, rank=7, decoded_token=None), 44273: Logprob(logprob=-14.938230514526367, rank=8, decoded_token=None), 83342: Logprob(logprob=-15.063230514526367, rank=9, decoded_token=None), 364: Logprob(logprob=-15.063230514526367, rank=10, decoded_token=None), 24448: Logprob(logprob=-15.500730514526367, rank=11, decoded_token=None), 574: Logprob(logprob=-15.563230514526367, rank=12, decoded_token=None), 28997: Logprob(logprob=-15.563230514526367, rank=13, decoded_token=None), 507: Logprob(logprob=-15.563230514526367, rank=14, decoded_token=None), 330: Logprob(logprob=-15.625730514526367, rank=15, decoded_token=None), 1536: Logprob(logprob=-15.750730514526367, rank=16, decoded_token=None), 323: Logprob(logprob=-16.375730514526367, rank=17, decoded_token=None), 596: Logprob(logprob=-16.563230514526367, rank=18, decoded_token=None), 256: Logprob(logprob=-16.625730514526367, rank=19, decoded_token=None), 80798: Logprob(logprob=-16.813230514526367, rank=20, decoded_token=None)}\n",
+      "279 {279: Logprob(logprob=-0.0008725888328626752, rank=1, decoded_token=None), 220: Logprob(logprob=-7.125872611999512, rank=2, decoded_token=None), 4846: Logprob(logprob=-10.125872611999512, rank=3, decoded_token=None), 889: Logprob(logprob=-11.875872611999512, rank=4, decoded_token=None), 1101: Logprob(logprob=-12.375872611999512, rank=5, decoded_token=None), 578: Logprob(logprob=-12.625872611999512, rank=6, decoded_token=None), 264: Logprob(logprob=-13.000872611999512, rank=7, decoded_token=None), 574: Logprob(logprob=-13.125872611999512, rank=8, decoded_token=None), 256: Logprob(logprob=-13.125872611999512, rank=9, decoded_token=None), 5270: Logprob(logprob=-13.250872611999512, rank=10, decoded_token=None), 320: Logprob(logprob=-13.750872611999512, rank=11, decoded_token=None), 459: Logprob(logprob=-14.125872611999512, rank=12, decoded_token=None), 364: Logprob(logprob=-14.125872611999512, rank=13, decoded_token=None), 1694: Logprob(logprob=-14.750872611999512, rank=14, decoded_token=None), 9405: Logprob(logprob=-15.250872611999512, rank=15, decoded_token=None), 720: Logprob(logprob=-15.313372611999512, rank=16, decoded_token=None), 34833: Logprob(logprob=-15.438372611999512, rank=17, decoded_token=None), 3778: Logprob(logprob=-15.438372611999512, rank=18, decoded_token=None), 813: Logprob(logprob=-15.563372611999512, rank=19, decoded_token=None), 33600: Logprob(logprob=-15.625872611999512, rank=20, decoded_token=None)}\n",
+      "9405 {9405: Logprob(logprob=-6.890059739816934e-05, rank=1, decoded_token=None), 220: Logprob(logprob=-10.125068664550781, rank=2, decoded_token=None), 38916: Logprob(logprob=-11.375068664550781, rank=3, decoded_token=None), 13118: Logprob(logprob=-12.500068664550781, rank=4, decoded_token=None), 279: Logprob(logprob=-12.625068664550781, rank=5, decoded_token=None), 7342: Logprob(logprob=-13.125068664550781, rank=6, decoded_token=None), 18307: Logprob(logprob=-13.750068664550781, rank=7, decoded_token=None), 256: Logprob(logprob=-13.875068664550781, rank=8, decoded_token=None), 11: Logprob(logprob=-14.000068664550781, rank=9, decoded_token=None), 16381: Logprob(logprob=-14.125068664550781, rank=10, decoded_token=None), 264: Logprob(logprob=-14.250068664550781, rank=11, decoded_token=None), 459: Logprob(logprob=-14.375068664550781, rank=12, decoded_token=None), 3604: Logprob(logprob=-14.625068664550781, rank=13, decoded_token=None), 40260: Logprob(logprob=-14.625068664550781, rank=14, decoded_token=None), 19073: Logprob(logprob=-14.750068664550781, rank=15, decoded_token=None), 65162: Logprob(logprob=-15.187568664550781, rank=16, decoded_token=None), 18665: Logprob(logprob=-15.750068664550781, rank=17, decoded_token=None), 16689: Logprob(logprob=-16.00006866455078, rank=18, decoded_token=None), 539: Logprob(logprob=-16.18756866455078, rank=19, decoded_token=None), 4872: Logprob(logprob=-16.25006866455078, rank=20, decoded_token=None)}\n",
+      "7250 {7250: Logprob(logprob=-0.000730128725990653, rank=1, decoded_token=None), 61628: Logprob(logprob=-7.625730037689209, rank=2, decoded_token=None), 473: Logprob(logprob=-8.625730514526367, rank=3, decoded_token=None), 220: Logprob(logprob=-10.000730514526367, rank=4, decoded_token=None), 45437: Logprob(logprob=-11.500730514526367, rank=5, decoded_token=None), 4761: Logprob(logprob=-12.625730514526367, rank=6, decoded_token=None), 80081: Logprob(logprob=-13.625730514526367, rank=7, decoded_token=None), 44273: Logprob(logprob=-14.938230514526367, rank=8, decoded_token=None), 83342: Logprob(logprob=-15.063230514526367, rank=9, decoded_token=None), 364: Logprob(logprob=-15.063230514526367, rank=10, decoded_token=None), 24448: Logprob(logprob=-15.500730514526367, rank=11, decoded_token=None), 574: Logprob(logprob=-15.563230514526367, rank=12, decoded_token=None), 28997: Logprob(logprob=-15.563230514526367, rank=13, decoded_token=None), 507: Logprob(logprob=-15.563230514526367, rank=14, decoded_token=None), 330: Logprob(logprob=-15.625730514526367, rank=15, decoded_token=None), 1536: Logprob(logprob=-15.750730514526367, rank=16, decoded_token=None), 323: Logprob(logprob=-16.375730514526367, rank=17, decoded_token=None), 596: Logprob(logprob=-16.563230514526367, rank=18, decoded_token=None), 256: Logprob(logprob=-16.625730514526367, rank=19, decoded_token=None), 80798: Logprob(logprob=-16.813230514526367, rank=20, decoded_token=None)}\n",
+      "tensor([-0.0614, -0.3096, -0.9614, -0.9980, -1.0096, -1.0379, -0.0465, -0.3464,\n",
+      "        -0.5465, -0.8464, -0.8967, -1.3967], device='cuda:0')\n",
+      "['The question of the meaning', 'The meaning of life is', 'The question of the meaning', 'The concept of the meaning', 'The meaning of life is', 'One of the most profound', 'Barack Obama, the', 'Barack Obama was born', 'Barack Obama, the', 'Barack Obama was born', 'Former US President Barack Obama', 'Former US President Barack Obama']\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Define the replacement\n",
+    "\n",
+    "N = test.lm_head.out_features\n",
+    "\n",
+    "beam_search_params = SamplingParams(logprobs=20,\n",
+    "                                    max_tokens=1,\n",
+    "                                    temperature=1.0,\n",
+    "                                    detokenize=False)\n",
+    "\n",
+    "def new_forward(self, prompts_batch, input_ids, *args, **kwargs):\n",
+    "    prompts_batch = deepcopy(prompts_batch)\n",
+    "    for i, j in zip(input_ids, prompts_batch):\n",
+    "        j['prompt_token_ids'] += i[1:].tolist()\n",
+    "    output = vllm.generate(prompts_batch,\n",
+    "                                    sampling_params=beam_search_params,\n",
+    "                                    use_tqdm=False)\n",
+    "    logits = torch.full((len(output), N), float('-inf'))\n",
+    "    for i, o in enumerate(output):\n",
+    "        o = o.outputs[0].logprobs[0]\n",
+    "        indices = torch.tensor(list(o.keys()))\n",
+    "        values = torch.tensor([i.logprob for i in o.values()])\n",
+    "        logits[i, indices] = values\n",
+    "\n",
+    "    logits = logits.unsqueeze(1)\n",
+    "    logits = logits.to(input_ids.device)\n",
+    "    # print(logits[:, -1].topk(5, dim=-1))\n",
+    "    # print(logits.shape)\n",
+    "    output_dict = CausalLMOutputWithPast(\n",
+    "            loss=None,\n",
+    "            logits=logits,\n",
+    "            past_key_values=None,\n",
+    "            hidden_states=None,\n",
+    "            attentions=None,\n",
+    "        )\n",
+    "    return output_dict\n",
+    "\n",
+    "# --- Usage ---\n",
+    "print(\"Using patched forward:\")\n",
+    "print(\"Patched forward signature:\", set(inspect.signature(model.forward).parameters))\n",
+    "token_prompts = [TokensPrompt(prompt_token_ids=tokenizer.encode(i, add_special_tokens=False)) for i in inputs]\n",
+    "token_prompts = [j for i in token_prompts for j in [deepcopy(i) for _ in range(num_beam_groups * beams_per_group)]]\n",
+    "test.forward = types.MethodType(new_forward, test)\n",
+    "out = test.generate(\n",
+    "                input_ids = torch.full((len(inputs), 1), tokenizer.bos_token_id, dtype=torch.long, device=model.device),\n",
+    "                # **tokd_input,\n",
+    "                max_new_tokens=5,\n",
+    "                # logits_processor=[logits_processor],\n",
+    "                pad_token_id=tokenizer.eos_token_id,\n",
+    "                do_sample = False, temperature=None, top_p=None,\n",
+    "                num_beams = num_beam_groups * beams_per_group,\n",
+    "                num_beam_groups = num_beam_groups,\n",
+    "                num_return_sequences = num_beam_groups * beams_per_group,\n",
+    "                diversity_penalty = 0.5, use_cache = False,\n",
+    "                prompts_batch = token_prompts,\n",
+    "                output_scores=True, return_dict_in_generate=True\n",
+    "                )\n",
+    "print(out.sequences_scores)\n",
+    "print(tokenizer.batch_decode(out.sequences[:,1:]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d2bac595",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Using original forward:\n",
+      "Restored forward signature: {'args', 'kwargs'}\n",
+      "tensor([-0.0608, -0.3091, -0.9608, -1.0058, -1.0091, -1.0627, -0.0466, -0.3464,\n",
+      "        -0.5466, -0.8464, -0.8951, -1.3951], device='cuda:0')\n",
+      "['The question of the meaning', 'The meaning of life is', 'The question of the meaning', 'The concept of the meaning', 'The meaning of life is', 'One of the most profound', 'Barack Obama, the', 'Barack Obama was born', 'Barack Obama, the', 'Barack Obama was born', 'Former US President Barack Obama', 'Former US President Barack Obama']\n"
+     ]
+    }
+   ],
+   "source": [
+    "def print_forward(self, *args, **kwargs):\n",
+    "    out = self.__class__.forward(self, *args, **kwargs)\n",
+    "    out.logits = out.logits.float()\n",
+    "    # logits = out.logits\n",
+    "    # print(logits[:, -1].float().log_softmax(dim=-1).topk(5, dim=-1))\n",
+    "    return out\n",
+    "\n",
+    "# Swap back to the original\n",
+    "# model.forward = model._original_forward\n",
+    "model.forward = types.MethodType(print_forward, model)\n",
+    "print(\"\\nUsing original forward:\")\n",
+    "print(\"Restored forward signature:\", set(inspect.signature(model.forward).parameters))\n",
+    "out = model.generate(\n",
+    "                **tokd_input,\n",
+    "                max_new_tokens=5,\n",
+    "                # logits_processor=[logits_processor],\n",
+    "                pad_token_id=tokenizer.eos_token_id,\n",
+    "                do_sample = False, temperature=None, top_p=None,\n",
+    "                num_beams = num_beam_groups * beams_per_group,\n",
+    "                num_beam_groups = num_beam_groups,\n",
+    "                num_return_sequences = num_beam_groups * beams_per_group,\n",
+    "                diversity_penalty = 0.5, use_cache = False,\n",
+    "                output_scores=True, return_dict_in_generate=True\n",
+    "                )\n",
+    "print(out.sequences_scores)\n",
+    "print(tokenizer.batch_decode(out.sequences[:,tokd_input[\"input_ids\"].shape[1]:]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e146c195",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "inputs_ = inputs[:1]\n",
+    "tokd_input_ = tokenizer(inputs_, return_tensors=\"pt\", add_special_tokens=False, padding=True).to(model.device)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f3ecfb66",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# token_prompts = [TokensPrompt(prompt_token_ids=tokenizer.encode(i, add_special_tokens=False)) for i in inputs_ * 100]\n",
+    "# token_prompts = [j for i in token_prompts for j in [deepcopy(i) for _ in range(num_beam_groups * beams_per_group)]]\n",
+    "# model.forward = types.MethodType(new_forward, model)\n",
+    "# timeit(lambda: model.generate(\n",
+    "#                 input_ids = torch.full((len(inputs_), 1), tokenizer.bos_token_id, dtype=torch.long, device=model.device),\n",
+    "#                 # **tokd_input,\n",
+    "#                 max_new_tokens=10,\n",
+    "#                 # logits_processor=[logits_processor],\n",
+    "#                 pad_token_id=tokenizer.eos_token_id,\n",
+    "#                 do_sample = False, temperature=None, top_p=None,\n",
+    "#                 num_beams = num_beam_groups * beams_per_group,\n",
+    "#                 num_beam_groups = num_beam_groups,\n",
+    "#                 num_return_sequences = num_beam_groups * beams_per_group,\n",
+    "#                 diversity_penalty = 0.5, use_cache = False,\n",
+    "\n",
+    "#                 prompts_batch = token_prompts\n",
+    "#                 ), number=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4337d343",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# model.forward = model._original_forward\n",
+    "# timeit(lambda: model.generate(\n",
+    "#                 **tokd_input_,\n",
+    "#                 max_new_tokens=10,\n",
+    "#                 # logits_processor=[logits_processor],\n",
+    "#                 pad_token_id=tokenizer.eos_token_id,\n",
+    "#                 do_sample = False, temperature=None, top_p=None,\n",
+    "#                 num_beams = num_beam_groups * beams_per_group,\n",
+    "#                 num_beam_groups = num_beam_groups,\n",
+    "#                 num_return_sequences = num_beam_groups * beams_per_group,\n",
+    "#                 diversity_penalty = 0.5, use_cache = False\n",
+    "#                 ), number=100)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "vllm_waterfall",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

{waterfall-0.2.0 → waterfall-0.2.1}/waterfall/WatermarkerBase.py RENAMED Viewed

@@ -22,6 +22,8 @@ from waterfall.permute import Permute
 from waterfall.WatermarkingFn import WatermarkingFn
 from waterfall.WatermarkingFnFourier import WatermarkingFnFourier
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
 class PerturbationProcessor(LogitsProcessor):
     def __init__(self,
                  N : int = 32000,     # Vocab size

{waterfall-0.2.0 → waterfall-0.2.1}/waterfall/watermark.py RENAMED Viewed

@@ -15,8 +15,6 @@ from waterfall.WatermarkingFnFourier import WatermarkingFnFourier
 from waterfall.WatermarkingFnSquare import WatermarkingFnSquare
 from waterfall.WatermarkerBase import Watermarker
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
 PROMPT = (
     "Paraphrase the user provided text while preserving semantic similarity. "
     "Do not include any other sentences in the response, such as explanations of the paraphrasing. "