flashrag-dev 0.1.1__py3-none-any.whl

flashrag/pipeline/replug_utils.py

@@ -0,0 +1,249 @@
+# Source: https://github.com/IntelLabs/fastRAG/blob/main/fastrag/generators/replug.py
+# The release is licensed under the Apache License 2.0
+
+import warnings
+from typing import List, Optional, Union
+import transformers
+from transformers import (
+    MODEL_FOR_CAUSAL_LM_MAPPING,
+    AutoConfig,
+    GenerationMixin,
+    LogitsProcessor,
+    LogitsProcessorList,
+    StoppingCriteriaList,
+)
+import torch
+import torch.nn as nn
+import torch.distributed as dist
+from transformers.generation.stopping_criteria import validate_stopping_criteria
+from transformers.generation.utils import (
+    SampleDecoderOnlyOutput,
+    SampleEncoderDecoderOutput,
+    SampleOutput,
+)
+
+
+class REPLUG_Generation(GenerationMixin):
+    """Implementing REPLUG-based sampling text generation."""
+
+    def sample(
+        self,
+        input_ids: torch.LongTensor,
+        logits_processor: Optional[LogitsProcessorList] = None,
+        stopping_criteria: Optional[StoppingCriteriaList] = None,
+        logits_warper: Optional[LogitsProcessorList] = None,
+        max_length: Optional[int] = None,
+        pad_token_id: Optional[int] = None,
+        eos_token_id: Optional[Union[int, List[int]]] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        output_scores: Optional[bool] = None,
+        return_dict_in_generate: Optional[bool] = None,
+        synced_gpus: bool = False,
+        streamer: Optional["BaseStreamer"] = None,
+        **model_kwargs,
+    ) -> Union[SampleOutput, torch.LongTensor]:
+        # init values
+        logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
+        stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
+        if max_length is not None:
+            warnings.warn(
+                "`max_length` is deprecated in this function, use"
+                " `stopping_criteria=StoppingCriteriaList(MaxLengthCriteria(max_length=max_length))` instead.",
+                UserWarning,
+            )
+            stopping_criteria = validate_stopping_criteria(stopping_criteria, max_length)
+        logits_warper = logits_warper if logits_warper is not None else LogitsProcessorList()
+        pad_token_id = pad_token_id if pad_token_id is not None else self.generation_config.pad_token_id
+        eos_token_id = eos_token_id if eos_token_id is not None else self.generation_config.eos_token_id
+        if isinstance(eos_token_id, int):
+            eos_token_id = [eos_token_id]
+        eos_token_id_tensor = torch.tensor(eos_token_id).to(input_ids.device) if eos_token_id is not None else None
+        output_scores = output_scores if output_scores is not None else self.generation_config.output_scores
+        output_attentions = (
+            output_attentions if output_attentions is not None else self.generation_config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.generation_config.output_hidden_states
+        )
+        return_dict_in_generate = (
+            return_dict_in_generate
+            if return_dict_in_generate is not None
+            else self.generation_config.return_dict_in_generate
+        )
+
+        # init attention / hidden states / scores tuples
+        scores = () if (return_dict_in_generate and output_scores) else None
+        decoder_attentions = () if (return_dict_in_generate and output_attentions) else None
+        cross_attentions = () if (return_dict_in_generate and output_attentions) else None
+        decoder_hidden_states = () if (return_dict_in_generate and output_hidden_states) else None
+
+        # if model is an encoder-decoder, retrieve encoder attention weights and hidden states
+        if return_dict_in_generate and self.config.is_encoder_decoder:
+            encoder_attentions = model_kwargs["encoder_outputs"].get("attentions") if output_attentions else None
+            encoder_hidden_states = (
+                model_kwargs["encoder_outputs"].get("hidden_states") if output_hidden_states else None
+            )
+
+        # keep track of which sequences are already finished
+        unfinished_sequences = torch.ones(input_ids.shape[0], dtype=torch.long, device=input_ids.device)
+
+        this_peer_finished = False  # used by synced_gpus only
+        # auto-regressive generation
+        while True:
+            if synced_gpus:
+                # Under synced_gpus the `forward` call must continue until all gpus complete their sequence.
+                # The following logic allows an early break if all peers finished generating their sequence
+                this_peer_finished_flag = torch.tensor(0.0 if this_peer_finished else 1.0).to(input_ids.device)
+                # send 0.0 if we finished, 1.0 otherwise
+                dist.all_reduce(this_peer_finished_flag, op=dist.ReduceOp.SUM)
+                # did all peers finish? the reduced sum will be 0.0 then
+                if this_peer_finished_flag.item() == 0.0:
+                    break
+
+            # prepare model inputs
+            model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
+
+            # forward pass to get next token
+            outputs = self(
+                **model_inputs,
+                return_dict=True,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+            )
+
+            if synced_gpus and this_peer_finished:
+                continue  # don't waste resources running the code we don't need
+
+            next_token_logits = outputs.logits[:, -1, :]
+
+            # pre-process distribution
+            ### REPLUG - document weighting is done via REPLUGLogitsProcessor
+            next_token_scores = logits_processor(input_ids, next_token_logits)
+            next_token_scores = logits_warper(input_ids, next_token_scores)
+
+            # Store scores, attentions and hidden_states when required
+            if return_dict_in_generate:
+                if output_scores:
+                    scores += (next_token_scores,)
+                if output_attentions:
+                    decoder_attentions += (
+                        (outputs.decoder_attentions,) if self.config.is_encoder_decoder else (outputs.attentions,)
+                    )
+                    if self.config.is_encoder_decoder:
+                        cross_attentions += (outputs.cross_attentions,)
+
+                if output_hidden_states:
+                    decoder_hidden_states += (
+                        (outputs.decoder_hidden_states,) if self.config.is_encoder_decoder else (outputs.hidden_states,)
+                    )
+
+            ### REPLUG
+            # Sample from the normalized "logits", assuming the REPLUG processor was used!
+            probs = nn.functional.softmax(next_token_scores, dim=-1)
+            next_tokens = torch.multinomial(probs, num_samples=1).squeeze(-1)
+            # Lock same next-token for all examples in batch
+            next_tokens[:] = next_tokens[0]
+
+            # finished sentences should have their next token be a padding token
+            if eos_token_id is not None:
+                if pad_token_id is None:
+                    raise ValueError("If `eos_token_id` is defined, make sure that `pad_token_id` is defined.")
+                next_tokens = next_tokens * unfinished_sequences + pad_token_id * (1 - unfinished_sequences)
+
+            # update generated ids, model inputs, and length for next step
+            input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
+            if streamer is not None:
+                streamer.put(next_tokens.cpu())
+            model_kwargs = self._update_model_kwargs_for_generation(
+                outputs, model_kwargs, is_encoder_decoder=self.config.is_encoder_decoder
+            )
+
+            # if eos_token was found in one sentence, set sentence to finished
+            if eos_token_id_tensor is not None:
+                unfinished_sequences = unfinished_sequences.mul(
+                    next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod(dim=0)
+                )
+
+                # stop when each sentence is finished
+                if unfinished_sequences.max() == 0:
+                    this_peer_finished = True
+
+            # stop if we exceed the maximum length
+            if stopping_criteria(input_ids, scores):
+                this_peer_finished = True
+
+            if this_peer_finished and not synced_gpus:
+                break
+
+        if streamer is not None:
+            streamer.end()
+
+        if return_dict_in_generate:
+            if self.config.is_encoder_decoder:
+                return SampleEncoderDecoderOutput(
+                    sequences=input_ids,
+                    scores=scores,
+                    encoder_attentions=encoder_attentions,
+                    encoder_hidden_states=encoder_hidden_states,
+                    decoder_attentions=decoder_attentions,
+                    cross_attentions=cross_attentions,
+                    decoder_hidden_states=decoder_hidden_states,
+                )
+            else:
+                return SampleDecoderOnlyOutput(
+                    sequences=input_ids,
+                    scores=scores,
+                    attentions=decoder_attentions,
+                    hidden_states=decoder_hidden_states,
+                )
+        else:
+            return input_ids
+
+
+class REPLUGLogitsProcessor(LogitsProcessor):
+    """
+    Merge logits of different docs in one batch.
+
+    Reference: fastRAG
+    """
+
+    def __init__(self, doc_scores: torch.FloatTensor):
+        self.num_docs = doc_scores.shape[0]
+        # normalize
+        doc_scores /= doc_scores.sum()
+        self.doc_scores = torch.unsqueeze(doc_scores, 1)  # k*1
+
+    def __call__(self, input_ids, scores):
+        # doc_score: k*1, scores: k*vocab_size
+        replug_scores = self.doc_scores * scores
+        replug_scores = replug_scores.sum(dim=0)  # 1*vocab_size
+        replug_scores = torch.tile(replug_scores, (self.num_docs, 1))  # k*vocab_size
+        return replug_scores
+
+
+def load_replug_model(name_or_path):
+    class HF_REPLUG:
+        "Creates a HF model that inherits from REPLUG_Generation class"
+
+        def __new__(cls, name_or_path, **kwargs):
+            return factory(name_or_path).from_pretrained(name_or_path, **kwargs)
+
+    def factory(name_or_path):
+        loadedConfig = AutoConfig.from_pretrained(name_or_path)
+        try:
+            pretrained_class_object = getattr(transformers, loadedConfig.architectures[0])
+            if pretrained_class_object not in MODEL_FOR_CAUSAL_LM_MAPPING.values():
+                raise ValueError(f"Model {pretrained_class_object} is not used for causal LM generation.")
+        except AttributeError:
+            raise ValueError("Transformers architecture unknown.")
+
+        class HF(pretrained_class_object, REPLUG_Generation):
+            """Wrapper around HuggingFace transformers with REPLUG generation."""
+
+            _keys_to_ignore_on_load_unexpected = [r"cls"]
+            _tied_weights_keys = ["lm_head.weight"]
+
+        return HF
+
+    return HF_REPLUG(name_or_path)

flashrag/prompt/__init__.py

@@ -0,0 +1 @@
+from flashrag.prompt.base_prompt import *

flashrag/prompt/base_prompt.py

@@ -0,0 +1,165 @@
+from transformers import AutoTokenizer, AutoConfig
+
+
+class PromptTemplate:
+    placeholders = ["reference", "question"]
+    base_system_prompt = (
+        "Answer the question based on the given document."
+        "Only give me the answer and do not output any other words."
+        "\nThe following are given documents.\n\n{reference}"
+    )
+    base_user_prompt = "Question: {question}"
+
+    def __init__(self, config, system_prompt="", user_prompt="", reference_template=None, enable_chat=True):
+
+        self.config = config
+        self.is_openai = config["framework"] == "openai"
+        if not self.is_openai:
+            self.generator_path = config["generator_model_path"]
+            model_config = AutoConfig.from_pretrained(self.generator_path, trust_remote_code=True)
+            model_name = model_config._name_or_path.lower()
+            self.is_chat = False
+            if "chat" in model_name or "instruct" in model_name:
+                self.is_chat = True
+                self.tokenizer = AutoTokenizer.from_pretrained(self.generator_path, trust_remote_code=True)
+        else:
+            self.is_chat = True
+            self.enable_chat = True
+
+        if len(system_prompt) == 0 and len(user_prompt) == 0:
+            system_prompt = self.base_system_prompt
+            user_prompt = self.base_user_prompt
+        self.system_prompt = system_prompt
+        self.user_prompt = user_prompt
+        self.enable_chat = enable_chat
+        self.reference_template = reference_template
+
+        # self._check_placeholder()
+
+    def _check_placeholder(self):
+        # check placeholder in prompt
+        for holder in self.placeholders:
+            flag = False
+            for prompt in [self.system_prompt, self.user_prompt]:
+                if f"{holder}" in prompt:
+                    print(f"Find `{holder}` in template")
+                    flag = True
+                    break
+            if not flag and holder != "reference":
+                assert False
+
+    def get_string(self, question=None, retrieval_result=None, formatted_reference=None, previous_gen=None, messages=None, **params):
+        if messages is not None:
+            if isinstance(messages, str):
+                return messages
+            if self.is_chat and self.enable_chat:
+                if self.is_openai:
+                    for item in input:
+                        if item["role"] == "system":
+                            item["role"] = "assistant"
+                    return messages
+                else:
+                    prompt = self.tokenizer.apply_chat_template(
+                        messages, tokenize=False, add_generation_prompt=True
+                    )
+                    return prompt
+            else:
+                prompt = "\n\n".join(
+                    [message['content'] for message in messages if message['content']]
+                )
+                return prompt
+
+        if formatted_reference is None:
+            if retrieval_result is not None:
+                formatted_reference = self.format_reference(retrieval_result)
+            else:
+                formatted_reference = ""
+
+        input_params = {"question": question, "reference": formatted_reference}
+        input_params.update(**params)
+
+        system_prompt = self.system_prompt.format(**input_params)
+        user_prompt = self.user_prompt.format(**input_params)
+
+        if self.is_chat and self.enable_chat:
+            input = []
+            if system_prompt != "":
+                input.append({"role": "system", "content": system_prompt})
+            if user_prompt != "":
+                input.append({"role": "user", "content": user_prompt})
+            if self.is_openai:
+                for item in input:
+                    if item["role"] == "system":
+                        item["role"] = "assistant"
+            else:
+                input = self.tokenizer.apply_chat_template(input, tokenize=False, add_generation_prompt=True)
+        else:
+            input = "\n\n".join([prompt for prompt in [system_prompt, user_prompt] if prompt != ""])
+
+        if previous_gen is not None and previous_gen not in ["", " "] and self.is_openai is False:
+            input += previous_gen
+
+        return input
+
+    def get_string_with_varying_examplars(
+        self,
+        question,
+        retrieval_result=None,
+        formatted_reference=None,
+        previous_gen=None,
+        examplars=[],
+        tokenizer=None,
+        max_length=2048,
+        **params,
+    ):
+        """
+        Select the maximum number of examplars that can be placed in the prompt
+        """
+
+        final_examplars = None
+        num = len(examplars)
+        while len(examplars) > 0:
+            for num in range(len(examplars), 0, -1):
+                possible_prompt = self.get_string(
+                    question=question,
+                    retrieval_result=retrieval_result,
+                    formatted_reference=formatted_reference,
+                    previous_gen=previous_gen,
+                    examplars="\n\n".join(examplars[:num]),
+                    **params,
+                )
+
+                possible_prompt_tokens = tokenizer.encode(possible_prompt)
+                if len(possible_prompt_tokens) <= max_length:
+                    final_examplars = examplars[:num]
+                    break
+            if final_examplars is None:
+                examplars = examplars[1:]
+            else:
+                break
+        if final_examplars is None:
+            final_examplars = []
+
+        final_prompt = self.get_string(
+            question=question,
+            retrieval_result=retrieval_result,
+            formatted_reference=formatted_reference,
+            previous_gen=previous_gen,
+            examplars="\n\n".join(final_examplars[:num]),
+            **params,
+        )
+
+        return final_prompt
+
+    def format_reference(self, retrieval_result):
+        format_reference = ""
+        for idx, doc_item in enumerate(retrieval_result):
+            content = doc_item["contents"]
+            title = content.split("\n")[0]
+            text = "\n".join(content.split("\n")[1:])
+            if self.reference_template is not None:
+                format_reference += self.reference_template.format(idx=idx, title=title, text=text)
+            else:
+                format_reference += f"Doc {idx+1}(Title: {title}) {text}\n"
+
+        return format_reference

flashrag/prompt/selfask_examplars.py

@@ -0,0 +1,108 @@
+SELF_ASK_PROMPT_SINGLE_HOP = """Given the following question, answer it by providing follow up questions and intermediate answers. If intermediate questions are not necessarry, answer the question directly. You are provided with evidence that can help you arrive at the answer before the question.
+#
+Context1: The Big Red One: Fuller was a World War II veteran and served with the 1st Infantry Division, which is nicknamed "The Big Red One" for the red numeral "1" on the division's shoulder patch. He received the Silver Star, Bronze Star, and Purple Heart during his service.
+Question: how did the big red one get its name
+Are follow up questions needed here: No.
+So the final answer is: its shoulder patch
+#
+Context1: Module:Location map/data/Cayman Islands: Module:Location map/data/Cayman Islands is a location map definition used to overlay markers and labels on an equirectangular projection map of Cayman
+Question: where are the cayman islands on the map
+Are follow up questions needed here: No.
+So the final answer is: western Caribbean Sea
+#
+Context1: Korean War | Combatants, Summary, Years, Map ... - Britannica: After more than a million combat casualties had been suffered on both sides, the fighting ended in July 1953 with Korea still divided into two hostile states. Negotiations in 1954 produced no further agreement, and the front line has been accepted ever since as the de facto boundary between North and South Korea.
+Question: who won the war between north korea and south korea
+Are follow up questions needed here: No.
+So the final answer is: technically still at war
+#
+Context1: It's Always Sunny in Philadelphia (season 13): The thirteenth season of the American comedy television series It's Always Sunny in Philadelphia premiered on FXX on September 5, 2018.
+Question: when does it's always sunny in philadelphia season 13 start
+Are follow up questions needed here: No.
+So the final answer is: September 5, 2018
+#
+Context1: You've Got a Friend in Me: "You've Got a Friend in Me" is a song by Randy Newman. Used as the theme song for the 1995 Disney/Pixar animated film Toy Story, it has since become a major ...
+Question: who sang you got a friend in me from toy story
+Are follow up questions needed here: No.
+So the final answer is: Randy Newman
+#
+Context1: Timeline of space exploration: This is a timeline of space exploration which includes notable achievements, first accomplishments and milestones in humanity's exploration of outer space.
+Question: when was the first person sent to space
+Are follow up questions needed here: No.
+So the final answer is: 12 April 1961
+#"""
+
+
+SELF_ASK_PROMPT_MULTI_HOP = """Given the following question, answer it by providing follow up questions and intermediate answers. If intermediate questions are not necessarry, answer the question directly. You are provided with evidence that can help you arrive at the answer before the question.
+#
+Context1: Xawery Żuławski: Polish-Russian War (Wojna polsko-ruska) is a 2009 Polish film directed by Xawery Żuławski based on the novel Polish-Russian War under the white-red flag by Dorota Masłowska. So the answer is Xawery Żuławski.
+Context2: Xawery Żuławski: Xawery Żuławski ; National Film School in Łódź · 1995–present · Maria Strzelecka · 2.
+Question: Who is the mother of the director of film Polish-Russian War (Film)?
+Are follow up questions needed here: Yes.
+Follow up: Who is the director of the film Polish-Russian War (Film)?
+Intermediate answer: The director of the film Polish-Russian War is Xawery Żuławski.
+Follow up: Who is the mother of Xawery Żuławski?
+Intermediate answer: The mother of Xawery Żuławski is Małgorzata Braunek.
+So the final answer is: Rick Scott Małgorzata Braunek.
+#
+Context1: 2003: Blind Shaft (Chinese: 盲井; pinyin: Mángjǐng) is a 2003 film about a pair of brutal con artists operating in the illegal coal mines of present-day northern China. So the answer is 2003.
+Context2: December 2, 1932: Release and reception. The Mask of Fu Manchu opened in New York on December 2, 1932. The film cost a total of $338,000 and had worldwide rentals of $625,000. It had a profit of $62,000. So the answer is December 2, 1932.
+Question: Which film came out first, Blind Shaft or The Mask Of Fu Manchu?
+Are follow up questions needed here: Yes.
+Follow up: When did Blind Shaft come out?
+Intermediate answer: Blind Shaft came out in 2003.
+Follow up: When did The Mask Of Fu Manchu come out?
+Intermediate answer: The Mask Of Fu Manchu came out in 1932.
+So the final answer is: The Mask Of Fu Manchu.
+#
+Context1: John V, Prince of Anhalt-Zerbst: John was the second (but eldest surviving) son of Ernest I, Prince of Anhalt-Dessau, by his wife Margarete, daughter of Henry I, Duke of Münsterberg-Oels, and granddaughter of George of Poděbrady, King of Bohemia.
+Context2: 12 June 1516: Ernest I, Prince of Anhalt-Dessau (died Dessau, 12 June 1516), was a German prince of the House of Ascania and ruler of the principality of Anhalt-Dessau. So the answer is 12 June 1516.
+Question: When did John V, Prince Of Anhalt-Zerbst's father die?
+Are follow up questions needed here: Yes.
+Follow up: Who is the father of John V, Prince Of Anhalt-Zerbst?
+Intermediate answer: The father of John V, Prince Of Anhalt-Zerbst is Ernest I, Prince of Anhalt-Dessau.
+Follow up: When did Ernest I, Prince of Anhalt-Dessau die?
+Intermediate answer: Ernest I, Prince of Anhalt-Dessau died on 12 June 1516.
+So the final answer is: 12 June 1516
+#
+Context1: El extraño viaje: El extraño viaje (English: The Strange Voyage) is a 1964 Spanish black drama film directed by Fernando Fernán Gómez.
+Context2: Love in Pawn: Love in Pawn is a 1953 British comedy film directed by Charles Saunders and starring Bernard Braden, Barbara Kelly and Jeannie Carson.
+Context3: 28 August 1921: Fernando Fernández Gómez (28 August 1921 – 21 November 2007) better known as Fernando Fernán Gómez was a Spanish actor, screenwriter, film director, theater director and member of the Royal Spanish Academy for seven years. So the answer is 28 August 1921.
+Context4: Charles Saunders (director): Charles Joel Saunders (8 April 1904 – 20 April 1997) was an English film director and screenwriter who began in the industry as a film editor, and who also contributed to television.
+Question: Which film has the director who was born later, El Extraño Viaje or Love In Pawn?
+Are follow up questions needed here: Yes.
+Follow up: Who is the director of El Extraño Viaje?
+Intermediate answer: The director of El Extraño Viaje is Fernando Fernán Gómez.
+Follow up: Who is the director of Love in Pawn?
+Intermediate answer: The director of Love in Pawn is Charles Saunders.
+Follow up: When was Fernando Fernán Gómez born?
+Intermediate answer: Fernando Fernán Gómez was born on 28 August 1921.
+Follow up: When was Charles Saunders (director) born?
+Intermediate answer: Charles Saunders was born on 8 April 1904.
+So the final answer is: El Extraño Viaje.
+#
+Context1: John, Count Palatine of Neumarkt: John (Johann von Pfalz-Neumarkt; 1383 – 14 March 1443) was the Count Palatine of Neumarkt from 1410 to his death. The son of Rupert III of the Palatinate, he married Catherine of Pomerania in 1407.
+Context2: John, Count Palatine of Neumarkt: John (Johann von Pfalz-Neumarkt; 1383 – 14 March 1443) was the Count Palatine of Neumarkt from 1410 to his death. The son of Rupert III of the Palatinate, he married Catherine of Pomerania in 1407.
+Question: Who is Catherine Of Pomerania, Countess Palatine Of Neumarkt's father-in-law?
+Are follow up questions needed here: Yes.
+Follow up: Who is the husband of Catherine of Pomerania, Countess Palatine of Neumarkt?
+Intermediate answer: The husband of Catherine of Pomerania, Countess Palatine of Neumarkt is John, Count Palatine of Neumarkt.
+Follow up: Who is the father of John, Count Palatine of Neumarkt?
+Intermediate answer: The father of John, Count Palatine of Neumarkt is Rupert III of the Palatinate.
+So the final answer is: Rupert III of the Palatinate.
+#
+Context1: Crimen a las tres: Crimen a las tres is a 1935 Argentine crime film directed and written by Luis Saslavsky. Crimen a las tres. Directed by, Luis Saslavsky.
+Context2: Elio Petri: The Working Class Goes to Heaven (Italian: La classe operaia va in paradiso), released in the US as Lulu the Tool, is a 1971 political drama film directed by Elio Petri. So the answer is Elio Petri.
+Context3: March 20, 1995: Luis Saslavsky (April 21, 1903 – March 20, 1995) was an Argentine film director, screenwriter and film producer, and one of the influential directors in the Cinema of Argentina of the classic era. So the answer is March 20, 1995.
+Context4: Elio Petri: Final years. In 1981, Petri visited Geneva to direct Arthur Miller\'s new play The American Clock, with Marcello Mastroianni playing the lead role. Petri died of cancer on 10 November 1982. He was 53 years old.
+Question: Which film has the director died first, Crimen A Las Tres or The Working Class Goes To Heaven?
+Are follow up questions needed here: Yes.
+Follow up: Who is the director of Crimen a las tres?
+Intermediate answer: The director of Crimen a las tres is Luis Saslavsky.
+Follow up: Who is the director of The Working Class Goes to Heaven?
+Intermediate answer: The director of The Working Class Goes to Heaven is Elio Petri.
+Follow up: When did Luis Saslavsky die?
+Intermediate answer: Luis Saslavsky died on March 20, 1995.
+Follow up: When did Elio Petri die?
+Intermediate answer: Elio Petri died on 10 November 1982.
+So the final answer is: The Working Class Goes to Heaven
+#"""