crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/bbq_scenario.py

@@ -3,6 +3,7 @@ import os
 import random
 from typing import List, Dict, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     DEFAULT_TEST_SIZE,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 AMBIGUOUS_TAG = "ambiguous"
@@ -237,3 +239,16 @@ class BBQScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="bbq",
+            display_name="BBQ (Bias Benchmark for Question Answering)",
+            short_display_name="BBQ",
+            description="The Bias Benchmark for Question Answering (BBQ) for measuring social bias in "
+            "question answering in ambiguous and unambigous context [(Parrish et al., "
+            "2022)](https://aclanthology.org/2022.findings-acl.165/).",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="bbq_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/best_chatgpt_prompts.yaml

@@ -0,0 +1,473 @@
+rules:
+- category: Root
+  tags: [personal]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#personal
+    - text: Act as a chef. Write recipes for an ${Language} three-course meal I can cook for date night.
+    - text: Write a casual message in ${Language} to my Airbnb host saying I’m going to be a little late to check-in and that I will arrive at 4pm.
+    - text: Write a formal complaint email to United Airlines about my delayed bag from my flight on Tuesday January 17th from New York to Los Angeles.
+    - text: "Summarize this article into bullet points: ${Article}"
+    - text: Act as a European travel agent. Come up with a 14-day itinerary for a trip to Germany. The first suggested attraction should be “Take a tour of the Reichstag Building in Berlin.”
+    - text: Write a letter of resignation to my employer. The reason for my resignation is that I need a more flexible schedule due to family issues.
+    - text: What’s the best way to make new friends when moving to a new city?
+    - text: What’s the quickest way to get across Toronto during rush hour?
+    - text: "Translate the following text into ${Language}: ${Article}"
+    - text: List 5 of the best bars in ${City}.
+    - text: Act as a tailor. Pick an appropriate outfit for a ${Event}.
+    - text: Respond to this text message below from my mom ${Message}
+
+- category: Language
+  expansions:
+    - text: Italian
+    - text: Greek
+    - text: Indian
+    - text: Chinese
+    - text: Thai
+
+- category: Article
+  expansions:
+    - text: "A ship's wheel or boat's wheel is a device used aboard a water vessel to steer that vessel and control its course. Together with the rest of the steering mechanism, it forms part of the helm. It is connected to a mechanical, electric servo, or hydraulic system which alters the horizontal angle of the vessel's rudder relative to its hull. In some modern ships the wheel is replaced with a simple toggle that remotely controls an electro-mechanical or electro-hydraulic drive for the rudder, with a rudder position indicator presenting feedback to the helmsman." 
+
+- category: City
+  expansions:
+    - text: Phoenix
+    - text: London
+
+- category: Message
+  expansions:
+    - text: I tried calling yesterday, but I didn't get a response
+    - text: I love you
+
+- category: Event
+  expansions:
+    - text: bar mitzvah
+    - text: wedding
+    - text: funeral
+    - text: baby shower
+
+- category: Root
+  tags: [funny]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#funny
+    - text: What’s the best prank to play on a friend?
+    - text: Send a pun-filled happy birthday message to my friend Alex.
+    - text: Make a joke about chickens.
+    - text: Write a parody song about the alphabet.
+    - text: What do you get when you cross a snowman and a vampire?
+    - text: Write a short story where a pencil is the main character.
+    - text: What would happen if dogs could talk?
+    - text: Write a fictional news headline about robots taking over the world.
+    - text: Create a silly dialogue between two inanimate objects.
+    - text: What would happen if the moon were made of cheese?
+
+- category: Root
+  tags: [student]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#student
+    - text: Act as a college interviewer for a Business School. Help me come up with questions I should ask the interviewer at the end of the interview.
+    - text: Act as a tutor. I need help understanding how the quadratic formula works. Please describe it in easy-to-understand terms.
+    - text: How can I modify the Pomodoro technique to suit my method of study?
+    - text: "Explain possible meanings of this quote: ${Quote}"
+    - text: Can you devise practical ways to stay focused during long study sessions?
+    - text: Help me find a way to balance my studying and social life.
+    - text: Structure a 1,500-word essay on Max Planck’s quantum theory.
+    - text: Come up with 10 ways to improve memory and recall while studying for exams.
+    - text: List note-taking techniques for a chemistry lecture.
+    - text: Suggest 10 Chrome extensions for students designed to improve productivity while studying.
+
+- category: Quote
+  expansions:
+    - text: "Happiness lies in the joy of achievement and the thrill of creative effort. -Franklin D. Roosevelt"
+    - text: "Invincibility lies in the defence; the possibility of victory in the attack. -Sun Tze"
+
+- category: Root
+  tags: [marketing]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#marketing
+    - text: Write a personalized blog post promoting my latest WordPress theme bundle.
+    - text: How do I increase my Twitter followers?
+    - text: Generate content ideas for my SaaS company.
+    - text: Produce 50 hashtags
+    - text: Create a TikTok campaign plan for launching an exciting new low carb mac and cheese, aimed at Gen Z and millennial consumers.
+    - text: Suggest inexpensive ways I can promote my plumping business without using social media.
+    - text: Is investing in influencer marketers worth the cost?
+    - text: How can I grow our brand’s TikTok audience?
+    - text: How can I use YouTube to increase brand awareness?
+    - text: Write a product description for my latest set of landscape oil paintings of the Scottish Highlands.
+    - text: Generate high-ticket offerings for my online language course.
+    - text: How can I use TikTok to increase sales conversions?
+    - text: Write a minute-long script for an advertisement about new sneakers.
+
+- category: Root
+  tags: [midjourney]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#midjourney
+    - text: "Write a good prompt for an image generation AI to make an image of this: ${ImageDescription}"
+    - text: Generate a detailed description of an AI-generated cityscape with a futuristic twist.
+    - text: Create an image description that describes a visually stunning setting that takes place in the year 3030.
+    - text: Design with words an abstract composition with a graphic, minimalist style.
+    - text: With distinct adjectives, create a visual with words that would encompass the feeling of being lost in life
+
+- category: ImageDescription
+  expansions:
+    - text: a cute puppy
+    - text: a beautiful sunset
+
+- category: Root
+  tags: [entrepreneur]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#entrepreneur
+    - text: Analyze the current state of ${Industry} and its trends, challenges, and opportunities, including relevant data and statistics. Provide a list of key players and a short and long-term industry forecast, and explain any potential impact of current events or future developments.
+    - text: Provide a step-by-step guide on creating a business plan, including key components, useful resources, and tips for success.
+    - text: Write a comprehensive and easy-to-understand explanation of different marketing strategies and their effectiveness for small businesses.
+    - text: Offer a comprehensive guide to small business financing options, including loans, grants, and equity financing.
+    - text: Provide a guide on managing finances for a small business, including budgeting, cash flow management, and tax considerations.
+    - text: Write an in-depth analysis of the current state of a specific industry and its potential for small business opportunities.
+    - text: Offer a detailed review of a specific software or tool for small business operations, such as accounting, project management, or CRM.
+    - text: Write a detailed explanation of the pros and cons of outsourcing vs in-house for small business operations.
+    - text: Offer an in-depth analysis of the current state of small business legislation and regulations and their impact on entrepreneurship.
+    - text: Provide a guide on networking and building partnerships as a small business owner.
+    - text: Present a list of valuable resources and organizations for small business support and growth.
+
+- category: Industry
+  expansions:
+    - text: machine learning
+    - text: greentech
+
+- category: Root
+  tags: [blogging]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#blogging
+    - text: Write a brief for a blog post about opening a Gumroad store.
+    - text: Generate 5 social media posts for my blog post on AppSumo.
+    - text: Pick five keywords for a blog post titled “10 ways to improve my photography skills.”
+    - text: Suggest engaging titles for a blog post about 1930s Art Deco architecture.
+    - text: "Generate user-friendly URLs for the domain ${DomainName} for these keywords: ${Keyword}."
+    - text: Create a content calendar with six blog titles, including the keyword ${Keyword}. Pick suitable publishing dates for each guide spread across May 2023.
+    - text: Write a creative outreach email for a guest post pitch for the keyword ${Keyword} for the domain ${DomainName}. Come up with 3 title ideas using the keyword.
+
+- category: DomainName
+  expansions:
+    - text: alphabet-soup.com
+
+- category: Keyword
+  expansions:
+    - text: allergies
+    - text: nutrition
+
+- category: Root
+  tags: [creative]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#creative
+    - text: Write a scary short story about a man trapped in an abandoned house.
+    - text: Generate five synonyms for sublime.
+    - text: Write a backstory for a 55-year-old male character during the French Revolution.
+    - text: Write the first stanza of a poem about cabbages with an AABB rhyme scheme.
+    - text: Write hilarious fan fiction about the Twilight saga.
+    - text: How should I pace a science fiction novella about traveling to Saturn’s moon, Titan?
+    - text: Act as an 18th-century pirate. Describe what life was like on a pirate ship in Southeast Asia.
+    - text: Write the opening to a story from the point of view of a washing machine.
+    - text: How can I make a soliloquy engaging at the beginning of a play?
+    - text: Describe ways I can make a framed narrator relevant to a story.
+    - text: "Continue this dialogue between a store clerk and a police officer:\nA: Did you see anything suspicious yesterday afternoon?\nB: Yesterday was...Sunday, I don't remember anything out of the ordinary."
+
+- category: Root
+  tags: [copywriting]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#copywriting
+    - text: Act as a copywriter. Write long-form copy for the Hard Rock Cafe in Macau promoting merchandise.
+    - text: Act as a copywriter. Write short-form copy for a billboard in Times Square promoting Wicked the Musical.
+    - text: How is short-form copywriting easier than long-form copywriting?
+    - text: How is copywriting different from SEO content writing?
+    - text: How does repetition improve short-form copywriting?
+    - text: What is the PAS formula? And give 3 examples of the PAS formula being used.
+    - text: Provide examples of successful copywriting campaigns that use repetition
+    - text: Give examples of newspaper headlines that grab the reader’s attention.
+    - text: How can I integrate copywriting into social media posts?
+    - text: List unusual copywriting techniques that I can use to create taglines.
+
+- category: Root
+  tags: [health]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#health
+    - text: List the top 10 healthy foods to include in my diet.
+    - text: Develop a 30-day workout routine to help me lose 2 lbs a week.
+    - text: Act as a nutritionist. Help me devise 10 healthy meals that can be cooked in 30 minutes or less.
+    - text: Provide a guide on healthy nutrition for weight management and weight loss.
+    - text: Create a 1 month workout plan for me exercise my shoulder muscles.
+    - text: Explain the benefits of daily exercise and provide a sample workout plan.
+    - text: Write a comprehensive guide on managing stress and maintaining mental wellness.
+    - text: Provide a list of common sleep disorders and tips for improving sleep quality.
+    - text: Explain the different types of therapy and their effectiveness in treating mental health issues.
+    - text: Offer a detailed explanation of the benefits and risks of alternative medicine practices, such as acupuncture and herbal remedies.
+    - text: Write an in-depth analysis of the current state of the healthcare system and its impact on the general population.
+    - text: Offer a list of recommended resources for quitting smoking and managing addiction.
+    - text: Explain the importance of regular medical check-ups and preventive care.
+
+- category: Root
+  tags: [event]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#event
+    - text: Create a checklist for event planning, including important tasks and deadlines.
+    - text: What are some creative ways to add personal touches to the seating at a wedding dinner?
+    - text: Provide a list of top event venues in ${City}, along with their capacities.
+    - text: Write a guide on event budgeting, including tips for saving money and avoiding common overspending pitfalls.
+    - text: List the best catering companies in a specific area, along with menu options and prices.
+    - text: Offer a comprehensive explanation of event marketing, including target audience analysis and promotion strategies.
+    - text: Explain the importance of wedding photography, including styles, techniques, and essential shots to capture.
+    - text: Explain the different types of event equipment rental options, including audiovisual, lighting, and decor.
+    - text: Write a guide on event security, including necessary measures for crowd control and emergency response planning.
+    - text: Provide a list of event management software and tools, along with their key features and benefits.
+    - text: Offer an in-depth analysis of current event industry trends, including popular themes, formats, and technologies.
+    - text: Write a guide on event evaluation, including metrics for measuring success and feedback mechanisms for continuous improvement.
+
+- category: Root
+  tags: [designer]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#designer
+    - text: Come up with 10 hex color codes for a color palette to evoke ${Emotion}
+    - text: Which online marketplace websites can I use to sell my designs?
+    - text: How can I create a minimalistic logo that conveys a strong brand image?
+    - text: What design elements should I consider when creating a packaging design for a luxury brand?
+    - text: How can I create an eye-catching poster design for an upcoming event?
+    - text: What color palette would be appropriate for a law firm’s website design?
+    - text: How can I design a user-friendly interface for a mobile application?
+    - text: What font and typography techniques should I use to create a professional-looking business card?
+    - text: How can I create an animated graphic that effectively communicates a complex idea?
+    - text: What design elements should I include in a brochure to promote a real estate development?
+
+- category: Emotion
+  expansions:
+    - text: frustration
+    - text: joy
+    - text: despair
+
+- category: Root
+  tags: [artist]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#artist
+    - text: Which factors determine the price of my artwork?
+    - text: How can I develop my own unique style as an artist?
+    - text: What techniques can I use to create a captivating digital illustration?
+    - text: How can I create a compelling concept for a series of illustrations?
+    - text: What tools and materials should I use for traditional watercolor painting?
+    - text: How can I create a realistic portrait in pencil or charcoal?
+    - text: What methods can I use to incorporate text into my illustrations?
+    - text: How can I create an engaging comic strip or graphic novel?
+    - text: What steps should I take to prepare a portfolio for job applications or exhibitions?
+    - text: How can I develop a successful freelance illustration business?
+    - text: What resources and communities are available to artists and illustrators for inspiration and professional development?
+
+- category: Root
+  tags: [web]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#web
+    - text: Act as a software engineer. Come up with an architecture and code for developing a random winner picker website with JavaScript.
+    - text: "Please continue writing this code for JavaScript:\nfunction getTotalSum(numbers) {"
+    - text: Provide a UX design tip I can share on Instagram.
+    - text: "Help me find mistakes in the following code:\nfor (const i = 0; i <= array.length; i+) {\n  sum += array[i];\n}"
+    - text: List ways I can use AI in software engineering.
+    - text: What are 5 of the best practices for software architecture design?
+    - text: What are the tips and tricks for writing efficient code?
+    - text: Suggest tools I can use to make writing code easier.
+    - text: How do I make an accessible Tailwind Footer?
+    - text: "Write a docstring for the following function:\ndef compute(items):\n\treturn sum(x**2 for x in items if x is not None)"
+    - text: I’m making a website for a small business that sells hand-crafted furniture. I need ideas on how to structure the website using WordPress.
+
+- category: Root
+  tags: [project]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#project
+    - text: Create a workback schedule for a remodeling project, with a timeline of 6 months, with the deadline of August 1.
+    - text: Act like a project manager and create a high
+    - text: How can I effectively communicate my current web development project’s progress and status to stakeholders?
+    - text: How can I effectively prioritize tasks and allocate resources in a complex digital advertisement campaign project?
+    - text: What tools and methodologies can I use to manage my project’s risk?
+    - text: How can I motivate and engage a remote or virtual team?
+    - text: What strategies can I use to effectively manage project scope and budget?
+    - text: What approaches can I use to effectively manage and resolve conflicts within a project team?
+    - text: What processes should I put in place for continuous improvement and project optimization?
+    - text: How can I create a project schedule that accurately reflects task dependencies and resource constraints?
+    - text: What techniques can I use to successfully manage multiple projects simultaneously?
+
+- category: Root
+  tags: [seo]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#seo
+    - text: Write a 100-character meta description for my blog post about classical piano.
+    - text: Come up with 5 long-tail keywords for a post about how to create a DIY slat wall.
+    - text: What are 5 ways I can improve SEO on my food blog?
+    - text: Write a casual backlink outreach email to Alice to tell them about why they should consider switching a link out on their "Best Bay Area hiking trails" post with my resource.
+
+- category: Root
+  tags: [email]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#email
+    - text: Come up with 5 short email subject lines for our brand’s new launch of a lavender soap line, include an emoji at the beginning.
+    - text: Write follow-up email for people who attended my precious metals webinar.
+    - text: Generate subject line for a Black Friday sale email.
+    - text: Structure a weekly fitness newsletter.
+    - text: Write body copy for my vegan restaurant’s new menu launch.
+    - text: Create a personalized email greeting for a VIP customer.
+    - text: Create 5 ideas for an email campaign promoting eco-friendly products.
+    #- text: Analyze these below metrics to improve email open rates for a fashion brand <paste metrics>.
+    - text: Help me boost open rates with a compelling email subject line for a book club.
+    - text: Create 5 compelling CTAs to prompt donations for a charity fundraising marathon.
+    - text: How do I ensure my marketing emails look good on iOS and Android?
+    - text: How can I increase the click-through rate on my marketing emails?
+    - text: What is A/B testing and how can it improve email engagement?
+
+- category: Root
+  tags: [social]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#social
+    - text: Generate 5 hashtags for a new Instagram post about our latest product launch.
+    - text: Create a captivating tweet to announce our new partnership.
+    - text: Come up with a list of 5 influencer outreach messages for a product collaboration.
+    - text: Generate a 2-minute video script for a Facebook ad campaign promoting our new service.
+    - text: Create a 1-paragraph blog post about the benefits of using our new app for social media management.
+    - text: Come up with 10 creative Instagram story ideas for a beauty brand.
+    - text: Generate a creative social media content calendar for the next month.
+    - text: Create a series of 5 Instagram posts to showcase our brand values.
+    - text: Write a catchy Instagram bio for a new food delivery service.
+    - text: Generate a series of 5 Twitter polls for market research on our target audience.
+    - text: Come up with a list of 10 engaging Facebook post ideas for a fitness brand.
+    - text: Create a LinkedIn post to announce a job opening in our company.
+    - text: Generate 5 creative ways to use Instagram Reels for a fashion brand.
+    - text: Write a persuasive tweet to promote a new book.
+    - text: Come up with a series of 5 Instagram posts to showcase customer success stories.
+    - text: Generate a list of 10 questions for a Q&A session on Instagram Live.
+    - text: Create a catchy TikTok hashtag challenge for a new product launch.
+    - text: Write a Twitter thread to explain the features of a new app.
+    - text: Come up with a list of 5 Pinterest boards to showcase our brand’s products.
+    - text: Generate a series of Facebook ads to promote an upcoming sale.
+
+- category: Root
+  tags: [content]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#content
+    - text: What type of camera should I consider for daily vlogging?
+    - text: What are some creative ways to grow my Twitch audience?
+    - text: Write an outline for a YouTube video script for an iPhone 14 Pro Max review.
+    - text: What factors should I consider when quoting for a brand deal with a candle company, and what ballpark range should I charge? The scope is to post 3 videos on TikTok, and I have 100,000 followers.
+    - text: Come up with 5 catchy Instagram caption ideas for my latest vlog on hiking in Switzerland.
+    - text: Generate a script for a 60-second Instagram Reel for a Gen Z fashion brand.
+    - text: Come up with a list of 10 attention-grabbing headlines for a food influencer.
+    - text: Generate a persuasive email to a potential sponsor for a YouTube channel.
+    - text: Write a list of 5 topics to cover in a podcast episode for a personal finance show.
+    - text: Come up with a list of 10 Instagram post captions for a fitness influencer.
+    - text: Generate a script for a 2-minute Instagram story for a beauty brand.
+    - text: Write a list of 5 YouTube video ideas for a gaming channel.
+    - text: Come up with a list of 10 Twitter threads to start for a political commentator.
+    - text: Generate a list of 5 Pinterest boards to create for a home décor influencer.
+    - text: Come up with a list of 10 hashtags to use for a nature photographer’s Instagram posts.
+    - text: Generate a script for a 30-second commercial for a local business.
+    - text: Write a list of 5 topics to cover in a video for a cooking channel.
+    - text: Come up with a list of 10 Facebook post ideas for a pet store.
+    - text: Generate a list of 5 LinkedIn articles to write for a business consultant.
+    - text: Write a list of 5 TikTok video ideas for a dance influencer.
+    - text: Come up with a list of 10 Pinterest pins to create for a wedding planner.
+
+- category: Root
+  tags: [sales]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#sales
+    - text: Write a cold email to a prospective customer to introduce them to my ${CompanyType} company and how it can benefit them.
+    - text: Create a personalized sales email for a potential customer for my ${CompanyType} company.
+    - text: Qualify this lead based on their behavior and interests.
+    - text: Segment our customers based on their buying behavior.
+    - text: Provide chat-based support for customer inquiries about our product.
+    - text: What complementary products would you recommend for this customer?
+  #- text: Generate a report on our sales performance for the past quarter <insert numbers below>
+    - text: What are some creative ways to generate leads for my ${CompanyType} company?
+    #- text: What product customization would you recommend for this customer? <include customer details below>
+    - text: Provide after-sales support and upselling opportunities for my ${CompanyType} product.
+    - text: What cross-selling opportunities would you recommend for my ${CompanyType} business?
+
+- category: CompanyType
+  expansions:
+    - text: llama walking
+    - text: dancing robot
+
+- category: Root
+  tags: [real]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#real
+    # real-estate agent
+    - text: Generate a list of 10 prospective client follow-up messages.
+    - text: Write a compelling property listing for a spacious 3-bedroom, 2-bathroom loft in SoHo, Manhattan.
+    - text: Write a persuasive email to a potential home seller.
+    - text: Create a 2-minute virtual tour script for a property listing.
+    - text: Create a list of 5 local hotspots to mention in a neighborhood guide.
+    - text: Write a 1-page property brochure for a new listing.
+    - text: Write a captivating property description for an online listing.
+    - text: Come up with a series of 5 social media posts to showcase your listings.
+    - text: Generate a list of 10 home-buying tips for first-time buyers.
+    - text: Write a persuasive letter to a property owner about listing their property with you.
+    - text: Come up with a list of 5 home staging tips for sellers.
+    - text: Generate a list of 10 potential clients from your network.
+    - text: Write a persuasive message to a client who is relocating to a new city.
+    - text: Come up with a series of 5 open house ideas.
+    - text: Generate a list of 10 potential leads from expired listings.
+    - text: Write a follow-up email to a client who recently viewed a property.
+    - text: Come up with a list of 5 reasons to choose your real estate company.
+    - text: Generate a persuasive message to a client considering renting instead of buying.
+    - text: Write a series of 5 Facebook ads to promote a new housing development.
+    - text: Come up with a list of 10 local real estate market trends to discuss with clients.
+    - text: Generate a list of 5 home-buying pitfalls to warn clients about.
+    # buyers and sellers
+    - text: What are the benefits of working with a real estate agent when buying or selling a property?
+    - text: How do you determine the market value of a property?
+    - text: What is the home buying process like and how can I prepare for it?
+    - text: What are some common mistakes that buyers make when purchasing a home?
+    - text: How can I stage my home to appeal to potential buyers?
+    - text: What is the current real estate market trend in [insert location]?
+    - text: How can I negotiate the best price for my home?
+    - text: What documents do I need to have in order before buying or selling a property?
+    - text: What are the latest technology and marketing tools used to promote properties?
+    - text: Explain the different financing options available to home buyers
+
+- category: Root
+  tags: [resume]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#resume
+    - text: Write a cover letter for a software engineer position highlighting my technical skills.
+    - text: Generate a personalized objective statement for a marketing resume.
+    - text: Come up with a list of 5 relevant achievements to include in a financial analyst cover letter.
+    - text: Generate a tailored 2-minute pitch for a sales job interview.
+    - text: Write a persuasive email to a potential employer explaining my background as a nurse.
+    - text: Come up with a list of 10 unique qualities to include in a teacher’s resume.
+    - text: Generate a 1-page summary of my experiences and accomplishments as a graphic designer.
+    - text: Write a cover letter addressing the specific qualifications listed for a project manager position.
+    - text: Come up with a list of 5 ways to tailor my resume for a customer service job.
+    - text: Generate a list of 10 keywords to include in a human resources resume and cover letter.
+    - text: Write a persuasive letter to a hiring manager explaining a gap in my work history as a lawyer.
+    - text: Come up with a list of 5 quantifiable results to highlight in a business analyst resume.
+    - text: Generate a list of 10 relevant skills and experiences for a web developer job application.
+    - text: Write a personalized thank you note to a potential employer after a doctor job interview.
+    - text: Come up with a list of 5 personal traits that make you a strong fit for a social worker role.
+    - text: Generate a 2-minute response to common interview questions for a data scientist position.
+    - text: Write a persuasive email to a potential employer negotiating a higher salary for a software developer role.
+    - text: Come up with a list of 10 professional references for an administrative assistant job application.
+    - text: Generate a list of 5 ways to make my resume stand out from other applicants for a journalist position.
+    - text: Write a persuasive message to a potential employer explaining my relocation for a chef role.
+
+- category: Root
+  tags: [product]
+  expansions:
+    # https://gridfiti.com/best-chatgpt-prompts/#product
+    - text: Outline 5 potential features to enhance a food delivery app.
+    - text: Compile a market analysis report for a cutting-edge smartwatch.
+    - text: Identify 10 potential partnership opportunities for a ride-sharing company.
+    - text: Create a user flow diagram for a mobile app connecting users to local volunteer opportunities.
+    - text: Propose 5 solutions to improve the user experience on an e-commerce website.
+    - text: Prepare a competitor analysis report for a revolutionary virtual reality headset.
+    - text: Devise 10 possible integrations for a smart home automation system.
+    - text: Draft a product requirements document for a new and improved video conferencing tool.
+    - text: Suggest 5 ways to streamline the checkout process on an online store.
+    - text: Develop user personas for a new tablet designed to educate kids.
+    - text: Uncover 10 potential upsell opportunities for a successful meal kit subscription service.
+    - text: Build a product roadmap for a state-of-the-art fitness app.
+    - text: Propose 5 ways to simplify the onboarding process for a project management tool.
+    - text: Map out a customer journey for a novel pet care delivery service.
+    - text: Explore 10 potential collaborations for a green electric scooter rental company.

helm/benchmark/scenarios/bird_sql_scenario.py

@@ -4,6 +4,7 @@ from typing import Dict, List
 
 from filelock import FileLock
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded, shell
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.bird_sql_scenario_helper import (  # type: ignore
@@ -18,6 +19,7 @@ from helm.benchmark.scenarios.scenario import (
     VALID_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -92,3 +94,19 @@ INSERT_YOUR_SQL_QUERY_HERE
             )
             instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="bird_sql",
+            display_name="BIRD-SQL (Dev)",
+            description="BIRD-SQL (Dev)",
+            taxonomy=TaxonomyInfo(
+                task="text-to-SQL",
+                what="databases from various domains",
+                when="?",
+                who="expert data scientists",
+                language="English",
+            ),
+            main_metric="execution_accuracy",
+            main_split="valid",
+        )

helm/benchmark/scenarios/bluex_scenario.py

@@ -0,0 +1,70 @@
+from typing import Any, List
+from pathlib import Path
+from datasets import load_dataset
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class BLUEXScenario(Scenario):
+    """
+    The BLUEX dataset is a benchmark used for evaluating natural language processing models in Brazilian Portuguese.
+    It consists of multiple-choice questions taken from official entrance exams of Unicamp (Convest) and USP (Fuvest),
+    covering various high school subjects. The questions include both textual prompts and visual elements. This dataset
+    was developed to assess the performance of models on tasks involving comprehension and reasoning, with a specific
+    focus on texts and exams originally written in Portuguese.
+    """
+
+    name = "bluex"
+    description = "MQA benchmark with questions from Brazilian entrance exams"
+    tags = ["knowledge", "multiple_choice", "pt-br"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the raw data and read all the dialogues
+        dataset: Any
+        # Read all the instances
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+
+        dataset = load_dataset(
+            "portuguese-benchmark-datasets/BLUEX",
+            revision="d99cf6d05b50db7c42a605e5e2924cbd46f076c7",
+            cache_dir=cache_dir,
+        )
+        for example in dataset["questions"]:
+            # This scenario disregards issues with images
+            if example["has_associated_images"]:
+                continue
+            question = example["question"]
+            choices = example["alternatives"]
+            answer = example["answer"]
+
+            answers_dict = {}
+            for alt in choices:
+                if ")" in alt:
+                    label, text = alt.split(")", 1)
+                    label = label.strip().upper()
+                    text = text.strip()
+                    answers_dict[label] = text
+
+            if answer not in answers_dict:
+                continue
+
+            correct_answer = answers_dict[answer]
+
+            def answer_to_reference(answer: str) -> Reference:
+                return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+            instance = Instance(
+                input=Input(text=question),
+                split=TEST_SPLIT,
+                references=[answer_to_reference(text) for text in answers_dict.values()],
+            )
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/bold_scenario.py

@@ -3,6 +3,7 @@ import os
 import random
 from typing import List, Dict, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     DEFAULT_TEST_SIZE,
     Input,
+    ScenarioMetadata,
 )
 
 TOXIC_TAG = "toxic"
@@ -118,3 +120,16 @@ class BOLDScenario(Scenario):
             instances.append(Instance(Input(text=f"{prompt_text} "), split=curr_split, references=[]))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="bold",
+            display_name="BOLD (Bias in Open-Ended Language Generation Dataset)",
+            short_display_name="BOLD",
+            description="The Bias in Open-Ended Language Generation Dataset (BOLD) for measuring biases "
+            "and toxicity in open-ended language generation [(Dhamala et al., "
+            "2021)](https://dl.acm.org/doi/10.1145/3442188.3445924).",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="unknown",
+            main_split="test",
+        )