Skip to content

Instantly share code, notes, and snippets.

@alexeygrigorev

alexeygrigorev/judge.py

Last active March 13, 2026 14:44
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save alexeygrigorev/a4448dca73cd577a7e034e20af007e76 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save alexeygrigorev/a4448dca73cd577a7e034e20af007e76 to your computer and use it in GitHub Desktop.
Download ZIP
Week 6 Homework: Recipe Assistant starter code
Raw
judge.py
import json
from pydantic import BaseModel
from typing import Literal
from pydantic_ai import Agent
class JudgeEvaluation(BaseModel):
reasoning: str
label: Literal["good", "bad"]
judge_instructions = """You are an expert evaluator assessing a recipe assistant that answers
cooking questions using a fixed recipe collection.
A response is "good" if:
1. It accurately answers using ONLY information from the recipe collection
2. It correctly identifies when a recipe is not available and says so
3. It correctly identifies out-of-scope questions (not about cooking/recipes) and declines
A response is "bad" if ANY of these apply:
1. It makes up recipes, ingredients, or instructions not in the collection (hallucination)
2. It provides cooking advice (substitutions, nutrition info, storage tips) that it
invented rather than found in the recipe data
3. It answers a question it should have declined (not about recipes, or recipe not available)
4. It gives incorrect information from the recipes (wrong times, wrong ingredients, wrong steps)
5. It says it cannot help when a matching recipe exists in the collection
Be strict about hallucination. The assistant should only provide information it found
in the recipe data. General cooking knowledge that goes beyond the recipe text counts
as hallucination.
"""
judge_agent = Agent(
'openai:gpt-4o-mini',
output_type=JudgeEvaluation,
instructions=judge_instructions,
)
with open('results.json') as f:
results = json.load(f)
for i, row in enumerate(results):
prompt = f"""Question: {row['question']}
Agent response: {row['output']}"""
evaluation = judge_agent.run_sync(prompt)
row['judge_label'] = evaluation.output.label
row['judge_reasoning'] = evaluation.output.reasoning
print(f"[{i+1}/{len(results)}] {row['judge_label']}: {row['question']}")
with open('results_judged.json', 'w') as f:
json.dump(results, f, indent=2)
print(f"\nSaved judged results to results_judged.json")
Raw
recipe_agent.py
from pydantic_ai import Agent
from recipe_tools import search_recipes, get_recipe
instructions = """You are a recipe assistant. You help users find recipes and answer cooking questions.
1. Use search_recipes to find recipes matching the user's request
2. Use get_recipe to get full details including instructions
3. Answer based on the recipe data you have - do not make up recipes or ingredients
4. If asked about something not in the recipe collection, say you don't have that recipe
5. You can suggest alternatives from the collection if you don't have an exact match
"""
agent = Agent(
'openai:gpt-4o-mini',
tools=[search_recipes, get_recipe],
instructions=instructions,
)
Raw
recipe_tools.py
import json
from minsearch import Index
with open('recipes.json') as f:
RECIPES = json.load(f)
index = Index(
text_fields=["name", "cuisine", "instructions", "tags_text", "ingredients_text"],
keyword_fields=["id", "difficulty", "cuisine"]
)
documents = []
for r in RECIPES:
doc = dict(r)
doc['tags_text'] = ', '.join(r['tags'])
doc['ingredients_text'] = ', '.join(r['ingredients'])
documents.append(doc)
index.fit(documents)
def search_recipes(query: str, cuisine: str = None) -> str:
"""Search for recipes by name, ingredient, cuisine, or description.
Args:
query: What to search for (e.g. 'chicken', 'easy dessert', 'Italian')
cuisine: Optional filter by cuisine (e.g. 'Italian', 'Thai', 'Mexican')
"""
filter_dict = {}
if cuisine:
filter_dict['cuisine'] = cuisine
results = index.search(query, filter_dict=filter_dict, num_results=3)
if not results:
return "No recipes found matching your search."
lines = []
for r in results:
lines.append(f"[{r['id']}] {r['name']} ({r['cuisine']}, {r['difficulty']})")
lines.append(f" Prep: {r['prep_time']}min, Cook: {r['cook_time']}min, Serves: {r['servings']}")
lines.append(f" Ingredients: {r['ingredients_text']}")
lines.append(f" Tags: {r['tags_text']}")
lines.append("")
return "\n".join(lines)
def get_recipe(recipe_id: int) -> str:
"""Get full recipe details including instructions.
Args:
recipe_id: The recipe ID number
"""
for r in RECIPES:
if r['id'] == recipe_id:
ingredients = '\n'.join('- ' + i for i in r['ingredients'])
return f"""Recipe: {r['name']}
Cuisine: {r['cuisine']}
Difficulty: {r['difficulty']}
Prep time: {r['prep_time']} minutes
Cook time: {r['cook_time']} minutes
Servings: {r['servings']}
Ingredients:
{ingredients}
Instructions:
{r['instructions']}
Tags: {', '.join(r['tags'])}"""
return f"Recipe with ID {recipe_id} not found."
Raw
recipes.json
[
{
"id": 1,
"name": "Classic Margherita Pizza",
"cuisine": "Italian",
"difficulty": "easy",
"prep_time": 20,
"cook_time": 15,
"servings": 4,
"ingredients": [
"pizza dough", "San Marzano tomatoes", "fresh mozzarella",
"fresh basil", "olive oil", "salt"
],
"instructions": "Preheat oven to 475°F. Roll out dough on a floured surface. Spread crushed tomatoes evenly. Tear mozzarella into pieces and distribute over sauce. Bake for 12-15 minutes until crust is golden. Top with fresh basil and a drizzle of olive oil.",
"tags": ["vegetarian", "classic"]
},
{
"id": 2,
"name": "Chicken Tikka Masala",
"cuisine": "Indian",
"difficulty": "medium",
"prep_time": 30,
"cook_time": 40,
"servings": 4,
"ingredients": [
"chicken breast", "yogurt", "garam masala", "turmeric",
"cumin", "tomato sauce", "heavy cream", "garlic",
"ginger", "onion", "cilantro", "rice"
],
"instructions": "Marinate chicken in yogurt, garam masala, turmeric, and cumin for at least 1 hour. Grill or broil chicken until charred. In a pan, sauté onion, garlic, and ginger. Add tomato sauce and spices, simmer 15 minutes. Add cream and cooked chicken. Simmer 10 more minutes. Serve over rice with cilantro.",
"tags": ["protein-rich", "spicy"]
},
{
"id": 3,
"name": "Caesar Salad",
"cuisine": "American",
"difficulty": "easy",
"prep_time": 15,
"cook_time": 0,
"servings": 2,
"ingredients": [
"romaine lettuce", "parmesan cheese", "croutons",
"Caesar dressing", "lemon juice", "anchovy paste",
"garlic", "olive oil", "egg yolk"
],
"instructions": "Wash and chop romaine lettuce. Make dressing: whisk egg yolk, anchovy paste, garlic, lemon juice, and olive oil. Toss lettuce with dressing. Top with shaved parmesan and croutons.",
"tags": ["salad", "quick"]
},
{
"id": 4,
"name": "Beef Tacos",
"cuisine": "Mexican",
"difficulty": "easy",
"prep_time": 15,
"cook_time": 20,
"servings": 4,
"ingredients": [
"ground beef", "taco shells", "lettuce", "tomato",
"cheddar cheese", "sour cream", "taco seasoning",
"onion", "salsa"
],
"instructions": "Brown ground beef with diced onion. Add taco seasoning and water, simmer until thickened. Warm taco shells. Serve beef in shells topped with lettuce, tomato, cheese, sour cream, and salsa.",
"tags": ["quick", "family-friendly"]
},
{
"id": 5,
"name": "Vegetable Stir Fry",
"cuisine": "Chinese",
"difficulty": "easy",
"prep_time": 15,
"cook_time": 10,
"servings": 3,
"ingredients": [
"broccoli", "bell pepper", "carrot", "snap peas",
"soy sauce", "sesame oil", "garlic", "ginger",
"cornstarch", "rice"
],
"instructions": "Cut all vegetables into bite-sized pieces. Heat sesame oil in a wok over high heat. Add garlic and ginger, stir 30 seconds. Add vegetables starting with carrots and broccoli, then softer vegetables. Stir in soy sauce and cornstarch slurry. Cook until sauce thickens. Serve over rice.",
"tags": ["vegetarian", "vegan", "quick", "healthy"]
},
{
"id": 6,
"name": "Spaghetti Carbonara",
"cuisine": "Italian",
"difficulty": "medium",
"prep_time": 10,
"cook_time": 20,
"servings": 4,
"ingredients": [
"spaghetti", "guanciale", "egg yolks", "pecorino romano",
"black pepper", "salt"
],
"instructions": "Cook spaghetti in salted water. Crisp guanciale in a pan. Whisk egg yolks with grated pecorino and black pepper. Drain pasta, reserving some water. Toss hot pasta with guanciale, then quickly stir in egg mixture off heat. Add pasta water as needed for a creamy sauce. Serve immediately with extra pecorino.",
"tags": ["classic", "pasta"]
},
{
"id": 7,
"name": "Thai Green Curry",
"cuisine": "Thai",
"difficulty": "medium",
"prep_time": 20,
"cook_time": 25,
"servings": 4,
"ingredients": [
"chicken thigh", "coconut milk", "green curry paste",
"bamboo shoots", "Thai basil", "fish sauce",
"palm sugar", "eggplant", "bell pepper", "jasmine rice"
],
"instructions": "Heat coconut cream in a wok until oil separates. Fry curry paste 2 minutes. Add sliced chicken, cook until sealed. Pour in coconut milk, bring to simmer. Add eggplant and bamboo shoots, cook 10 minutes. Season with fish sauce and palm sugar. Add bell pepper and Thai basil. Serve over jasmine rice.",
"tags": ["spicy", "coconut"]
},
{
"id": 8,
"name": "Greek Salad",
"cuisine": "Greek",
"difficulty": "easy",
"prep_time": 10,
"cook_time": 0,
"servings": 4,
"ingredients": [
"cucumber", "tomato", "red onion", "Kalamata olives",
"feta cheese", "olive oil", "oregano", "red wine vinegar"
],
"instructions": "Chop cucumber, tomatoes, and red onion into chunks. Combine in a bowl. Add Kalamata olives and crumbled feta. Dress with olive oil, red wine vinegar, and oregano. Toss gently. Let sit 5 minutes before serving.",
"tags": ["vegetarian", "salad", "no-cook", "healthy"]
},
{
"id": 9,
"name": "Chocolate Lava Cake",
"cuisine": "French",
"difficulty": "hard",
"prep_time": 20,
"cook_time": 14,
"servings": 4,
"ingredients": [
"dark chocolate", "butter", "eggs", "sugar",
"flour", "vanilla extract", "powdered sugar"
],
"instructions": "Melt chocolate and butter together. Whisk eggs and sugar until thick. Fold chocolate mixture into eggs. Add flour and vanilla, mix gently. Pour into greased ramekins. Bake at 425°F for exactly 12-14 minutes - edges should be firm but center soft. Invert onto plates and dust with powdered sugar. Serve immediately.",
"tags": ["dessert", "chocolate"]
},
{
"id": 10,
"name": "Miso Soup",
"cuisine": "Japanese",
"difficulty": "easy",
"prep_time": 5,
"cook_time": 10,
"servings": 4,
"ingredients": [
"dashi stock", "white miso paste", "silken tofu",
"wakame seaweed", "green onion"
],
"instructions": "Bring dashi stock to a gentle simmer. Add cubed tofu and rehydrated wakame. Ladle some broth into a bowl, dissolve miso paste in it, then pour back into pot. Do not boil after adding miso. Serve topped with sliced green onion.",
"tags": ["soup", "quick", "healthy", "vegan-adaptable"]
},
{
"id": 11,
"name": "Banana Pancakes",
"cuisine": "American",
"difficulty": "easy",
"prep_time": 10,
"cook_time": 15,
"servings": 2,
"ingredients": [
"ripe bananas", "eggs", "flour", "baking powder",
"milk", "butter", "maple syrup", "vanilla extract"
],
"instructions": "Mash bananas in a bowl. Whisk in eggs, milk, and vanilla. Mix flour and baking powder, add to wet ingredients. Stir until just combined - lumps are fine. Heat butter in a pan over medium heat. Pour batter to form pancakes. Cook until bubbles form on surface, flip. Serve with maple syrup.",
"tags": ["breakfast", "sweet", "family-friendly"]
},
{
"id": 12,
"name": "Lamb Shawarma Bowl",
"cuisine": "Middle Eastern",
"difficulty": "medium",
"prep_time": 25,
"cook_time": 20,
"servings": 4,
"ingredients": [
"lamb leg", "cumin", "coriander", "paprika",
"turmeric", "garlic", "lemon juice", "tahini",
"pita bread", "pickled turnips", "hummus",
"tomato", "cucumber", "lettuce"
],
"instructions": "Slice lamb thinly, marinate with cumin, coriander, paprika, turmeric, garlic, and lemon juice for at least 30 minutes. Cook lamb in a hot skillet until browned and cooked through. Make tahini sauce: mix tahini, lemon juice, garlic, and water. Assemble bowls with rice or pita, lamb, hummus, vegetables, and pickled turnips. Drizzle with tahini sauce.",
"tags": ["protein-rich", "bowl"]
},
{
"id": 13,
"name": "Mushroom Risotto",
"cuisine": "Italian",
"difficulty": "medium",
"prep_time": 10,
"cook_time": 35,
"servings": 4,
"ingredients": [
"arborio rice", "mixed mushrooms", "onion", "garlic",
"white wine", "vegetable broth", "parmesan cheese",
"butter", "olive oil", "thyme"
],
"instructions": "Sauté mushrooms in olive oil until golden, set aside. In the same pan, cook onion and garlic in butter. Add arborio rice, toast 2 minutes. Pour in white wine, stir until absorbed. Add warm broth one ladle at a time, stirring frequently, until rice is creamy and al dente (about 18-20 minutes). Stir in mushrooms, parmesan, and fresh thyme.",
"tags": ["vegetarian", "comfort-food"]
},
{
"id": 14,
"name": "Fish and Chips",
"cuisine": "British",
"difficulty": "medium",
"prep_time": 20,
"cook_time": 25,
"servings": 4,
"ingredients": [
"cod fillets", "flour", "beer", "baking powder",
"potatoes", "vegetable oil", "salt", "malt vinegar",
"tartar sauce", "lemon", "peas"
],
"instructions": "Cut potatoes into thick chips, soak in cold water 30 minutes. Pat dry and fry at 325°F for 5 minutes, drain. Make batter: whisk flour, baking powder, and beer until smooth. Heat oil to 375°F. Dip fish in flour, then batter. Fry fish 5-7 minutes until golden. Re-fry chips at 375°F until crispy. Serve with malt vinegar, tartar sauce, and lemon.",
"tags": ["fried", "classic", "seafood"]
},
{
"id": 15,
"name": "Falafel Wrap",
"cuisine": "Middle Eastern",
"difficulty": "medium",
"prep_time": 30,
"cook_time": 15,
"servings": 4,
"ingredients": [
"chickpeas", "onion", "garlic", "parsley", "cilantro",
"cumin", "coriander", "flour", "baking powder",
"pita bread", "tahini", "lettuce", "tomato", "pickles"
],
"instructions": "Soak dried chickpeas overnight (do not use canned). Blend chickpeas with onion, garlic, herbs, and spices until coarse. Add flour and baking powder, mix. Form into balls or patties. Deep fry at 350°F for 3-4 minutes until dark golden. Serve in pita with tahini sauce, lettuce, tomato, and pickles.",
"tags": ["vegetarian", "vegan", "street-food"]
}
]
Raw
run_scenarios.py
import csv
import json
import time
from recipe_agent import agent
MODEL_PRICES = {
"gpt-4o-mini": {"input": 0.15, "output": 0.60},
}
def calculate_cost(usage, model="gpt-4o-mini"):
prices = MODEL_PRICES[model]
input_cost = (usage.input_tokens / 1_000_000) * prices["input"]
output_cost = (usage.output_tokens / 1_000_000) * prices["output"]
return round(input_cost + output_cost, 6)
def run_all():
with open('scenarios.csv') as f:
scenarios = list(csv.DictReader(f))
results = []
for i, scenario in enumerate(scenarios):
question = scenario['question']
print(f"[{i+1}/{len(scenarios)}] {question}")
start = time.time()
result = agent.run_sync(question)
elapsed = time.time() - start
usage = result.usage()
cost = calculate_cost(usage)
results.append({
'question': question,
'category': scenario['category'],
'type': scenario['type'],
'output': result.output,
'execution_time': round(elapsed, 2),
'tokens': {
'input_tokens': usage.input_tokens,
'output_tokens': usage.output_tokens,
'total_tokens': usage.total_tokens,
},
'cost': cost,
})
print(f" Done in {elapsed:.1f}s (${cost})")
with open('results.json', 'w') as f:
json.dump(results, f, indent=2)
total_cost = sum(r['cost'] for r in results)
print(f"\nSaved {len(results)} results to results.json")
print(f"Total cost: ${total_cost:.4f}")
if __name__ == "__main__":
run_all()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment