Skip to content

Instantly share code, notes, and snippets.

@davidefiocco

davidefiocco/factchecking_agent.ipynb

Created July 27, 2025 21:10
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save davidefiocco/525478bfd90f07a600bced04ea78602c to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save davidefiocco/525478bfd90f07a600bced04ea78602c to your computer and use it in GitHub Desktop.
Download ZIP
Verify statements extracted from a snippet of text comparing them with web sources
Display the source blob
Display the rendered blob
Raw
factchecking_agent.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/davide.fiocco/Projects/smolagent2/.venv/lib/python3.13/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"from smolagents import OpenAIServerModel#, InferenceClientModel\n",
"from smolagents import CodeAgent, VisitWebpageTool, FinalAnswerTool\n",
"from smolagents.tools import tool, Tool\n",
"from exa_py import Exa\n",
"from datetime import datetime\n",
"from typing import List\n",
"import os\n",
"import json"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"env: OPENAI_API_KEY=<add-key-here>\n"
]
}
],
"source": [
"%env OPENAI_API_KEY=<add-key-here>"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"env: EXA_API_KEY=<add-key-here>\n"
]
}
],
"source": [
"%env EXA_API_KEY=<add-key-here>\n",
"\n",
"EXA_API_KEY = os.getenv(\"EXA_API_KEY\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"model = OpenAIServerModel(model_id=\"gpt-4.1\")\n",
"#model = InferenceClientModel(model=\"Qwen/Qwen2.5-Coder-32B-Instruct\")\n",
"#model = InferenceClientModel(model=\"google/gemma-3-27b-it\")"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"class ExaSearchTool(Tool):\n",
" name = \"exa_search\"\n",
" description = \"Search the web for the given query using Exa with date filtering and text extraction capabilities.\"\n",
" inputs = {\n",
" \"query\": {\n",
" \"type\": \"string\", \n",
" \"description\": \"The search query to perform.\"\n",
" },\n",
" \"start_published_date\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The start date of the search (YYYY-MM-DD format).\"\n",
" },\n",
" \"end_published_date\": {\n",
" \"type\": \"string\", \n",
" \"description\": \"The end date of the search (YYYY-MM-DD format).\"\n",
" },\n",
" \"extract_text\": {\n",
" \"type\": \"boolean\",\n",
" \"description\": \"Whether to extract text from the search results pages.\",\n",
" \"nullable\": True\n",
" }\n",
" }\n",
" output_type = \"string\"\n",
"\n",
" def __init__(self, api_key: str = None, **kwargs):\n",
" super().__init__(**kwargs)\n",
" self.api_key = api_key\n",
" if self.api_key is None:\n",
" raise ValueError(\"Missing API key. Make sure you have 'EXA_API_KEY' in your env variables or pass it as api_key parameter.\")\n",
" \n",
" try:\n",
" from exa_py import Exa\n",
" except ImportError as e:\n",
" raise ImportError(\n",
" \"You must install `exa_py` to run this tool: for instance run `pip install exa_py`.\"\n",
" ) from e\n",
" \n",
" self.exa = Exa(api_key=self.api_key)\n",
"\n",
" def forward(self, query: str, start_published_date: str, end_published_date: str, extract_text: bool = True) -> str:\n",
" \"\"\"Search the web using Exa API with date filtering and text extraction.\n",
" \n",
" Args:\n",
" query (str): The search query.\n",
" start_published_date (str): The start date of the search.\n",
" end_published_date (str): The end date of the search.\n",
" extract_text (bool): Whether to extract text from the search results pages.\n",
" \n",
" Returns:\n",
" str: The formatted search results.\n",
" \"\"\"\n",
" try:\n",
" result = self.exa.search_and_contents(\n",
" query,\n",
" text=extract_text,\n",
" type=\"auto\",\n",
" start_published_date=start_published_date,\n",
" end_published_date=end_published_date\n",
" )\n",
" \n",
" # Format the results for better readability\n",
" if hasattr(result, 'results') and result.results:\n",
" formatted_results = []\n",
" for idx, item in enumerate(result.results, 1):\n",
" title = getattr(item, 'title', 'No title')\n",
" url = getattr(item, 'url', 'No URL')\n",
" text_content = getattr(item, 'text', 'No content') if extract_text else 'Text extraction disabled'\n",
" \n",
" formatted_result = f\"{idx}. [{title}]({url})\\n{text_content}\"\n",
" formatted_results.append(formatted_result)\n",
" \n",
" return \"## Exa Search Results\\n\\n\" + \"\\n\\n\".join(formatted_results)\n",
" else:\n",
" return f\"No results found for query: '{query}' between {start_published_date} and {end_published_date}.\"\n",
" \n",
" except Exception as e:\n",
" return f\"Error performing Exa search: {str(e)}\" "
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"class BreakdownIntoStatementsTool(Tool):\n",
" name = \"breakdown_into_statements\"\n",
" description = \"Break down the given text into a list of statements, recording the appropriate context to make them easier to fact-check.\"\n",
" inputs = {\n",
" \"text\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The text to break down into statements that are to be fact-checked.\"\n",
" }\n",
" }\n",
" output_type = \"string\"\n",
"\n",
" def __init__(self, **kwargs):\n",
" super().__init__(**kwargs)\n",
"\n",
" def forward(self, text: str) -> str:\n",
" \"\"\"\n",
" Break down the given text into a list of statements that will be fact-checked.\n",
" Each statement should be a single fact with sufficient context (also from the rest of the texts) that can be fact-checked independently.\n",
" Add a CONTEXT section that can be used to contextualize the statements.\n",
"\n",
" Args:\n",
" text (str): The text to break down into statements.\n",
"\n",
" Returns:\n",
" str: A string with the comma-separated list of statements that will be fact-checked and a CONTEXT section that was used to make them.\n",
" \"\"\"\n",
" \n",
" messages = [\n",
" {\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": f\"Break down the following text into a comma-separated list of statements that will be fact-checked. Add a CONTEXT section that can be used to contextualize the statements: {text}\"}]},\n",
" ]\n",
" response = model.generate(messages)\n",
" return response.content\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"exa_search_tool = ExaSearchTool(api_key=EXA_API_KEY)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"statement_checker_agent = CodeAgent(\n",
" tools=[\n",
" exa_search_tool,\n",
" VisitWebpageTool(),\n",
" FinalAnswerTool()\n",
" ], \n",
" model=model,\n",
" max_steps=25,\n",
" verbosity_level=0,\n",
" name=\"statement_checker_agent\",\n",
" description=\"\"\"\n",
" This agent is responsible for fact-checking an input statement with its context.\n",
" It uses the Exa search tool to search for information on the web and the VisitWebpageTool to visit the webpage and extract text from sources.\n",
" It then uses the FinalAnswerTool to output the fact-checked statement as a JSON object with the following fields:\n",
" - statement: the statement to be fact-checked\n",
" - is_true: whether the statement is true or false\n",
" - revised_statement: the revised statement if the statement is false\n",
" - source: the source of the information\n",
" Example:\n",
" {{\n",
" 'statement': 'Queen Elizabeth II was born on April 21, 1929, in London, England.',\n",
" 'is_true': false, # can be true, false or unsure\n",
" 'revised_statement': 'Queen Elizabeth II was born on April 21, 1926, in London, England.' # or None if the statement is true\n",
" 'source': 'https://en.wikipedia.org/wiki/Queen_Elizabeth_II',\n",
" }}\n",
" \"\"\"\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"agent = CodeAgent(\n",
" tools=[\n",
" BreakdownIntoStatementsTool(),\n",
" FinalAnswerTool()\n",
" ], \n",
" managed_agents=[statement_checker_agent],\n",
" model=model,\n",
" max_steps=25,\n",
" verbosity_level=0,\n",
" name=\"fact_checker_agent\",\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"input_text = \"\"\"\n",
"Queen Elizabeth II was born on April 21, 1929, in London, England.\n",
"She is the mother of Princess Diana.\n",
"\"\"\""
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"input_text_2 = \"\"\"\n",
"From a New York Times article 27/07/2025:\n",
"The deal sets a 15 percent tariff on European goods imported to the U.S.\n",
"The European Union and the United States agreed on Sunday to a broad-brush trade deal that sets a 15 percent tariff on most E.U. goods, including cars, averting what could have become a painful trade war with a bloc that is the United States’ single biggest source of imports.\n",
"President Trump said that the European Union had agreed to purchase $750 billion of American energy, which Ursula von der Leyen, the president of the E.U.’s executive branch, told reporters would be spread out over three years.\n",
"\"\"\""
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"prompt = f\"\"\"Break down the text into statements that are to be fact-checked.\n",
"Use the statement_checker_agent to fact-check each statement that can be extracted from it.\n",
"Here is the text to break down into statements: {input_text}\n",
"Time now is {datetime.now()}\"\"\"\n",
"\n",
"prompt_2 = f\"\"\"Break down the text into statements that are to be fact-checked.\n",
"Use the statement_checker_agent to fact-check each statement that can be extracted from it.\n",
"Here is the text to break down into statements: {input_text_2}\n",
"Time now is {datetime.now()}\"\"\""
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"result = agent.run(prompt)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'statement': 'Queen Elizabeth II was born on April 21, 1929.',\n",
" 'is_true': False,\n",
" 'revised_statement': 'Queen Elizabeth II was born on April 21, 1926.',\n",
" 'source': 'https://www.royal.uk/the-queen'},\n",
" {'statement': 'Queen Elizabeth II was born in London, England.',\n",
" 'is_true': True,\n",
" 'revised_statement': None,\n",
" 'source': 'https://www.rct.uk/collection/themes/Trails/queen-elizabeth-ii/childhood-of-her-majesty-the-queen'},\n",
" {'statement': 'Queen Elizabeth II is the mother of Princess Diana.',\n",
" 'is_true': False,\n",
" 'revised_statement': \"Queen Elizabeth II is the mother-in-law (not mother) of Princess Diana. Princess Diana's parents are John Spencer, 8th Earl Spencer, and Frances Shand Kydd.\",\n",
" 'source': 'https://en.wikipedia.org/wiki/Diana,_Princess_of_Wales'}]"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"result"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"result_2 = agent.run(prompt_2)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'statement': 'The deal sets a 15 percent tariff on European goods imported to the U.S.',\n",
" 'is_true': 'broadly true, but important exceptions exist (some categories exempt, not literally all goods); statement is accurate but imprecise.',\n",
" 'source': 'NBC News, National Post, Australian Financial Review, etc.'},\n",
" {'statement': 'The European Union and the United States agreed on Sunday to a broad-brush trade deal that sets a 15 percent tariff on most E.U. goods, including cars',\n",
" 'is_true': 'true',\n",
" 'source': 'NBC News, National Post, Reuters, Bloomberg'},\n",
" {'statement': 'The trade deal averted what could have become a painful trade war between the United States and the European Union',\n",
" 'is_true': 'no evidence or reporting found for this event or deal as of July 27, 2025; cannot be verified.',\n",
" 'source': 'No corroborating news sources or NYT article found.'},\n",
" {'statement': 'The European Union is the United States’ single biggest source of imports',\n",
" 'is_true': 'false (Mexico is the top source as of 2024-2025, E.U. is not)',\n",
" 'source': 'U.S. Census Bureau, Statista, Investopedia, OEC, Congressional Research Service'},\n",
" {'statement': 'President Trump said that the European Union had agreed to purchase $750 billion of American energy',\n",
" 'is_true': \"true as to what Trump publicly claimed (multiple sources quote this remark), but as a statement of fact/agreement, this is Trump's statement, not confirmed as an official contract\",\n",
" 'source': 'CBS News, NBC News'},\n",
" {'statement': 'Ursula von der Leyen, the president of the E.U.’s executive branch, told reporters the $750 billion in American energy purchases would be spread out over three years',\n",
" 'is_true': 'unable to verify; no credible reporting or direct quotes from von der Leyen confirming this claim or the three-year timeline',\n",
" 'source': 'Reuters, Bloomberg, Irish Times, CBS, New York Times (archive searches)'}]"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"result_2"
]
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.13.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment