Skip to content

Instantly share code, notes, and snippets.

@shiyuangu

shiyuangu/trulens_internal_record

Last active April 7, 2024 22:26
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save shiyuangu/d6979f089c67ca03b6ae947fcea879f3 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save shiyuangu/d6979f089c67ca03b6ae947fcea879f3 to your computer and use it in GitHub Desktop.
Download ZIP
trulens internal records
Display the source blob
Display the rendered blob
Raw
trulens_source.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Where are the recorder result "
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"🦑 Tru initialized with db url sqlite:///with-vtt.sqlit .\n",
"🛑 Secret keys may be written to the database. See the `database_redact_keys` option of Tru` to prevent this.\n"
]
}
],
"source": [
"# How the records are saved \n",
"# Cf: https://github.com/truera/trulens/blob/main/trulens_eval/trulens_eval/Leaderboard.py\n",
"from trulens_eval import Tru\n",
"database_url = \"sqlite:///with-vtt.sqlit\"\n",
"tru = Tru(database_url=database_url)\n",
"lms = tru.db\n",
"df, feedback_col_names = lms.get_records_and_feedback([])\n",
"feedback_defs = lms.get_feedback_defs()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"app_id sentence window engine 1\n",
"app_json {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...\n",
"type RetrieverQueryEngine(llama_index.core.query_en...\n",
"record_id record_hash_6d098233496bf3b5d8af439d2e02f8d0\n",
"input \"How can I slow down a video?\"\n",
"output \"You can slow down a video by using the Speed/...\n",
"tags -\n",
"record_json {\"record_id\": \"record_hash_6d098233496bf3b5d8a...\n",
"cost_json {\"n_requests\": 1, \"n_successful_requests\": 1, ...\n",
"perf_json {\"start_time\": \"2024-03-25T14:01:15.560753\", \"...\n",
"ts 2024-03-25T14:01:22.914426\n",
"Answer Relevance 1.0\n",
"Groundedness 0.15\n",
"Context Relevance 0.5\n",
"Answer Relevance_calls [{'args': {'prompt': 'How can I slow down a vi...\n",
"Groundedness_calls [{'args': {'source': 'To select just the video...\n",
"Context Relevance_calls [{'args': {'prompt': 'How can I slow down a vi...\n",
"latency 7\n",
"total_tokens 381\n",
"total_cost 0.000591\n"
]
}
],
"source": [
"print(df.iloc[0].to_string())"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'args': {'source': 'To select just the video or audio part of a linked clip, right-click the clip and choose \\nUnlink\\n.\\n You can turn video and audio linking off or on for the whole sequence by clicking the \\nLinked Selection\\n button at the top left of the Timeline panel.\\n Use the \\nRazor tool\\n to divide a clip into two pieces you can move separately.\\n',\n",
" 'statement': 'You can slow down a video by using the Speed/Duration option in the video editing software. This allows you to adjust the speed of the video to make it play at a slower pace.'},\n",
" 'ret': 0.0,\n",
" 'meta': {'reasons': 'TEMPLATE: \\nStatement Sentence: You can slow down a video by using the Speed/Duration option in the video editing software, \\nSupporting Evidence: NOTHING FOUND\\nScore: 0\\n\\nTEMPLATE: \\nStatement Sentence: This allows you to adjust the speed of the video to make it play at a slower pace., \\nSupporting Evidence: NOTHING FOUND\\nScore: 0\\n\\nGive me the INFORMATION OVERLAP of this SOURCE and STATEMENT.'}},\n",
" {'args': {'source': 'If you added In and Out marks to a clip, only the selected portion of the clip will be added to your sequence.\\n Use the \\nDrag Video Only\\n or \\nDrag Audio Only\\n icon at the bottom of the Source Monitor to drag only the video or audio into your sequence.\\n 3\\nRemove clips from a sequence\\nSome video editors prefer to add everything to a sequence and then remove the clips they don’t want, gradually reducing the content until only the best material remains.\\n',\n",
" 'statement': 'You can slow down a video by using the Speed/Duration option in the video editing software. This allows you to adjust the speed of the video to make it play at a slower pace.'},\n",
" 'ret': 0.15,\n",
" 'meta': {'reasons': 'TEMPLATE: \\nStatement Sentence: You can slow down a video by using the Speed/Duration option in the video editing software., \\nSupporting Evidence: NOTHING FOUND\\nScore: 0\\n\\nTEMPLATE: \\nStatement Sentence: This allows you to adjust the speed of the video to make it play at a slower pace., \\nSupporting Evidence: If you added In and Out marks to a clip, only the selected portion of the clip will be added to your sequence.\\nScore: 3'}}]"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# sgu: these values is displayed at https://github.com/truera/trulens/blob/7974d12c13847a02577f8d98c2339f2725c8f03c/trulens_eval/trulens_eval/pages/Evaluations.py#L377\n",
"d = df.iloc[0].to_dict()\n",
"d['Groundedness_calls']"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'{\"tru_class_info\": {\"name\": \"TruLlama\", \"module\": {\"package_name\": \"trulens_eval\", \"module_name\": \"trulens_eval.tru_llama\"}, \"bases\": [{\"name\": \"TruLlama\", \"module\": {\"package_name\": \"trulens_eval\", \"module_name\": \"trulens_eval.tru_llama\"}, \"bases\": null}, {\"name\": \"App\", \"module\": {\"package_name\": \"trulens_eval\", \"module_name\": \"trulens_eval.app\"}, \"bases\": null}, {\"name\": \"AppDefinition\", \"module\": {\"package_name\": \"trulens_eval\", \"module_name\": \"trulens_eval.schema\"}, \"bases\": null}, {\"name\": \"WithClassInfo\", \"module\": {\"package_name\": \"trulens_eval.utils\", \"module_name\": \"trulens_eval.utils.pyschema\"}, \"bases\": null}, {\"name\": \"SerialModel\", \"module\": {\"package_name\": \"trulens_eval.utils\", \"module_name\": \"trulens_eval.utils.serial\"}, \"bases\": null}, {\"name\": \"BaseModel\", \"module\": {\"package_name\": \"pydantic\", \"module_name\": \"pydantic.main\"}, \"bases\": null}, {\"name\": \"WithInstrumentCallbacks\", \"module\": {\"package_name\": \"trulens_eval\", \"module_name\": \"trulens_eval.instruments\"}, \"bases\": null}, {\"name\": \"Hashable\", \"module\": {\"package_name\": \"collections\", \"module_name\": \"collections.abc\"}, \"bases\": null}, {\"name\": \"Generic\", \"module\": {\"package_name\": \"\", \"module_name\": \"typing\"}, \"bases\": null}, {\"name\": \"object\", \"module\": {\"package_name\": \"\", \"module_name\": \"builtins\"}, \"bases\": null}]}, \"app_id\": \"sentence window engine 1\", \"tags\": \"-\", \"metadata\": {}, \"feedback_definitions\": [], \"feedback_mode\": \"with_app_thread\", \"root_class\": {\"name\": \"RetrieverQueryEngine\", \"module\": {\"package_name\": \"llama_index.core.query_engine\", \"module_name\": \"llama_index.core.query_engine.retriever_query_engine\"}, \"bases\": null}, \"app\": {\"__tru_non_serialized_object\": {\"cls\": {\"name\": \"RetrieverQueryEngine\", \"module\": {\"package_name\": \"llama_index.core.query_engine\", \"module_name\": \"llama_index.core.query_engine.retriever_query_engine\"}, \"bases\": null}, \"id\": 5981704480, \"init_bindings\": null}}, \"initial_app_loader_dump\": null, \"app_extra_json\": {}, \"selector_check_warning\": false, \"selector_nocheck\": false}'"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"d['app_json']"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['Context Relevance', 'Answer Relevance', 'Groundedness']"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"feedback_col_names"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>feedback_definition_id</th>\n",
" <th>feedback_json</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>feedback_definition_hash_a2e2675280894adc12297...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>feedback_definition_hash_bb4f8ee70e16515ff871c...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>feedback_definition_hash_a221da5600060805a5468...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>feedback_definition_hash_af5c7ab52e7947f2fc692...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>feedback_definition_hash_eb8dba8e796beb1c1389e...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>feedback_definition_hash_b82592238d8f431dad364...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>feedback_definition_hash_39887a4c756c79bdad5cd...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>feedback_definition_hash_c2d5c05859331fa524531...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>feedback_definition_hash_af1572e0fb91d84c2f99e...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>feedback_definition_hash_555ec9e193658eaa6972e...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>feedback_definition_hash_7debaed0cb30f964fb2ce...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>feedback_definition_hash_9f214eda942533733e6dc...</td>\n",
" <td>{'tru_class_info': {'name': 'Feedback', 'modul...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" feedback_definition_id \\\n",
"0 feedback_definition_hash_a2e2675280894adc12297... \n",
"1 feedback_definition_hash_bb4f8ee70e16515ff871c... \n",
"2 feedback_definition_hash_a221da5600060805a5468... \n",
"3 feedback_definition_hash_af5c7ab52e7947f2fc692... \n",
"4 feedback_definition_hash_eb8dba8e796beb1c1389e... \n",
"5 feedback_definition_hash_b82592238d8f431dad364... \n",
"6 feedback_definition_hash_39887a4c756c79bdad5cd... \n",
"7 feedback_definition_hash_c2d5c05859331fa524531... \n",
"8 feedback_definition_hash_af1572e0fb91d84c2f99e... \n",
"9 feedback_definition_hash_555ec9e193658eaa6972e... \n",
"10 feedback_definition_hash_7debaed0cb30f964fb2ce... \n",
"11 feedback_definition_hash_9f214eda942533733e6dc... \n",
"\n",
" feedback_json \n",
"0 {'tru_class_info': {'name': 'Feedback', 'modul... \n",
"1 {'tru_class_info': {'name': 'Feedback', 'modul... \n",
"2 {'tru_class_info': {'name': 'Feedback', 'modul... \n",
"3 {'tru_class_info': {'name': 'Feedback', 'modul... \n",
"4 {'tru_class_info': {'name': 'Feedback', 'modul... \n",
"5 {'tru_class_info': {'name': 'Feedback', 'modul... \n",
"6 {'tru_class_info': {'name': 'Feedback', 'modul... \n",
"7 {'tru_class_info': {'name': 'Feedback', 'modul... \n",
"8 {'tru_class_info': {'name': 'Feedback', 'modul... \n",
"9 {'tru_class_info': {'name': 'Feedback', 'modul... \n",
"10 {'tru_class_info': {'name': 'Feedback', 'modul... \n",
"11 {'tru_class_info': {'name': 'Feedback', 'modul... "
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"feedback_defs"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "pp-assistant-JAdr8icG",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment