Skip to content

Instantly share code, notes, and snippets.

@mattbullen

mattbullen/unit11_model_performance_measurement.ipynb

Created October 16, 2025 12:04
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save mattbullen/5cb790c559a2e5301214fbaec935df6b to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save mattbullen/5cb790c559a2e5301214fbaec935df6b to your computer and use it in GitHub Desktop.
Download ZIP
Unit11_model_Performance_Measurement.ipynb
Display the source blob
Display the rendered blob
Raw
unit11_model_performance_measurement.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/mattbullen/5cb790c559a2e5301214fbaec935df6b/unit11_model_performance_measurement.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "2OF2Tid94xDP"
},
"source": [
"### Model performance measurement\n",
"\n",
"#### Confusion Matrix"
],
"id": "2OF2Tid94xDP"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "XWvqx0Su4xDQ"
},
"outputs": [],
"source": [
"from sklearn.metrics import confusion_matrix"
],
"id": "XWvqx0Su4xDQ"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "YlD4as4Q4xDR"
},
"outputs": [],
"source": [
"tn, fp, fn, tp = confusion_matrix([0, 1, 0, 1], [1, 1, 1, 0]).ravel()\n",
"(tn, fp, fn, tp)"
],
"id": "YlD4as4Q4xDR"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "LOT-yWOQ4xDR"
},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"from sklearn.datasets import make_classification\n",
"from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.svm import SVC\n",
"X, y = make_classification(random_state=0)\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y,\n",
" random_state=0)\n",
"clf = SVC(random_state=0)\n",
"clf.fit(X_train, y_train)\n",
"SVC(random_state=0)\n",
"predictions = clf.predict(X_test)\n",
"cm = confusion_matrix(y_test, predictions, labels=clf.classes_)\n",
"disp = ConfusionMatrixDisplay(confusion_matrix=cm,\n",
" display_labels=clf.classes_)\n",
"disp.plot()\n",
"\n",
"plt.show()\n"
],
"id": "LOT-yWOQ4xDR"
},
{
"cell_type": "markdown",
"metadata": {
"id": "1Un_T1eK4xDR"
},
"source": [
"#### F1, Accuracy, Recall, AUC and Precision scores"
],
"id": "1Un_T1eK4xDR"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "_Km-S2X24xDR"
},
"outputs": [],
"source": [
"from sklearn.metrics import f1_score\n",
"\n",
"y_true = [0, 1, 2, 0, 1, 2]\n",
"y_pred = [0, 2, 1, 0, 0, 1]\n",
"\n",
"print(f\"Macro f1 score: {f1_score(y_true, y_pred, average='macro')}\")\n",
"\n",
"print(f\"Micro F1: {f1_score(y_true, y_pred, average='micro')}\")\n",
"\n",
"print(f\"Weighted Average F1: {f1_score(y_true, y_pred, average='weighted')}\")\n",
"\n",
"print(f\"F1 No Average: {f1_score(y_true, y_pred, average=None)}\")\n",
"\n",
"y_true = [0, 0, 0, 0, 0, 0]\n",
"y_pred = [0, 0, 0, 0, 0, 0]\n",
"f1_score(y_true, y_pred, zero_division=1)\n",
"\n",
"# multilabel classification\n",
"y_true = [[0, 0, 0], [1, 1, 1], [0, 1, 1]]\n",
"y_pred = [[0, 0, 0], [1, 1, 1], [1, 1, 0]]\n",
"print(f\"F1 No Average: {f1_score(y_true, y_pred, average=None)}\")\n"
],
"id": "_Km-S2X24xDR"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "slumDbl04xDR"
},
"outputs": [],
"source": [
" from sklearn.metrics import accuracy_score\n",
"y_pred = [0, 2, 1, 3]\n",
"y_true = [0, 1, 2, 3]\n",
"accuracy_score(y_true, y_pred)"
],
"id": "slumDbl04xDR"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "a-c2FIn04xDS"
},
"outputs": [],
"source": [
"from sklearn.metrics import precision_score\n",
"y_true = [0, 1, 2, 0, 1, 2]\n",
"y_pred = [0, 2, 1, 0, 0, 1]\n",
"precision_score(y_true, y_pred, average='macro')"
],
"id": "a-c2FIn04xDS"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "9J10Jdrt4xDS"
},
"outputs": [],
"source": [
"from sklearn.metrics import recall_score\n",
"y_true = [0, 1, 2, 0, 1, 2]\n",
"y_pred = [0, 2, 1, 0, 0, 1]\n",
"recall_score(y_true, y_pred, average='macro')"
],
"id": "9J10Jdrt4xDS"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "LdlH6LTl4xDS"
},
"outputs": [],
"source": [
"from sklearn.metrics import classification_report\n",
"y_true = [0, 1, 2, 2, 2]\n",
"y_pred = [0, 0, 2, 2, 1]\n",
"target_names = ['class 0', 'class 1', 'class 2']\n",
"print(classification_report(y_true, y_pred, target_names=target_names))"
],
"id": "LdlH6LTl4xDS"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "thM7_1NO4xDS"
},
"outputs": [],
"source": [
"from sklearn.datasets import load_breast_cancer\n",
"from sklearn.linear_model import LogisticRegression\n",
"from sklearn.metrics import roc_auc_score\n",
"X, y = load_breast_cancer(return_X_y=True)\n",
"clf = LogisticRegression(solver=\"liblinear\", random_state=0).fit(X, y)\n",
"roc_auc_score(y, clf.predict_proba(X)[:, 1])\n"
],
"id": "thM7_1NO4xDS"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "FaabCeaF4xDS"
},
"outputs": [],
"source": [
"#multiclass case\n",
"from sklearn.datasets import load_iris\n",
"X, y = load_iris(return_X_y=True)\n",
"clf = LogisticRegression(solver=\"liblinear\").fit(X, y)\n",
"roc_auc_score(y, clf.predict_proba(X), multi_class='ovr')"
],
"id": "FaabCeaF4xDS"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "sUisA-EV4xDT"
},
"outputs": [],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from itertools import cycle\n",
"\n",
"from sklearn import svm, datasets\n",
"from sklearn.metrics import roc_curve, auc\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.preprocessing import label_binarize\n",
"from sklearn.multiclass import OneVsRestClassifier\n",
"from sklearn.metrics import roc_auc_score\n",
"\n",
"# Import some data to play with\n",
"iris = datasets.load_iris()\n",
"X = iris.data\n",
"y = iris.target\n",
"\n",
"# Binarize the output\n",
"y = label_binarize(y, classes=[0, 1, 2])\n",
"n_classes = y.shape[1]\n",
"\n",
"# Add noisy features to make the problem harder\n",
"random_state = np.random.RandomState(0)\n",
"n_samples, n_features = X.shape\n",
"X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]\n",
"\n",
"# shuffle and split training and test sets\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0)\n",
"\n",
"# Learn to predict each class against the other\n",
"classifier = OneVsRestClassifier(\n",
" svm.SVC(kernel=\"linear\", probability=True, random_state=random_state)\n",
")\n",
"y_score = classifier.fit(X_train, y_train).decision_function(X_test)\n",
"\n",
"# Compute ROC curve and ROC area for each class\n",
"fpr = dict()\n",
"tpr = dict()\n",
"roc_auc = dict()\n",
"for i in range(n_classes):\n",
" fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])\n",
" roc_auc[i] = auc(fpr[i], tpr[i])\n",
"\n",
"# Compute micro-average ROC curve and ROC area\n",
"fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel())\n",
"roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])"
],
"id": "sUisA-EV4xDT"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "FRpetmqt4xDT"
},
"outputs": [],
"source": [
"plt.figure()\n",
"lw = 2\n",
"plt.plot(\n",
" fpr[2],\n",
" tpr[2],\n",
" color=\"darkorange\",\n",
" lw=lw,\n",
" label=\"ROC curve (area = %0.2f)\" % roc_auc[2],\n",
")\n",
"plt.plot([0, 1], [0, 1], color=\"navy\", lw=lw, linestyle=\"--\")\n",
"plt.xlim([0.0, 1.0])\n",
"plt.ylim([0.0, 1.05])\n",
"plt.xlabel(\"False Positive Rate\")\n",
"plt.ylabel(\"True Positive Rate\")\n",
"plt.title(\"Receiver operating characteristic example\")\n",
"plt.legend(loc=\"lower right\")\n",
"plt.show()\n"
],
"id": "FRpetmqt4xDT"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "b0bSKk8a4xDT"
},
"outputs": [],
"source": [
"from sklearn.metrics import log_loss\n",
"log_loss([\"spam\", \"ham\", \"ham\", \"spam\"], [[.1, .9], [.9, .1], [.8, .2], [.35, .65]])"
],
"id": "b0bSKk8a4xDT"
},
{
"cell_type": "markdown",
"metadata": {
"id": "mNeLKWzh4xDT"
},
"source": [
"### Regression metrics"
],
"id": "mNeLKWzh4xDT"
},
{
"cell_type": "markdown",
"metadata": {
"id": "uO4FHSLD4xDT"
},
"source": [
"#### RMSE"
],
"id": "uO4FHSLD4xDT"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "pQneEcVD4xDT"
},
"outputs": [],
"source": [
"from sklearn.metrics import mean_squared_error\n",
"y_true = [3, -0.5, 2, 7]\n",
"y_pred = [2.5, 0.0, 2, 8]\n",
"mean_squared_error(y_true, y_pred)"
],
"id": "pQneEcVD4xDT"
},
{
"cell_type": "markdown",
"metadata": {
"id": "wL05PMOu4xDT"
},
"source": [
"#### MAE"
],
"id": "wL05PMOu4xDT"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "iynsSxy_4xDT"
},
"outputs": [],
"source": [
"from sklearn.metrics import mean_absolute_error\n",
"y_true = [3, -0.5, 2, 7]\n",
"y_pred = [2.5, 0.0, 2, 8]\n",
"mean_absolute_error(y_true, y_pred)"
],
"id": "iynsSxy_4xDT"
},
{
"cell_type": "markdown",
"metadata": {
"id": "83gv6ejI4xDT"
},
"source": [
"#### r squared"
],
"id": "83gv6ejI4xDT"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "YRyrz89X4xDU"
},
"outputs": [],
"source": [
"from sklearn.metrics import r2_score\n",
"\n",
"r2_score(y_true, y_pred)"
],
"id": "YRyrz89X4xDU"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "irMmtgFH4xDU"
},
"outputs": [],
"source": [],
"id": "irMmtgFH4xDU"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "m1ACCmZf4xDU"
},
"outputs": [],
"source": [],
"id": "m1ACCmZf4xDU"
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
},
"colab": {
"provenance": [],
"include_colab_link": true
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment