Skip to content

Instantly share code, notes, and snippets.

@botcs

botcs/coco-map-2GT-3PRED.ipynb

Last active December 23, 2019 01:05
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save botcs/b4d929f90860f32c2de6470345884a27 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save botcs/b4d929f90860f32c2de6470345884a27 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
coco-map-2GT-3PRED.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import numpy as np\n",
"from pycocotools.coco import COCO\n",
"from pycocotools.cocoeval import COCOeval\n",
"from copy import deepcopy\n",
"\n",
"from matplotlib import pyplot as plt\n",
"\n",
"orig_gt_path = \"/home/botoscs/data/MS-COCO/annotations/instances_minival2014.json\"\n",
"gt = json.load(open(orig_gt_path))\n",
"\n",
"# Overwrite annotations with toy examples\n",
"gt[\"annotations\"] = [\n",
" {'area': 1,\n",
" 'iscrowd': 0,\n",
" 'image_id': 289343,\n",
" 'bbox': [0, 0, 1, 1],\n",
" 'category_id': 18,\n",
" 'id': 3},\n",
" {'area': 1,\n",
" 'iscrowd': 0,\n",
" 'image_id': 289343,\n",
" 'bbox': [1, 1, 2, 2],\n",
" 'category_id': 18,\n",
" 'id': 4},\n",
"]\n",
"\n",
"# Duplicate the detections for the very same object\n",
"pred = deepcopy(gt)\n",
"pred[\"annotations\"] = [\n",
" {'area': 1,\n",
" 'iscrowd': 0,\n",
" 'image_id': 289343,\n",
" 'bbox': [0, 0, 1, 1],\n",
" 'category_id': 18,\n",
" 'score': 1.0,\n",
" 'id': 5},\n",
" {'area': 1,\n",
" 'iscrowd': 0,\n",
" 'image_id': 289343,\n",
" 'bbox': [0, 0, 1, 1],\n",
" 'category_id': 18,\n",
" 'score': 0.9,\n",
" 'id': 6},\n",
" {'area': 1,\n",
" 'iscrowd': 0,\n",
" 'image_id': 289343,\n",
" 'bbox': [1, 1, 2, 2],\n",
" 'category_id': 18,\n",
" 'score': 0.8,\n",
" 'id': 7},\n",
"]\n",
"pred = pred[\"annotations\"]\n",
"\n",
"json.dump(gt, open(\"gt.json\", \"w\"))\n",
"json.dump(pred, open(\"pred.json\", \"w\"))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Sanity check\n",
"\n",
"- area is correct, non-zero\n",
"- instance ids are different for both pred and gt\n",
"- category labels are the same\n",
"- image ids are the same"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'area': 1,\n",
" 'iscrowd': 0,\n",
" 'image_id': 289343,\n",
" 'bbox': [0, 0, 1, 1],\n",
" 'category_id': 18,\n",
" 'id': 3},\n",
" {'area': 1,\n",
" 'iscrowd': 0,\n",
" 'image_id': 289343,\n",
" 'bbox': [1, 1, 2, 2],\n",
" 'category_id': 18,\n",
" 'id': 4}]"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"gt[\"annotations\"]"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'area': 1,\n",
" 'iscrowd': 0,\n",
" 'image_id': 289343,\n",
" 'bbox': [0, 0, 1, 1],\n",
" 'category_id': 18,\n",
" 'score': 1.0,\n",
" 'id': 5},\n",
" {'area': 1,\n",
" 'iscrowd': 0,\n",
" 'image_id': 289343,\n",
" 'bbox': [0, 0, 1, 1],\n",
" 'category_id': 18,\n",
" 'score': 0.9,\n",
" 'id': 6},\n",
" {'area': 1,\n",
" 'iscrowd': 0,\n",
" 'image_id': 289343,\n",
" 'bbox': [1, 1, 2, 2],\n",
" 'category_id': 18,\n",
" 'score': 0.8,\n",
" 'id': 7}]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pred"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"loading annotations into memory...\n",
"Done (t=0.01s)\n",
"creating index...\n",
"index created!\n",
"Loading and preparing results...\n",
"DONE (t=0.00s)\n",
"creating index...\n",
"index created!\n"
]
}
],
"source": [
"cocoGt = COCO(\"gt.json\")\n",
"cocoDt = cocoGt.loadRes(\"pred.json\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Fix all parameters for evaluation"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"cocoEval = COCOeval(cocoGt,cocoDt,\"bbox\")\n",
"cocoEval.params.iouThrs = [1.]\n",
"cocoEval.params.imgIds = [289343]\n",
"cocoEval.params.catIds = [18]\n",
"cocoEval.params.areaRng = [[0, 100000.0]]\n",
"cocoEval.params.maxDets = [10]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Quest to reconstruct original characteristic of the PR curve\n",
"\n",
"- if you have `N` predictions in a given category, then you will have `N` precision and recall value.\n",
"- in the pycocotools implementation a fixed recall grid is provided: `[0:.01:1] R=101 recall thresholds for evaluation`\n",
"- each true recall value is looked up in the grid using `np.searchsorted` and right next to the closest point. \n",
"- each corresponding precision value is inserted into an empty array with the same shape as the *recall grid*\n",
"\n",
"Details @ https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py#L402\n",
"\n",
"\n",
"\n",
"#### So how can we unroll this discretization step? \n",
"With ridiculously small stepsizes.\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Running per image evaluation...\n",
"Evaluate annotation type *bbox*\n",
"DONE (t=0.00s).\n",
"Accumulating evaluation results...\n",
"DONE (t=0.03s).\n"
]
}
],
"source": [
"cocoEval.params.recThrs = np.linspace(0, 1, 10000)\n",
"\n",
"cocoEval.evaluate()\n",
"cocoEval.accumulate()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAASd0lEQVR4nO3df7DddX3n8efLAHWtQdZN3HFIMLgNi6lada/AFqdlB7YTmJbsjB0lK21hGDPbFtet9gdOO7ZD/1nXWXbGltamo2u1K4idbjez0mFai0OrhOYiKwu46UYUCNImWqUWVH70vX+cb+Ds5d7cb5LzPYdzP8/HzJ2c749z7vvDTXjdz+fz/X6+qSokSe16wawLkCTNlkEgSY0zCCSpcQaBJDXOIJCkxp006wKO1YYNG2rLli2zLkOS5sqdd975tarauNyxuQuCLVu2sLi4OOsyJGmuJHlgpWMODUlS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktS4wYIgyYeTHEpyzwrHk+QDSQ4kuTvJG4aqRZK0siF7BB8Bth/l+MXA1u5rF/DbA9YiSVrBYE8oq6rbkmw5yik7gI9WVQF7k5yW5OVV9cgQ9dy6/xCfunuQj5YAWP/Ck/il7WfzwpPXzboU6ZjM8lGVpwMPjW0f7PY95//WSXYx6jVwxhlnHNc3++o3v83tX/r6cb1XWs13nnyarz/2BDtedzqv23zarMuRjslcPLO4qnYDuwEWFhbqeD7jbee+gred+4qJ1iUdcev/OcSVH9k36zKk4zLLq4YeBjaPbW/q9klzazTSKc2XWQbBHuAnu6uHzgMeHWp+QBpcZl2AdPwGGxpKcgNwAbAhyUHgV4GTAarqg8DNwCXAAeBx4MqhapGmxf6A5tGQVw3tXOV4AT871PeXJPXjncXSBBwZGXKKQPPIIJCkxhkE0gQkz/QJZlqHdDwMAklqnEEgTYBzBJpnBoEkNc4gkCbgyBSBHQLNI4NAkhpnEEgTkG6WwDkCzSODQJIaZxBIE/DMHIFdAs0hg0CSGmcQSBPgKtSaZwaBNEEODGkeGQTSJNgl0BwzCKQJcq5Y88ggkCYgdgk0xwwCaYLKWQLNIYNAmoDYIdAcMwikSbJDoDlkEEhS4wwCaQJ8UKXmmUEgSY0zCKQJOPLweu8j0DwyCCSpcQaBNAHPPqrSLoHmj0EgSY0zCKQJeOaqITsEmkMGgSQ1ziCQJuDZOQJp/hgEktS4QYMgyfYk+5McSHLNMsfPSHJrkruS3J3kkiHrkYZz5D4C+wSaP4MFQZJ1wPXAxcA2YGeSbUtO+xXgpqp6PXAZ8FtD1SNJWt6QPYJzgANVdX9VPQHcCOxYck4Bp3avXwJ8dcB6pMG4DLXm2ZBBcDrw0Nj2wW7fuF8DLk9yELgZeMdyH5RkV5LFJIuHDx8eolZpIhwY0jya9WTxTuAjVbUJuAT4WJLn1FRVu6tqoaoWNm7cOPUipdXYIdA8GzIIHgY2j21v6vaNuwq4CaCqbgdeCGwYsCZpWHYJNIeGDIJ9wNYkZyY5hdFk8J4l5zwIXAiQ5FWMgsCxH82dOEmgOTZYEFTVU8DVwC3AFxldHXRvkmuTXNqd9m7g7Um+ANwAXFFef6c55qJzmkcnDfnhVXUzo0ng8X3vHXt9H3D+kDVIko5u1pPF0prgonOaZwaBJDXOIJAm4JlF5+wRaA4ZBJLUOINAmoAcWXRuxnVIx8MgkKTGGQTSBDw7R2CfQPPHIJCkxhkE0gTZH9A8MggkqXEGgTQB3kegeWYQSFLjDAJpAvLsakMzrUM6HgaBJDXOIJAmwOfSaJ4ZBNIEOVmseWQQSBNgj0DzzCCQJsgOgeaRQSBNwLNXDUnzZ9BnFkut+cCn/y8fv+PBWZehNerK87dw4av+6cQ/1yCQJuCMl76IC89+Gd/89pN8+8mnZ12O1qgnnx5m8NEgkCbgH52yjg9d8cZZlyEdF+cIJKlxBoEkNa7X0FCS7wHeDGwZf09VXTtMWZKkaek7R/A/gEeBO4HvDleOJGna+gbBpqraPmglkqSZ6DtH8Lkkrxm0EknSTPTtEbwJuCLJlxkNDQWoqnrtYJVJkqaibxBcPGgVkqSZ6TU0VFUPAKcBP9Z9ndbtO6ok25PsT3IgyTUrnPOWJPcluTfJx4+leEnSiesVBEneCfw34GXd1+8neccq71kHXM+oN7EN2Jlk25JztgLvAc6vqu8H/sMxt0CSdEL6Dg1dBZxbVY8BJHkfcDvwG0d5zznAgaq6v3vPjcAO4L6xc94OXF9V3wCoqkPHVr4k6UT1vWoowPhKWk93+47mdOChse2D3b5xZwFnJflskr1Jlr1ENcmuJItJFg8fPtyzZElSH317BP8VuCPJf++2/w3woQl9/63ABcAm4LYkr6mqb46fVFW7gd0ACwsLPvtDkiaoVxBU1XVJPsPoMlKAK6vqrlXe9jCweWx7U7dv3EHgjqp6Evhykr9iFAz7+tQlSTpxRx0aSnJq9+dLga8Av999PdDtO5p9wNYkZyY5BbgM2LPknD9i1BsgyQZGQ0X3H1sTJEknYrUewceBH2W0xtD4kEy67Veu9MaqeirJ1cAtwDrgw1V1b5JrgcWq2tMd+5Ek9zGad/iFqvr6cbdGknTMUjVfQ+4LCwu1uLg46zIkaa4kubOqFpY71vc+gvOTfG/3+vIk1yU5Y5JFSpJmo+/lo78NPJ7kB4B3A18CPjZYVZKkqekbBE/VaAxpB/CbVXU9sH64siRJ09L3PoJvJXkPcDnwQ0leAJw8XFmSpGnp2yN4K6Plp6+qqr9mdE/A+werSpI0NX1vKPtr4Lqx7QeBjw5VlCRpeo4aBEn+oqrelORbLHMfQVWdOmh1kqTBHTUIqupN3Z9ODEvSGtX3PoLzkqwf216f5NzhypIkTcux3Efw92Pbj3X7JElzrvfzCGpsLYqq+gf6X3oqSXoe6xsE9yf590lO7r7eiauEStKa0DcI/h3wg4yeJ3AQOBfYNVRRkqTp6XsfwSFGzxOQJK0xfa8aOivJp5Pc022/NsmvDFuaJGka+g4N/S7wHuBJgKq6G3sIkrQm9A2CF1XVXy7Z99Ski5EkTV/fIPhakn9Gt8xEkh8HHhmsKknS1PS9F+Bngd3A2UkeBr4MvG2wqiRJU7NqEHTPHlioqou6x1W+oKq+NXxpkqRpWHVoqLuL+Be7148ZApK0tvSdI/jTJD+fZHOSlx75GrQySdJU9J0jeCujieKfWbL/lZMtR5I0bX2DYBujEHgTo0D4c+CDQxUlSZqevkHwe8DfAR/otv9tt+8tQxQlSZqevkHw6qraNrZ9a5L7hihIkjRdfSeLP5/kvCMb3dPJFocpSZI0TX17BP8C+FySB7vtM4D9Sf43o4fYv3aQ6iRJg+sbBNsHrUKSNDN9n0fwwNCFSJJmo+8cgSRpjRo0CJJsT7I/yYEk1xzlvDcnqSQLQ9YjSXquwYIgyTrgeuBiRjek7UyybZnz1gPvBO4YqhZJ0sqG7BGcAxyoqvur6gngRmDHMuf9OvA+4DsD1iJJWsGQQXA68NDY9sFu3zOSvAHYXFWfOtoHJdmVZDHJ4uHDhydfqSQ1bGaTxd1zDq4D3r3auVW1u6oWqmph48aNwxcnSQ0ZMggeBjaPbW/q9h2xHng18JkkXwHOA/Y4YSxJ0zVkEOwDtiY5M8kpwGXAniMHq+rRqtpQVVuqaguwF7i0qly6QpKmaLAgqKqngKuBW4AvAjdV1b1Jrk1y6VDfV5J0bPouMXFcqupm4OYl+967wrkXDFmLJGl53lksSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGjdoECTZnmR/kgNJrlnm+LuS3Jfk7iSfTvKKIeuRJD3XYEGQZB1wPXAxsA3YmWTbktPuAhaq6rXAHwD/aah6JEnLG7JHcA5woKrur6ongBuBHeMnVNWtVfV4t7kX2DRgPZKkZQwZBKcDD41tH+z2reQq4I+XO5BkV5LFJIuHDx+eYImSpOfFZHGSy4EF4P3LHa+q3VW1UFULGzdunG5xkrTGnTTgZz8MbB7b3tTt+/8kuQj4ZeCHq+q7A9YjSVrGkD2CfcDWJGcmOQW4DNgzfkKS1wO/A1xaVYcGrEWStILBgqCqngKuBm4BvgjcVFX3Jrk2yaXdae8HXgx8Msn/SrJnhY+TJA1kyKEhqupm4OYl+9479vqiIb+/JGl1z4vJYknS7BgEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXGDBkGS7Un2JzmQ5Jpljn9Pkk90x+9IsmXIeiRJzzVYECRZB1wPXAxsA3Ym2bbktKuAb1TV9wH/BXjfUPVIkpY3ZI/gHOBAVd1fVU8ANwI7lpyzA/i97vUfABcmyYA1SZKWOGnAzz4deGhs+yBw7krnVNVTSR4F/gnwtfGTkuwCdnWbf59k/3HWtGHpZzfANrfBNrfhRNr8ipUODBkEE1NVu4HdJ/o5SRaramECJc0N29wG29yGodo85NDQw8Dmse1N3b5lz0lyEvAS4OsD1iRJWmLIINgHbE1yZpJTgMuAPUvO2QP8VPf6x4E/q6oasCZJ0hKDDQ11Y/5XA7cA64APV9W9Sa4FFqtqD/Ah4GNJDgB/yygshnTCw0tzyDa3wTa3YZA2x1/AJalt3lksSY0zCCSpcWsyCFpc2qJHm9+V5L4kdyf5dJIVrymeF6u1eey8NyepJHN/qWGfNid5S/ezvjfJx6dd46T1+Lt9RpJbk9zV/f2+ZBZ1TkqSDyc5lOSeFY4nyQe6/x53J3nDCX/TqlpTX4wmpr8EvBI4BfgCsG3JOT8DfLB7fRnwiVnXPYU2/yvgRd3rn26hzd1564HbgL3AwqzrnsLPeStwF/CPu+2XzbruKbR5N/DT3ettwFdmXfcJtvmHgDcA96xw/BLgj4EA5wF3nOj3XIs9ghaXtli1zVV1a1U93m3uZXRfxzzr83MG+HVGa1h9Z5rFDaRPm98OXF9V3wCoqkNTrnHS+rS5gFO71y8BvjrF+iauqm5jdBXlSnYAH62RvcBpSV5+It9zLQbBcktbnL7SOVX1FHBkaYt51afN465i9BvFPFu1zV2XeXNVfWqahQ2oz8/5LOCsJJ9NsjfJ9qlVN4w+bf414PIkB4GbgXdMp7SZOdZ/76uaiyUmNDlJLgcWgB+edS1DSvIC4DrgihmXMm0nMRoeuoBRr++2JK+pqm/OtKph7QQ+UlX/Ocm/ZHRv0qur6h9mXdi8WIs9ghaXtujTZpJcBPwycGlVfXdKtQ1ltTavB14NfCbJVxiNpe6Z8wnjPj/ng8Ceqnqyqr4M/BWjYJhXfdp8FXATQFXdDryQ0eJsa1Wvf+/HYi0GQYtLW6za5iSvB36HUQjM+7gxrNLmqnq0qjZU1Zaq2sJoXuTSqlqcTbkT0efv9h8x6g2QZAOjoaL7p1nkhPVp84PAhQBJXsUoCA5Ptcrp2gP8ZHf10HnAo1X1yIl84JobGqrn59IWg+rZ5vcDLwY+2c2LP1hVl86s6BPUs81rSs823wL8SJL7gKeBX6ique3t9mzzu4HfTfJzjCaOr5jnX+yS3MAozDd08x6/CpwMUFUfZDQPcglwAHgcuPKEv+cc//eSJE3AWhwakiQdA4NAkhpnEEhS4wwCSWqcQSBJjTMIpClKsuXIqpJJLkjyP2ddk2QQSD10N+/470Vrkn+xpRV0v73vT/JR4B7gJ5LcnuTzST6Z5MXdeW9M8rkkX0jyl0nWd+/98+7czyf5wdm2RlrZmruzWJqwrYyWIzkA/CFwUVU9luSXgHcl+Y/AJ4C3VtW+JKcC3wYOAf+6qr6TZCtwA6PF/qTnHYNAOroHqmpvkh9l9NCTz3ZLdJwC3A78c+CRqtoHUFV/B5Dke4HfTPI6Rks9nDWL4qU+DALp6B7r/gzwJ1W1c/xgktes8L6fA/4G+AFGQ7Br4cE4WqOcI5D62Qucn+T7YPQbf5KzgP3Ay5O8sdu/fmxp80e6NfF/gtGCadLzkkEg9VBVhxk95OaGJHczGhY6u3t84luB30jyBeBPGC2D/FvAT3X7zubZnoX0vOPqo5LUOHsEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ17v8BR6TgUHc6sZwAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plt.figure()\n",
"plt.plot(cocoEval.params.recThrs, cocoEval.eval[\"precision\"].flatten())\n",
"plt.ylabel(\"precision\")\n",
"plt.xlabel(\"recall\")\n",
"plt.ylim(0, 1.1)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment