{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Starter Code ADE20K\n",
    "This notebook contains a tutorial on how to explore data in ADE20K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ln: failed to create symbolic link './utils': File exists\n",
      "ln: failed to create symbolic link './dataset': File exists\n",
      "--2021-02-28 10:56:34--  http://groups.csail.mit.edu/vision/datasets/ADE20K/toolkit/index_ade20k.pkl\n",
      "Resolving groups.csail.mit.edu (groups.csail.mit.edu)... 128.30.2.44\n",
      "Connecting to groups.csail.mit.edu (groups.csail.mit.edu)|128.30.2.44|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 817537016 (780M)\n",
      "Saving to: ‘index_ade20k.pkl’\n",
      "\n",
      "index_ade20k.pkl    100%[===================>] 779.66M  65.8MB/s    in 12s     \n",
      "\n",
      "2021-02-28 10:57:44 (63.1 MB/s) - ‘index_ade20k.pkl’ saved [817537016/817537016]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Starter setup\n",
    "! sh setup.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import IPython.display\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "import numpy as np\n",
    "import pickle as pkl\n",
    "from utils import utils_ade20k"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load index with global information about ADE20K\n",
    "DATASET_PATH = '../dataset/'\n",
    "index_file = 'index_ade20k.pkl'\n",
    "with open('{}/{}'.format(DATASET_PATH, index_file), 'rb') as f:\n",
    "    index_ade20k = pkl.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File loaded, description of the attributes:\n",
      "--------------------------------------------\n",
      "* filename: array of length N=27574 with the image file names\n",
      "* folder: array of length N with the image folder names.\n",
      "* scene: array of length N providing the scene name (same classes as the Places database) for each image.\n",
      "* objectIsPart: array of size [C, N] counting how many times an object is a part in each image. objectIsPart[c,i]=m if in image i object class c is a part of another object m times. For objects, objectIsPart[c,i]=0, and for parts we will find: objectIsPart[c,i] = objectPresence(c,i)\n",
      "* objectPresence: array of size [C, N] with the object counts per image. objectPresence(c,i)=n if in image i there are n instances of object class c.\n",
      "* objectcounts: array of length C with the number of instances for each object class.\n",
      "* objectnames: array of length C with the object class names.\n",
      "* proportionClassIsPart: array of length C with the proportion of times that class c behaves as a part. If proportionClassIsPart[c]=0 then it means that this is a main object (e.g., car, chair, ...). See bellow for a discussion on the utility of this variable.\n",
      "* wordnet_found: array of length C. It indicates if the objectname was found in Wordnet.\n",
      "* wordnet_level1: list of length C. WordNet associated.\n",
      "* wordnet_synset: list of length C. WordNet synset for each object name. Shows the full hierarchy separated by .\n",
      "* wordnet_hypernym: list of length C. WordNet hypernyms for each object name.\n",
      "* wordnet_gloss: list of length C. WordNet definition.\n",
      "* wordnet_synonyms: list of length C. Synonyms for the WordNet definition.\n",
      "* wordnet_frequency: array of length C. How many times each wordnet appears\n",
      "--------------------------------------------\n",
      "\n",
      "The dataset has 27574 images\n",
      "The image at index 16868 is ADE_train_00016869.jpg\n",
      "It is located at ADE20K_2021_17_01/images/ADE/training/urban/street/ADE_train_00016869.jpg\n",
      "It happens in a /street\n",
      "It has 70.0 objects, of which 19.0 are parts\n",
      "The most common object is object car, auto, automobile, machine, motorcar (400), which appears 11.0 times\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(\"File loaded, description of the attributes:\")\n",
    "print('--------------------------------------------')\n",
    "for attribute_name, desc in index_ade20k['description'].items():\n",
    "    print('* {}: {}'.format(attribute_name, desc))\n",
    "print('--------------------------------------------\\n')\n",
    "\n",
    "i = 16868 # 16899, 16964\n",
    "nfiles = len(index_ade20k['filename'])\n",
    "file_name = index_ade20k['filename'][i]\n",
    "num_obj = index_ade20k['objectPresence'][:, i].sum()\n",
    "num_parts = index_ade20k['objectIsPart'][:, i].sum()\n",
    "count_obj = index_ade20k['objectPresence'][:, i].max()\n",
    "obj_id = np.where(index_ade20k['objectPresence'][:, i] == count_obj)[0][0]\n",
    "obj_name = index_ade20k['objectnames'][obj_id]\n",
    "full_file_name = '{}/{}'.format(index_ade20k['folder'][i], index_ade20k['filename'][i])\n",
    "print(\"The dataset has {} images\".format(nfiles))\n",
    "print(\"The image at index {} is {}\".format(i, file_name))\n",
    "print(\"It is located at {}\".format(full_file_name))\n",
    "print(\"It happens in a {}\".format(index_ade20k['scene'][i]))\n",
    "print(\"It has {} objects, of which {} are parts\".format(num_obj, num_parts))\n",
    "print(\"The most common object is object {} ({}), which appears {} times\".format(obj_name, obj_id, count_obj))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading a segmentation object\n",
    "We will visualize the same object we were studying before"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 202716 characters]\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlwAAAEACAYAAABmuKihAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAACJFJREFUeJzt3c+r5XUdx/H3+86Q4ghFGsS1QSaniHAhAzYwUARhk4NBBUJBBNFCaDGrqEVBq1qFkP9AiwIhHbOkxKSFGzEmxF0DTmNhTpkRltoYOvfTYu6VOz/0esd5eb7nex+P1bnnnO+Xz9kcnvf9+X7v7TFGAQCQs7LoBQAAzJ3gAgAIE1wAAGGCCwAgTHABAIQJLgCAMMEFABAmuICl1d1/7u4z3f1ydz/f3T/p7msv4zyf7u6/JtYIUCW4gOX3+THGtVV1oKpurarvbefg7t4dWRXAJoILmIUxxnNV9XBV3dzdX+/uP3b3S919qrvv2njfxjSru7/T3X+vqnvXj1tdn5S93N2r3f2J7v5Dd/9nfXp294I+GjADfrMDZqG791bVkap6oKr+UVV3VNWpqvpUVT3c3cfHGE+uv/2DVfX+qrqxzv3iebCqfjbG+NCm8x2rqh+PMX66vk1587v2YYDZEVzAsnuwu1+vqn9X1a+r6odjjDObXn+su39bVZ+sqo3gWquq748x/ldV1d2XOu9rVbW/u68fY/yzqp5IfQBg/mwpAsvuC2OM940xbhxjfHOMcaa7b+/uJ7r7X939Yp2bfF2/6ZgXxhivbnHeb1TVR6vqRHcf7+47Uh8AmD8TLmBWuvuqqjpWVV+rql+OMV7r7geravMYa1xw2IU/1xjj6ar6SnevVNWXqur+7r5ujPFKaOnAjJlwAXPznqq6qqpeqKrXu/v2qvrsFsc8X1XXdfd7N57o7q929wfGGGtV9eL602cTCwbmz4QLmJUxxkvdfbSqfl7nwuuhqvrVFsec6O57q+pUd++qqo9X1eeq6u7uvqaq/lJVX34b25AAl9RjXDRJBwDgCrKlCAAQJrgAAMIEFwBAmOACAAgTXAAAYZP+sxC3rdzpFkoAYCk8unbfJf9PWJUJFwBAnOACAAgTXAAAYYILACBMcAEAhAkuAIAwwQUAECa4AADCBBcAQJjgAgAIE1wAAGGCCwAgbNL/vBqWwSOnnzrv58OrtyxoJQBMlQkXs3Jh/ADAFJhwMQtCC4ApM+Fi6YktAKbOhAsmYnM4ug4MYF4EF0ycEANYfrYUAQDCTLjgHTJ1AmArgotZED0ATJktRZae2AJg6ky4YCKEI8B8CS6YOCEGsPxsKQIAhAkuAIAwwQUAECa4ltjKnj21smfPopcBAGzBRfNLSGQBwHIRXBO0OajWXnllgSsBAK4EW4oAAGGCawZsMQLAtNlSnDgxBQDLT3BNhLACgPmypTgBYgsA5k1wzYA7GQFg2mwpLhlxBQDLR3BNmLgCgHkQXBMgrABg3lzDBQAQJrgAAMIEFwBAmOACAAgTXAAAYYILACBMcAEAhAkuAIAwwQUAECa4AADCBBcAQJjgAgAIE1wAAGGCCwAgTHABAIQJLgCAMMEFABAmuAAAwgQXAECY4AIACBNcAABhggsAIExwAQCECS4AgDDBBQAQJrgAAMIEFwBAmOACAAgTXAAAYYILACBMcAEAhAkuAIAwwQUAECa4AADCBBcAQJjgAgAIE1wAAGGCCwAgTHABAIQJLgCAMMEFABAmuAAAwgQXAECY4AIACBNcAABhggsAIExwAQCECS4AgDDBBQAQJrgAAMIEFwBAmOACAAgTXAAAYYILACBMcAEAhAkuAICw3YteAOxEz3730BuP9/7g8QWuBIB3gwkXAECY4AIACBNcAABhggsAIExwAQCEuUsRFsCdiQA7iwkXAECY4AIACBNcAABhggsAIExwAQCECS4AgDDBBQAQJrgAAMIEFwBAmOACAAgTXAAAYYILACBMcAEAhAkuAIAwwQUTdNPxq+um41cvehkAXCG7F70A2OkuDKs/3frqglYCQIoJFwBAmAkXzNx/v3jwoueu+cXvF7ASgJ1LcMGMXSq2Njxy+qmLnju8ektyOcBM+P7YPluKMFHv9Fqut4qtKl+OwOW5VGyxNcEFE5SOLYDL8fQ9B+vD99+16GUsJVuKMAFX8s7E7cSWKRewHR85eu76z8NHz//uOP2tQ7Vajy9iSUvDhAsW7HJi6/S3DwVWUnX05Ik6evJE5NzAfK3+SGxtxYQLdrjfPPdkVVUdueFA3bP/YwteDcA8CS5YMrv276u9D/ytav++euixY+e9duSGA9s610ZsbX683XMAsDXBBUvm7Mlnatf+fXX25DNv6/3+5hawXWu/2/vG45XPPLuwc8yJ4IIl9FaxJbAApsdF8wDAea7kRMp065weYyx6DW/qtpU7p7s4mInN13G5fgvg8j26dl+/2Wu2FGGHE1kAebYUAQDCBBcAQJjgAgAIE1wAAGGCCwAgTHABAIQJLgCAMMEFABAmuAAAwgQXAECY4AIACBNcAABhggsAIExwAQCECS4AgDDBBQAQJrgAAMIEFwBAmOACAAgTXAAAYYILACBMcAEAhAkuAIAwwQUAECa4AADCBBcAQJjgAgAIE1wAAGGCCwAgTHABAIQJLgCAMMEFABAmuAAAwgQXAECY4AIACBNcAABhggsAIExwAQCECS4AgDDBBQAQJrgAAMIEFwBAmOACAAgTXAAAYYILACBMcAEAhAkuAIAwwQUAECa4AADCBBcAQJjgAgAIE1wAAGGCCwAgTHABAIQJLgCAMMEFABAmuAAAwgQXAECY4AIACBNcAABhggsAIExwAQCECS4AgDDBBQAQJrgAAMIEFwBAmOACAAgTXAAAYYILACBMcAEAhAkuAIAwwQUAECa4AADCBBcAQJjgAgAIE1wAAGE9xlj0GgAAZs2ECwAgTHABAIQJLgCAMMEFABAmuAAAwgQXAECY4AIACBNcAABhggsAIExwAQCECS4AgDDBBQAQJrgAAMIEFwBAmOACAAgTXAAAYYILACBMcAEAhAkuAIAwwQUAECa4AADCBBcAQJjgAgAI+z8MrxrOC7n4xQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "root_path = DATASET_PATH\n",
    "info = utils_ade20k.loadAde20K('{}/{}'.format(root_path, full_file_name))\n",
    "img = cv2.imread(info['img_name'])[:,:,::-1]\n",
    "seg = cv2.imread(info['segm_name'])[:,:,::-1]\n",
    "seg_mask = seg.copy()\n",
    "seg_mask[info['class_mask'] != obj_id+1] *= 0\n",
    "plt.figure(figsize=(15,5))\n",
    "\n",
    "plt.imshow(np.concatenate([img, seg, seg_mask], 1))\n",
    "plt.axis('off')\n",
    "if len(info['partclass_mask']):\n",
    "    plt.figure(figsize=(5*len(info['partclass_mask']), 5))\n",
    "    plt.title('Parts')\n",
    "    plt.imshow(np.concatenate(info['partclass_mask'],1))\n",
    "    plt.axis('off')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can also inspect the attributes `info['objects']` and `info['parts']` for information about object names, attributes etc."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Amodal segmentation and Polygons\n",
    "We also provide access to the original polygons annotated, as well as the amodal segmentation of object instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.5, 6143.5, 1535.5, -0.5)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3MAAAD0CAYAAAAxFH+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAABoxJREFUeJzt3dFO2zAAQFE88d+EL/cepjI2la5daZ3bniMhIVGERZU4NzZhzDlfAAAAaPmxegAAAABcTswBAAAEiTkAAIAgMQcAABAk5gAAAILEHAAAQJCYAwAACBJzAAAAQWIOAAAgSMwBAAAEva4ewCljjLl6DAAAAKvMOcdXX7MyBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBYg4AACBIzAEAAASJOQAAgCAxBwAAECTmAAAAgsQcAABAkJgDAAAIEnMAAABBr6sHAMAvc85/vmaMcYeRAJxnzum8BAtZmQPYgXNC7vC6c18LADw2MQcPbNu2j4v/cz+4j2t/794rYDXnIVjPNkt4QNu2vby9va0eBke4+AEewedzma2WsM7Y84XFGGO/g4Mdu+a4NiHf1nefc71fwAp/n8uci+B25pxfHmBW5uDB7PkGDb8veE69T+/v7x+fn1phdfEEAM/NyhxE3fLYFQkAfOWwrfLzPGTegNs5tTLnASgQdOubMHu+yQPAOsdCDlhHzAEAcJZjK3BW5WAdMQcx97ob6q4rAOcwX8A6Yg5ixhh/PCDjlj8HAID98gAUCDt2/F4SYdd+PwDP6zCHmDfgtjwABQAA4MH4P3MQdu3d0MMTyQ7bNrdt+4ZRAfBMDk+4BO5PzMGTMwEDcKnPWyz3/Cc78OhsswQAAAgScwAAAEFiDgCA/2a7Pqzjb+YAALiIgIN9sDIHAAAQJOYAAACCxBwAAECQmAMAAAgScwAAAEFiDgAAIEjMAQAABIk5AACAIDEHAAAQJOYAAACCxBwAAECQmAMAAAgScwAAAEFiDgAAIEjMAQAABIk5AACAIDEHAAAQJOYAAACCxBwAAECQmAMAAAgScwAAAEFiDgAAIEjMAQAABIk5AACAIDEHAAAQJOYAAACCxBwAAECQmAMAAAgScwAAAEFiDgAAIEjMAQAABIk5AACAIDEHAAAQJOYAAACCxBwAAECQmAMAAAgScwAAAEFiDgAAIEjMAQAABIk5AACAIDEHAAAQJOYAAACCxBwAAECQmAMAAAgScwAAAEFiDgAAIEjMAQAABIk5AACAIDEHAAAQJOYAAACCxBwAAECQmAMAAAgScwAAAEFiDgAAIEjMAQAABIk5AACAIDEHAAAQJOYAAACCxBwAAECQmAMAAAgScwAAAEFiDgAAIEjMAQAABIk5AACAIDEHAAAQJOYAAACCxBwAAECQmAMAAAgScwAAAEFiDgAAIEjMAQAABIk5AACAoDHnXD0GAAAALmRlDgAAIEjMAQAABIk5AACAIDEHAAAQJOYAAACCxBwAAECQmAMAAAgScwAAAEFiDgAAIEjMAQAABIk5AACAIDEHAAAQJOYAAACCxBwAAECQmAMAAAgScwAAAEFiDgAAIEjMAQAABIk5AACAIDEHAAAQJOYAAACCxBwAAECQmAMAAAj6CZVzvKT8tuWMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# We have a segmentation for every object instance, showing object bounds without occlusions\n",
    "instances = [4, 10, 19]\n",
    "images = []\n",
    "for instance in instances:\n",
    "    file_instance = '{}/{}/instance_{:03}_{}'.format(\n",
    "        root_path, full_file_name.replace('.jpg', ''), instance, file_name.replace('.jpg', '.png'))\n",
    "    aux = cv2.imread(file_instance)\n",
    "    images.append(aux)\n",
    "plt.figure(figsize=(5*len(images), 5))\n",
    "plt.imshow(np.concatenate(images, 1))\n",
    "plt.axis('off')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally we can get the original polygons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.5, 2047.5, 1535.5, -0.5)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 1350044 characters]\n",
      "text/plain": [
       "<Figure size 1080x1080 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "img = utils_ade20k.plot_polygon('{}/{}'.format(root_path, full_file_name), info)\n",
    "plt.figure(figsize=(15,15))\n",
    "plt.imshow(img[:, :, ::-1])\n",
    "plt.axis('off')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
