{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Starter Code ADE20K\n",
    "This notebook contains a tutorial on how to explore data in ADE20K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import IPython.display\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "import numpy as np\n",
    "import pickle as pkl\n",
    "import utils_ade20k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ade20k_starter.ipynb  index_ade20k.pkl  \u001b[0m\u001b[01;34m__pycache__\u001b[0m/  utils_ade20k.py\r\n"
     ]
    }
   ],
   "source": [
    "ls "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load index with global information about ADE20K\n",
    "DATASET_PATH = '/data/vision/torralba/datasets/ade20k/release/'\n",
    "index_file = 'index_ade20k.pkl'\n",
    "with open(index_file, 'rb') as f:\n",
    "    index_ade20k = pkl.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File loaded, description of the attributes:\n",
      "--------------------------------------------\n",
      "* filename: array of length N=27574 with the image file names\n",
      "* folder: array of length N with the image folder names.\n",
      "* scene: array of length N providing the scene name (same classes as the Places database) for each image.\n",
      "* objectIsPart: array of size [C, N] counting how many times an object is a part in each image. objectIsPart[c,i]=m if in image i object class c is a part of another object m times. For objects, objectIsPart[c,i]=0, and for parts we will find: objectIsPart[c,i] = objectPresence(c,i)\n",
      "* objectPresence: array of size [C, N] with the object counts per image. objectPresence(c,i)=n if in image i there are n instances of object class c.\n",
      "* objectcounts: array of length C with the number of instances for each object class.\n",
      "* objectnames: array of length C with the object class names.\n",
      "* proportionClassIsPart: array of length C with the proportion of times that class c behaves as a part. If proportionClassIsPart[c]=0 then it means that this is a main object (e.g., car, chair, ...). See bellow for a discussion on the utility of this variable.\n",
      "* wordnet_found: array of length C. It indicates if the objectname was found in Wordnet.\n",
      "* wordnet_level1: list of length C. WordNet associated.\n",
      "* wordnet_synset: list of length C. WordNet synset for each object name. Shows the full hierarchy separated by .\n",
      "* wordnet_hypernym: list of length C. WordNet hypernyms for each object name.\n",
      "* wordnet_gloss: list of length C. WordNet definition.\n",
      "* wordnet_synonyms: list of length C. Synonyms for the WordNet definition.\n",
      "* wordnet_frequency: array of length C. How many times each wordnet appears\n",
      "--------------------------------------------\n",
      "\n",
      "The dataset has 27574 images\n",
      "The image at index 20390 is ADE_train_00020391.jpg\n",
      "It is located at ADE20K_2021_17_01/images/ADE/training/home_or_hotel/bedroom/ADE_train_00020391.jpg\n",
      "It happens in a /bedroom\n",
      "It has 21.0 objects, of which 2.0 are parts\n",
      "The most common object is object painting, picture (1734), which appears 3.0 times\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(\"File loaded, description of the attributes:\")\n",
    "print('--------------------------------------------')\n",
    "for attribute_name, desc in index_ade20k['description'].items():\n",
    "    print('* {}: {}'.format(attribute_name, desc))\n",
    "print('--------------------------------------------\\n')\n",
    "\n",
    "i = 20390\n",
    "nfiles = len(index_ade20k['filename'])\n",
    "file_name = index_ade20k['filename'][i]\n",
    "num_obj = index_ade20k['objectPresence'][:, i].sum()\n",
    "num_parts = index_ade20k['objectIsPart'][:, i].sum()\n",
    "count_obj = index_ade20k['objectPresence'][:, i].max()\n",
    "obj_id = np.where(index_ade20k['objectPresence'][:, i] == count_obj)[0][0]\n",
    "obj_name = index_ade20k['objectnames'][obj_id]\n",
    "full_file_name = '{}/{}'.format(index_ade20k['folder'][i], index_ade20k['filename'][i])\n",
    "print(\"The dataset has {} images\".format(nfiles))\n",
    "print(\"The image at index {} is {}\".format(i, file_name))\n",
    "print(\"It is located at {}\".format(full_file_name))\n",
    "print(\"It happens in a {}\".format(index_ade20k['scene'][i]))\n",
    "print(\"It has {} objects, of which {} are parts\".format(num_obj, num_parts))\n",
    "print(\"The most common object is object {} ({}), which appears {} times\".format(obj_name, obj_id, count_obj))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ADE20K_2021_17_01/images/ADE/training/transportation/airport_terminal'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index_ade20k['folder'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading a segmentation object\n",
    "We will visualize the same obejct we were studying before"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'training': 25574, 'validation': 2000})"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter([x.split('/')[-3] for x in index_ade20k['folder']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 143016 characters]\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAADpCAYAAACz3SV8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAABWBJREFUeJzt3M+rnGcZxvHrjsUUKbSYFCRYuildlCxEMK6UbtS2RBCh0EIRpItCF27dCO5cBvwLXCgE1Ep/WErbVXfBSOnOghBaaEtrpaRNJJHEPC7OiV6aYFQ8zknm81kNc+Z9ed7Nd+7nzMw7a60AsOPAphcAsJ+IIkARRYAiigBFFAGKKAIUUQQooshGzMxbM3NhZs7PzAcz85OZueO/OM+DM/POXqyR7SSKbNI311p3JPliki8l+cF/cvDM3LYnq2KriSIbt9Z6N8lLSY7OzHdn5nczc25mzszMU1dfd3UqnJnvz8z7SU7uHndkd+I8PzNHZubYzPx2Zj7ZnUJPbOjSuAl5p2XjZuaeJI8k+VWSPyQ5nuRMkq8meWlmTq+1Xt99+eeSfDbJvdl5U/9ykp+ttT5f53smyY/XWj/d3ZIf/b9dDDc9UWSTnp2Zy0k+TvJikh+ttS7U31+bmVeSfCXJ1SheSfLDtdafk2RmrnfeS0num5nDa60/Jjm1VxfArcf2mU361lrrrrXWvWutp9daF2bm4Zk5NTMfzczZ7EyQh+uYD9daF29w3ieT3J/kzZk5PTPH9+oCuPWYFNk3ZuZgkmeSfCfJc2utSzPzbJIeB//5tk7X3OZprfX7JI/PzIEk307yy5k5tNb60x4tnVuISZH95NNJDib5MMnlmXk4yddvcMwHSQ7NzJ1Xn5iZJ2bm7rXWlSRnd5/+y14smFuPSZF9Y611bma+l+Tn2YnjC0mev8Exb87MySRnZuZTSR5I8lCSEzPzmSRvJ3ns39hyQ5Jk3GQW4O9snwGKKAIUUQQooghQRBGg7Iuv5HztwKM+Agf+51698ovr/g70XzEpAhRRBCiiCFBEEaCIIkARRYAiigBFFAGKKAIUUQQooghQRBGgiCJAEUWAIooARRQBiigCFFEEKKIIUEQRoIgiQBFFgCKKAEUUAYooAhRRBCiiCFBEEaCIIkARRYAiigBFFAGKKAIUUQQooghQRBGgiCJAEUWAIooARRQBiigCFFEEKKIIUEQRoIgiQBFFgCKKAEUUAYooAhRRBCiiCFBEEaCIIkARRYAiigBFFAGKKAIUUQQooghQRBGgiCJAEUWAIooARRQBiigCFFEEKKIIUEQRoIgiQBFFgCKKAEUUAYooAhRRBCiiCFBEEaCIIkARRYAiigBFFAGKKAIUUQQooghQRBGgiCJAEUWAIooARRQBiigCFFEEKKIIUEQRoIgiQBFFgCKKAEUUAYooAhRRBCiiCFBEEaCIIkARRYAiikCS5OX33sjL772x6WVsnCgC/xDDbQ+jKAIUUQTyjSNfuO7jbSSKwNZvmZsoAls/HTZRBCiiCNg+F1EEKKII+J9iEUWAIooARRQBH7QUUQQooghQRBGgiCLgKznltk0vANgfhHGHKAJJkovHj/3t8e2//s0GV7JZts/ANTqQ20YUgWts86Ro+wwk2e4QNpMiQBFFgCKKAEUUAYooAhRRBCiiCFBEEaCIIkARRYAiigBFFAGKKAIUUQQooghQRBGgiCJAEUWAIooARRQBiigCFFEEKKIIUEQRoIgiQBFFgCKKAEUUAYooAhRRBCiiCFBEEaCIIkARRYAiigBFFAGKKAIUUQQooghQRBGgiCJAEUWAIooARRQBiigCFFEEKKIIUEQRoIgiQBFFgCKKAEUUAYooAhRRBCiiCFBEEaCIIkARRYAiigBFFAGKKAKUWWtteg0A+4ZJEaCIIkARRYAiigBFFAGKKAIUUQQooghQRBGgiCJAEUWAIooARRQBiigCFFEEKKIIUEQRoIgiQBFFgCKKAEUUAYooAhRRBCh/BcMjtIAyicPKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "root_path = DATASET_PATH\n",
    "info = utils_ade20k.loadAde20K('{}/{}'.format(root_path, full_file_name))\n",
    "img = cv2.imread(info['img_name'])[:,:,::-1]\n",
    "seg = cv2.imread(info['segm_name'])[:,:,::-1]\n",
    "seg_mask = seg.copy()\n",
    "seg_mask[info['class_mask'] != obj_id+1] *= 0\n",
    "plt.figure(figsize=(15,5))\n",
    "\n",
    "plt.imshow(np.concatenate([img, seg, seg_mask], 1))\n",
    "plt.axis('off')\n",
    "if len(info['partclass_mask']):\n",
    "    plt.figure(figsize=(5*len(info['partclass_mask']), 5))\n",
    "    plt.title('Parts')\n",
    "    plt.imshow(np.concatenate(info['partclass_mask'],1))\n",
    "    plt.axis('off')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can also inspect the attributes `info['objects']` and `info['parts']` for information about object names, attributes etc."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Amodal segmentation and Polygons\n",
    "We also provide access to the original polygons annotated, as well as the amodal segmentation of object instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.5, 3071.5, 681.5, -0.5)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA20AAADdCAYAAAAo0JoiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAB8RJREFUeJzt3ctu2zoYRlEpyHvHenKewUFa17B8p/iRXAvIoM2EA1vm7k+zayllAQAAINNX6wUAAACwT7QBAAAEE20AAADBRBsAAEAw0QYAABBMtAEAAAQTbQAAAMFEGwAAQDDRBgAAEEy0AQAABPtuvYBlWZZ1XUvrNQAAALRSSln3fmfSBgAAEEy0AQAABBNtAAAAwUQbAABAMNEGAAAQTLQBAAAEE20AAADBRBsAAEAw0QYAABBMtAEAAAQTbQAAAMFEGwAAQDDRBgAAEEy0AQAABBNtAAAAwUQbAABAMNEGAAAQTLQBAAAEE20AAADBRBsAAEAw0QYAABBMtAEAAAQTbQAAAMFEGwAAQDDRBgAAEEy0AQAABBNtAAAAwUQbAABAMNEGAAAQLCLaSilLKaX1MgAAAOJ8t17AuWvhtq5rg5UAAABkiIq2a/YmcGIOAACYQXy07TmPOQEHAACMqttoO+dYJQAAMKohou0aIQcAAIxg2Gi75jLkRBwAAJBuqmi7ZBoHAACkmzrarhFyAABAEtH2AMcqAQCAVkTbC0zjAACAo4i2DxFyAABADaKtIscqAQCAd4m2A5nGAQAAzxJtjQk5AADgFtEWyLFKAADgl2jrgGkcAADMS7R1SsgBAMAcRNtAHKsEAIDxiLaBmcYBAED/RNtkhBwAAPRFtOFYJQBAA6UU+y4eEhFt27Yty7IsPz8/jVfCspjGMY7f17LXLwApru2z4J6IaPv1G297RF07Qo6e3fuA9FoGoDaxxjuiou2evagTc204Vsko9j5IvaYBeIdQ41O6irY9JnQZTOMYjQkdAM8SatQwRLTdI+ra8QVbRibqABBpHGGKaLvHsUugBscuAfomyEgh2m4woQNqMKEDyCXUSCTa3iDqgBpEHQBwTrRV5NglUINjlwAwF9HWgAkdUIMJHQCMSbQFEnVADaIOAPok2jrk2CVQg2OXAJBJtA3EhA6owYQOANoSbRMRdUANog4A6hJt/OHYJVCDY5cA8B7Rxl0mdEANJnQA8BjRxtvuRR3AK0QdAPzvq/UCAAAA2CfaAAAAgok2AACAYKINAAAgmGgDAAAIJtoAAACCiTYAAIBgog0AACCYaAMAAAgm2gAAAIKJNgAAgGCiDQAAIJhoAwAACCbaAAAAgok2AACAYKINAAAgmGgDAAAIJtoAAACCiTYAAIBgog0AACCYaAMAAAgm2gAAAIKJNgAAgGCiDQAAIJhoAwAACCbaAAAAgok2AACAYKINAAAgmGgDAAAIJtoAAACCiTYAAIBgog0AACCYaAMAAAgm2gAAAIKJNgAAgGDfrRdAv06nU+slAAPatu2fP3vWAEda1/Xq35dSDl4J/CXaeIlNFFDD3mYJoLXL55OI40iOR/I0wQbUINiAnqzr6rnFYUQbTxFsQA02PgCwT7QBAAAEE208zJQNqMGUDZiV5x+PchEJ//j5+dn9nWgDXmVjAgCvE22TuRVlt1xewQ1wTpQBQD2ibRCvxtijTNlgTmIM4LZ1XV+6/t/zlWeItnC1Y+wW0zUYl80CwOc8G26ewTxLtDXWMspu8TCBvnkPAxzjdDo9tZ/zj+K8Yg3539zLaC/g1Bi7x0aP0YQ84z7GexQgxzPBtm2br5twUyll90PepO1JvcbYI2wGoR3vP4D+PLIvFGt8gmi7YuQw22PDCPV5nwGM41aIed7zaVNF24wx9ggPFniP9xDAvHwGcIQhok2MvW607xLCJ/kgBmDP6XRy7JHDdBFtoqweDxtmJsoAgB7ERJswO54pGzMQZgBA72Ku/G+9gBnZzAIAQIZbV/5/HbkQAAAAniPaJmXKBgAAfRBtk9i2bVnX9c8PAADQh5iLSKhj2zY3RAIAQMdE24BM0gAAYByibRBCDQAAxuQ7bQMQbAAAMC7R1rHfy0UAAIBxibZOuWAEAADmsJZSWq9hWZYlYhE9MWEDAIBxlFJ2N/gmbR3atq31EgAAgIO4PTKciRoAAMxNtIURaQAAwDnHIxv5vfnx8gcAAOCcSdsBxBgAAPAqk7YKLqdoAAAArzJpe5MoAwAAajJpe5DvoAEAAC2YtN0gygAAgNZE2yLOAACAXNNFm0ADAAB6Mny0iTQAAKBnw1xE4qIQAABgRF1O2sQYAAAwiy6iTaQBAACzios2gQYAAPBXRLQJNQAAgOuGuYgEAABgRKINAAAgmGgDAAAIJtoAAACCiTYAAIBgog0AACCYaAMAAAgm2gAAAIKJNgAAgGCiDQAAIJhoAwAACCbaAAAAgok2AACAYKINAAAgmGgDAAAIJtoAAACCiTYAAIBgog0AACCYaAMAAAgm2gAAAIKJNgAAgGCiDQAAIJhoAwAACCbaAAAAgok2AACAYKINAAAgmGgDAAAIJtoAAACCiTYAAIBgog0AACCYaAMAAAgm2gAAAIKJNgAAgGCiDQAAIJhoAwAACCbaAAAAgq2llNZrAAAAYIdJGwAAQDDRBgAAEEy0AQAABBNtAAAAwUQbAABAMNEGAAAQTLQBAAAEE20AAADBRBsAAEAw0QYAABBMtAEAAAQTbQAAAMFEGwAAQDDRBgAAEEy0AQAABBNtAAAAwUQbAABAMNEGAAAQTLQBAAAEE20AAADBRBsAAEAw0QYAABBMtAEAAAT7DwfHTe3tCsYzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# We have a segmentation for every object instance, showing object bounds without occlusions\n",
    "instances = [4, 5, 7]\n",
    "images = []\n",
    "for instance in instances:\n",
    "    file_instance = '{}/{}/instance_{:03}_{}'.format(\n",
    "        root_path, full_file_name.replace('.jpg', ''), instance, file_name.replace('.jpg', '.png'))\n",
    "    aux = cv2.imread(file_instance)\n",
    "    images.append(aux)\n",
    "plt.figure(figsize=(5*len(images), 5))\n",
    "plt.imshow(np.concatenate(images, 1))\n",
    "plt.axis('off')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally we can get the original polygons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.5, 1023.5, 681.5, -0.5)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 817004 characters]\n",
      "text/plain": [
       "<Figure size 1080x1080 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "img = utils_ade20k.plot_polygon('{}/{}'.format(root_path, full_file_name), info)\n",
    "plt.figure(figsize=(15,15))\n",
    "plt.imshow(img[:, :, ::-1])\n",
    "plt.axis('off')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
