{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "from keras.layers import Dense, Input, GlobalMaxPooling1D, Reshape,LSTM\n",
    "from keras.layers import Conv1D, MaxPooling1D, Embedding, Concatenate, AveragePooling1D\n",
    "from keras.layers import Conv2D, MaxPooling2D, BatchNormalization, Flatten\n",
    "from keras.layers import GlobalMaxPooling2D, Conv3D, MaxPooling3D, ConvLSTM2D, Add\n",
    "from keras.models import Model\n",
    "from keras.initializers import Constant\n",
    "import tensorflow as tf\n",
    "\n",
    "import pickle\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TEXT SECTION OF THE MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_UTTERANCE_LENGTH = 20\n",
    "MAX_NUM_WORDS = 20000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load the data\n",
    "pickle_text = open(\"X_dev_text_utterances.pickle\",\"rb\")\n",
    "X_dev_text_utterances = pickle.load(pickle_text)\n",
    "\n",
    "# finally, vectorize the text samples into a 2D integer tensor\n",
    "tokenizer = Tokenizer(num_words=MAX_NUM_WORDS, filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n\\x97\\x92\\x98\\x85\\x91')\n",
    "tokenizer.fit_on_texts(X_dev_text_utterances)\n",
    "X_dev_text_sequences = tokenizer.texts_to_sequences(X_dev_text_utterances)\n",
    "X_dev_data_text = pad_sequences(X_dev_text_sequences, maxlen=MAX_UTTERANCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_dev_data_text.shape  # <---- This is the data to be fed as input to the NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_dev_embedding_matrix = open(\"X_dev_embedding.pickle\",\"rb\")\n",
    "X_dev_embedding_matrix = pickle.load(X_dev_embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_words = len(X_dev_embedding_matrix)\n",
    "EMBEDDING_DIM = X_dev_embedding_matrix.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1482\n"
     ]
    }
   ],
   "source": [
    "print(num_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load pre-trained word embeddings into an Embedding layer\n",
    "# note that we set trainable = False so as to keep the embeddings fixed\n",
    "embedding_layer = Embedding(num_words,\n",
    "                            EMBEDDING_DIM,\n",
    "                            embeddings_initializer=Constant(X_dev_embedding_matrix),\n",
    "                            input_length=MAX_UTTERANCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 20)                0         \n",
      "_________________________________________________________________\n",
      "embedding_2 (Embedding)      (None, 20, 300)           444600    \n",
      "_________________________________________________________________\n",
      "conv1d_5 (Conv1D)            (None, 16, 64)            96064     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1 (None, 8, 64)             0         \n",
      "_________________________________________________________________\n",
      "conv1d_6 (Conv1D)            (None, 4, 100)            32100     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_3 (Glob (None, 100)               0         \n",
      "=================================================================\n",
      "Total params: 572,764\n",
      "Trainable params: 128,164\n",
      "Non-trainable params: 444,600\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "input_text_ = Input(shape = (MAX_UTTERANCE_LENGTH,), dtype = 'int32')\n",
    "embedded_sequences = embedding_layer(input_text_)\n",
    "output_text_ = Conv1D(64, 5, activation='relu')(embedded_sequences)\n",
    "output_text_ = MaxPooling1D(2)(output_text_)\n",
    "output_text_ = Conv1D(100, 5, activation='relu')(output_text_)\n",
    "output_text_ = GlobalMaxPooling1D()(output_text_) #as in paper 100D textual features\n",
    "\n",
    "model_text = Model(inputs = input_text_, outputs = output_text_)\n",
    "model_text.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FRAME SECTION OF THE MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load frame data\n",
    "pickle_frame = open(\"X_dev_frame_float32.pickle\",\"rb\")\n",
    "X_dev_frame_float32 = pickle.load(pickle_frame)\n",
    "\n",
    "# Normalize Image\n",
    "X_dev_frame_float32 = X_dev_frame_float32/255.0\n",
    "# plt.imshow(X_dev_frame_float32[0][0])\n",
    "print(X_dev_frame_float32.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_frame_ = Input(shape = (10, 84, 150, 3))\n",
    "\n",
    "output_frame_ = ConvLSTM2D(filters=32, kernel_size=(3, 3),\n",
    "                   padding='valid', return_sequences=False)(input_frame_) #To understand more about return_sequences \n",
    "output_frame_ = BatchNormalization()(output_frame_)                                     #https://www.dlology.com/blog/how-to-use-return_state-or-return_sequences-in-keras/\n",
    "output_frame_ = MaxPooling2D((2, 2), strides = 2)(output_frame_)\n",
    "\n",
    "output_frame_ = Conv2D(64, kernel_size=(3, 3))(output_frame_)\n",
    "output_frame_ = BatchNormalization()(output_frame_)\n",
    "output_frame_ = MaxPooling2D((2, 2))(output_frame_)\n",
    "\n",
    "output_frame_ = Conv2D(128, kernel_size=(3, 3))(output_frame_)\n",
    "output_frame_ = BatchNormalization()(output_frame_)\n",
    "output_frame_ = MaxPooling2D((2, 2))(output_frame_)\n",
    "\n",
    "output_frame_ = Conv2D(64, kernel_size=(3, 3))(output_frame_)\n",
    "output_frame_ = BatchNormalization()(output_frame_)\n",
    "output_frame_ = MaxPooling2D((2, 2))(output_frame_)\n",
    "\n",
    "output_frame_ = Conv2D(32, kernel_size=(3, 3))(output_frame_)\n",
    "output_frame_ = BatchNormalization()(output_frame_)\n",
    "# output_frame_ = MaxPooling2D((2, 2))(output_frame_)\n",
    "\n",
    "output_frame_ = Flatten()(output_frame_)\n",
    "\n",
    "# model = Model(inputs = [input_frame_], outputs = output_frame_)\n",
    "# model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AUDIO SECTION OF THE MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load audio mfcc 40D features\n",
    "\n",
    "pickle_audio_mfcc = open(\"X_dev_audio_mfcc.pickle\",\"rb\")\n",
    "X_dev_audio_mfcc = pickle.load(pickle_audio_mfcc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_dev_audio_mfcc = X_dev_audio_mfcc.reshape(1102, 10, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1102, 10, 4)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_dev_audio_mfcc.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_audio_ = Input(shape = (10, 4))\n",
    "output_audio_ = Conv1D(16, 3, activation='relu')(input_audio_)\n",
    "output_audio_ = MaxPooling1D(2)(output_audio_)\n",
    "output_audio_ = Conv1D(32, 3, activation='relu')(output_audio_)\n",
    "output_audio_ = MaxPooling1D(2)(output_audio_)\n",
    "output_audio_ = Reshape((32,))(output_audio_)\n",
    "# model_audio = Model(inputs = input_audio_, outputs = output_audio_)\n",
    "# model_audio.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Concatenate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 279260 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Now concatenate all the layers to make a multimodal\n",
    "concatenate = Concatenate()([output_frame_, output_audio_, output_text_])\n",
    "y_predict_emotions = Dense(7, activation='softmax')(concatenate)\n",
    "y_predict_sentiments = Dense(3, activation='softmax')(concatenate)\n",
    "model = Model(inputs=[input_frame_, input_audio_, input_text_], outputs=[y_predict_emotions, y_predict_sentiments])\n",
    "tf.keras.utils.plot_model(model, to_file='model.png', show_shapes=True, show_layer_names=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 10, 84, 150,  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv_lst_m2d_1 (ConvLSTM2D)     (None, 82, 148, 32)  40448       input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 82, 148, 32)  128         conv_lst_m2d_1[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 41, 74, 32)   0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 39, 72, 64)   18496       max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 39, 72, 64)   256         conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)  (None, 19, 36, 64)   0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 17, 34, 128)  73856       max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 17, 34, 128)  512         conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)  (None, 8, 17, 128)   0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 6, 15, 64)    73792       max_pooling2d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 6, 15, 64)    256         conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2D)  (None, 3, 7, 64)     0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 1, 5, 32)     18464       max_pooling2d_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 1, 5, 32)     128         conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 160)          0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 7)            1127        flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 3)            483         flatten_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 227,946\n",
      "Trainable params: 227,306\n",
      "Non-trainable params: 640\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Y_prediction:  Label section of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle_y_dev = open(\"Y_dev_float32.pickle\",\"rb\")\n",
    "Y_dev_float32 = pickle.load(pickle_y_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1102, 7)\n",
      "(1102, 3)\n"
     ]
    }
   ],
   "source": [
    "Y_dev_float32_emotions = to_categorical(Y_dev_float32[:,0])   #keras.utils\n",
    "Y_dev_float32_sentiments = to_categorical(Y_dev_float32[:,1])\n",
    "\n",
    "print(Y_dev_float32_emotions.shape)\n",
    "print(Y_dev_float32_sentiments.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### If need to inverse the one_hot_encoding\n",
    "from numpy import argmax\n",
    "\n",
    "data = [1, 3, 2, 0, 3, 2, 2, 1, 0, 1]\n",
    "data = array(data)\n",
    "\n",
    "encoded = to_categorical(data) # one hot encoe\n",
    "\n",
    "inverted = argmax(encoded[0]) # invert encoding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# REMEMBER THE OPTIMIZATIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Optimizations => image for audio, separable conv, augmentation, dropout, who is the speaker in the dataset,\n",
    "# Optimizers = adam, etc. (Study), optimize nn using keras tuner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DEFINE MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import ModelCheckpoint, EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])\n",
    "\n",
    "# log_dir= path + \"logs/fit/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "\n",
    "filepath = \"save_checkpoint/\" + \"final_weights_improvement-{epoch:02d}-{val_acc:3f}.hdf5\"\n",
    "\n",
    "early_stop = EarlyStopping(monitor = 'val_acc', min_delta = 0.001, patience = 10, mode = 'max', verbose = 1)\n",
    "checkpoint = ModelCheckpoint(filepath, monitor = 'val_acc', verbose = 1, save_best_only = True, mode = 'max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 826 samples, validate on 276 samples\n",
      "Epoch 1/50\n"
     ]
    }
   ],
   "source": [
    "# fit network\n",
    "history = model.fit([X_dev_frame_float32], [Y_dev_float32_emotions, Y_dev_float32_sentiments],\n",
    "                    epochs = 50,\n",
    "                    batch_size=64,\n",
    "                    validation_split=0.25,\n",
    "                    callbacks = [early_stop])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'plt' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-35-bd0f29ff3e4e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'loss'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'train'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;31m#plt.plot(history.history['val_loss'], label='test')\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlegend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'plt' is not defined"
     ]
    }
   ],
   "source": [
    "plt.plot(history.history['loss'], label='train')\n",
    "#plt.plot(history.history['val_loss'], label='test')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save history pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
