{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Session 5: 训练情感分类模型 (Train an emotion classifier with keras)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. set up your data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘data’: File exists\n",
      "mkdir: cannot create directory ‘positive’: File exists\n",
      "mkdir: cannot create directory ‘negative’: File exists\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from auto_everything.base import Terminal\n",
    "t = Terminal()\n",
    "\n",
    "commands = \"\"\"\n",
    "mkdir data\n",
    "cd data\n",
    "\n",
    "mkdir positive\n",
    "mkdir negative\n",
    "\n",
    "cd ..\n",
    "cp data.txt data/negative/data.txt\n",
    "cp another_data.txt data/positive/another_data.txt\n",
    "\"\"\"\n",
    "\n",
    "t.run(commands)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. build model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yingshaoxo/.local/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "from pprint import pprint\n",
    "from auto_everything.base import IO\n",
    "io = IO()\n",
    "\n",
    "import jieba\n",
    "import codecs\n",
    "import pickle\n",
    "import random\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.layers.core import Dense\n",
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### define save or load function for reuse var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def __pickleStuff(filename, stuff):\n",
    "    save_stuff = open(filename, \"wb\")\n",
    "    pickle.dump(stuff, save_stuff)\n",
    "    save_stuff.close()\n",
    "    \n",
    "def __loadStuff(filename):\n",
    "    saved_stuff = open(filename,\"rb\")\n",
    "    stuff = pickle.load(saved_stuff)\n",
    "    saved_stuff.close()\n",
    "    return stuff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./data/positive/msgs.txt', './data/positive/another_data.txt']\n",
      "['./data/negative/data.txt', './data/negative/BilibiliComments.txt']\n"
     ]
    }
   ],
   "source": [
    "dataBaseDirPos = \"./data/positive/\"\n",
    "dataBaseDirNeg = \"./data/negative/\"\n",
    "\n",
    "positiveFiles = [dataBaseDirPos + f for f in listdir(dataBaseDirPos) if isfile(join(dataBaseDirPos, f)) and '.txt' in f]\n",
    "negativeFiles = [dataBaseDirNeg + f for f in listdir(dataBaseDirNeg) if isfile(join(dataBaseDirNeg, f)) and '.txt' in f]\n",
    "\n",
    "print(positiveFiles)\n",
    "print(negativeFiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "positive_nums: 7975\n",
      "negative_nums: 10600\n",
      "[('我对垃圾的断绝能力一直很低\\n\\n导致我在现实中经常很不爽\\n\\n\\n\\n\\n要是拒绝可以更坚决一点，就没那么多伤害了', 'pos'),\n",
      " ('喜剧之王 一点都不好看', 'pos'),\n",
      " ('构建一套系统真的没那么容易\\n'\n",
      "  '\\n'\n",
      "  '比如 找工作APP\\n'\n",
      "  '\\n'\n",
      "  '\\n'\n",
      "  '\\n'\n",
      "  '\\n'\n",
      "  '\\n'\n",
      "  '如何构建一个诚信机制，既能让没有任何认证的人找到工作，又不让企业吃亏\\n'\n",
      "  '\\n'\n",
      "  '\\n'\n",
      "  '\\n'\n",
      "  '(淘宝是怎么做的？让人数少的想赚钱的商家交保证金，人数多的消费者不交钱；\\n'\n",
      "  '\\n'\n",
      "  '当把这一套逻辑放在程序员身上，各种问题：\\n'\n",
      "  '\\n'\n",
      "  '万一公司想空手套白狼招人免费干活怎么办？\\n'\n",
      "  '\\n'\n",
      "  '万一有一些傻逼啥都不会恶意给企业差评怎么办？\\n'\n",
      "  '\\n'\n",
      "  '万一有企业派人刷好评洗白怎么办？\\n'\n",
      "  '\\n'\n",
      "  '这些都是问题)',\n",
      "  'pos')]\n"
     ]
    }
   ],
   "source": [
    "documents = []\n",
    "positive_nums = 0\n",
    "negative_nums = 0\n",
    "\n",
    "for filename in positiveFiles:\n",
    "    all_text = io.read(filename)\n",
    "    text_list = all_text.split(\"\\n\\n——————————————\\n\\n\")\n",
    "    for text in text_list:\n",
    "        documents.append((text, \"pos\"))\n",
    "        positive_nums += 1\n",
    "\n",
    "for filename in negativeFiles:\n",
    "    all_text = io.read(filename)\n",
    "    text_list = all_text.split(\"\\n\\n——————————————\\n\\n\")\n",
    "    for text in text_list:\n",
    "        documents.append((text, \"neg\"))\n",
    "        negative_nums += 1\n",
    "\n",
    "print('positive_nums:', positive_nums)\n",
    "print('negative_nums:', negative_nums)\n",
    "pprint(documents[:3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### shuffle data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('看着都肉疼', 'neg'),\n",
      " ('嫂子的店叫什么啊', 'neg'),\n",
      " ('https://github.com/Khan/tota11y\\n\\nKhan 专用 web 渲染库...', 'pos')]\n"
     ]
    }
   ],
   "source": [
    "random.shuffle(documents)\n",
    "\n",
    "pprint(documents[:3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### prepare data for model 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 0.675 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['看着', '都', '肉', '疼']]\n",
      "['neg']\n"
     ]
    }
   ],
   "source": [
    "# Tokenize only\n",
    "totalX = []\n",
    "totalY = [str(document[1]) for document in documents]\n",
    "\n",
    "for document in documents:\n",
    "    seg_list = jieba.cut(document[0], cut_all=False)\n",
    "    seg_list = list(seg_list)\n",
    "    totalX.append(seg_list)\n",
    "    \n",
    "print(totalX[:1])\n",
    "print(totalY[:1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### prepare data for model 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input vocab_size: 40821\n",
      "[[   0    0    0    0    0    0    0    0    0    0    0    0    0  855\n",
      "    19 3219 1774]]\n"
     ]
    }
   ],
   "source": [
    "h = sorted([len(sentence) for sentence in totalX])\n",
    "maxLength = h[int(len(h) * 0.60)]\n",
    "\n",
    "\n",
    "\n",
    "# Keras Tokenizer expect the words tokens to be seperated by space \n",
    "totalX = [\" \".join(wordslist) for wordslist in totalX]\n",
    "\n",
    "input_tokenizer = Tokenizer(30000) # Initial vocab size\n",
    "input_tokenizer.fit_on_texts(totalX)\n",
    "\n",
    "vocab_size = len(input_tokenizer.word_index) + 1\n",
    "print(\"input vocab_size:\",vocab_size)\n",
    "\n",
    "totalX = np.array(pad_sequences(input_tokenizer.texts_to_sequences(totalX), maxlen=maxLength))\n",
    "print(totalX[:1])\n",
    "\n",
    "\n",
    "\n",
    "__pickleStuff(\"./data/input_tokenizer_chinese.p\", input_tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### prepare data for model 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "output vocab_size: 3\n",
      "0 ('好看，中国加油！！！', 'neg')\n"
     ]
    }
   ],
   "source": [
    "target_tokenizer = Tokenizer(3)\n",
    "target_tokenizer.fit_on_texts(totalY)\n",
    "print(\"output vocab_size:\",len(target_tokenizer.word_index) + 1)\n",
    "\n",
    "totalY = np.array(target_tokenizer.texts_to_sequences(totalY)) -1\n",
    "totalY = totalY.reshape(totalY.shape[0])\n",
    "\n",
    "print(totalY[3], documents[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 0.]\n",
      " [1. 0.]\n",
      " [0. 1.]]\n"
     ]
    }
   ],
   "source": [
    "totalY = to_categorical(totalY, num_classes=2)\n",
    "print(totalY[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dimen = totalY.shape[1]\n",
    "\n",
    "target_reverse_word_index = {v: k for k, v in list(target_tokenizer.word_index.items())}\n",
    "sentiment_tag = [target_reverse_word_index[1],target_reverse_word_index[2]] \n",
    "metaData = {\"maxLength\":maxLength,\"vocab_size\":vocab_size,\"output_dimen\":output_dimen,\"sentiment_tag\":sentiment_tag}\n",
    "__pickleStuff(\"./data/meta_sentiment_chinese.p\", metaData)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### main model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = 256\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(vocab_size, embedding_dim,input_length = maxLength))\n",
    "# Each input would have a size of (maxLength x 256) and each of these 256 sized vectors are fed into the GRU layer one at a time.\n",
    "# All the intermediate outputs are collected and then passed on to the second GRU layer.\n",
    "model.add(GRU(256, dropout=0.9, return_sequences=True))\n",
    "# Using the intermediate outputs, we pass them to another GRU layer and collect the final output only this time\n",
    "model.add(GRU(256, dropout=0.9))\n",
    "# The output is then sent to a fully connected layer that would give us our final output_dim classes\n",
    "model.add(Dense(output_dimen, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 16717 samples, validate on 1858 samples\n",
      "Epoch 1/20\n",
      "16717/16717 [==============================] - 110s 7ms/step - loss: 0.4381 - acc: 0.7885 - val_loss: 0.2662 - val_acc: 0.8972\n",
      "Epoch 2/20\n",
      "16717/16717 [==============================] - 107s 6ms/step - loss: 0.2371 - acc: 0.9036 - val_loss: 0.1936 - val_acc: 0.9171\n",
      "Epoch 3/20\n",
      "16717/16717 [==============================] - 107s 6ms/step - loss: 0.1705 - acc: 0.9357 - val_loss: 0.1766 - val_acc: 0.9268\n",
      "Epoch 4/20\n",
      "16717/16717 [==============================] - 109s 7ms/step - loss: 0.1337 - acc: 0.9511 - val_loss: 0.1726 - val_acc: 0.9295\n",
      "Epoch 5/20\n",
      "16717/16717 [==============================] - 109s 7ms/step - loss: 0.1083 - acc: 0.9615 - val_loss: 0.1752 - val_acc: 0.9338\n",
      "Epoch 6/20\n",
      "16717/16717 [==============================] - 109s 7ms/step - loss: 0.0904 - acc: 0.9670 - val_loss: 0.1917 - val_acc: 0.9273\n",
      "Epoch 7/20\n",
      "16717/16717 [==============================] - 110s 7ms/step - loss: 0.0784 - acc: 0.9723 - val_loss: 0.2224 - val_acc: 0.9306\n",
      "Epoch 8/20\n",
      "16717/16717 [==============================] - 110s 7ms/step - loss: 0.0661 - acc: 0.9756 - val_loss: 0.2228 - val_acc: 0.9300\n",
      "Epoch 9/20\n",
      "16717/16717 [==============================] - 110s 7ms/step - loss: 0.0542 - acc: 0.9799 - val_loss: 0.2441 - val_acc: 0.9263\n",
      "Epoch 10/20\n",
      "16717/16717 [==============================] - 110s 7ms/step - loss: 0.0516 - acc: 0.9798 - val_loss: 0.2638 - val_acc: 0.9220\n",
      "Epoch 11/20\n",
      "16717/16717 [==============================] - 110s 7ms/step - loss: 0.0499 - acc: 0.9820 - val_loss: 0.2247 - val_acc: 0.9284\n",
      "Epoch 12/20\n",
      "16717/16717 [==============================] - 110s 7ms/step - loss: 0.0400 - acc: 0.9848 - val_loss: 0.3352 - val_acc: 0.9257\n",
      "Epoch 13/20\n",
      "16717/16717 [==============================] - 110s 7ms/step - loss: 0.0386 - acc: 0.9853 - val_loss: 0.3207 - val_acc: 0.9306\n",
      "Epoch 14/20\n",
      "16717/16717 [==============================] - 110s 7ms/step - loss: 0.0354 - acc: 0.9867 - val_loss: 0.3290 - val_acc: 0.9241\n",
      "Epoch 15/20\n",
      "16717/16717 [==============================] - 111s 7ms/step - loss: 0.0319 - acc: 0.9879 - val_loss: 0.3171 - val_acc: 0.9257\n",
      "Epoch 16/20\n",
      "16717/16717 [==============================] - 111s 7ms/step - loss: 0.0336 - acc: 0.9872 - val_loss: 0.3310 - val_acc: 0.9214\n",
      "Epoch 17/20\n",
      "16717/16717 [==============================] - 111s 7ms/step - loss: 0.0292 - acc: 0.9885 - val_loss: 0.2776 - val_acc: 0.9252\n",
      "Epoch 18/20\n",
      "16717/16717 [==============================] - 111s 7ms/step - loss: 0.0277 - acc: 0.9888 - val_loss: 0.3576 - val_acc: 0.9209\n",
      "Epoch 19/20\n",
      "16717/16717 [==============================] - 111s 7ms/step - loss: 0.0254 - acc: 0.9900 - val_loss: 0.3656 - val_acc: 0.9247\n",
      "Epoch 20/20\n",
      "16717/16717 [==============================] - 111s 7ms/step - loss: 0.0261 - acc: 0.9902 - val_loss: 0.3772 - val_acc: 0.9241\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fe507643f98>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(totalX, totalY, validation_split=0.1, batch_size=32, epochs=20, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('./data/sentiment_chinese_model.H5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "\n",
    "model = load_model('./data/sentiment_chinese_model.H5')\n",
    "\n",
    "metaData = __loadStuff(\"./data/meta_sentiment_chinese.p\")\n",
    "maxLength = metaData.get(\"maxLength\")\n",
    "vocab_size = metaData.get(\"vocab_size\")\n",
    "output_dimen = metaData.get(\"output_dimen\")\n",
    "sentiment_tag = metaData.get(\"sentiment_tag\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def findFeatures(text):\n",
    "    seg_list = jieba.cut(text, cut_all=False)\n",
    "    seg_list = list(seg_list)\n",
    "    text = \" \".join(seg_list)\n",
    "    textArray = [text]\n",
    "    input_tokenizer_load = __loadStuff(\"./data/input_tokenizer_chinese.p\")\n",
    "    textArray = np.array(pad_sequences(input_tokenizer_load.texts_to_sequences(textArray), maxlen=maxLength))\n",
    "    return textArray\n",
    "\n",
    "def predict(text):\n",
    "    if model is None:\n",
    "        print(\"Please run \\\"loadModel\\\" first.\")\n",
    "        return None\n",
    "    features = findFeatures(text)\n",
    "    predicted = model.predict(features)[0] # we have only one sentence to predict, so take index 0\n",
    "    predicted = np.array(predicted)\n",
    "    probab = predicted.max()\n",
    "    predition = sentiment_tag[predicted.argmax()]\n",
    "    return predition, probab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('pos', 0.7405749)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict(\"还好，床很大而且很干净，前台很友好，很满意，下次还来。\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('pos', 0.99951136)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict(\"体验太差\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('pos', 0.9999857)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict(\"hi, i'm yingshaoxo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('neg', 0.99982953)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict(\"说了多少次了！老子不叫塞尔达！那个臭女人才叫塞尔达\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('pos', 0.99990094)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict(\"\"\"这上面讲只有 Open your mind，才更容易学会新知识\n",
    "我同意，因为只有在放长假的时候，我才能不担心忘记学校教的垃圾知识，\n",
    "我才能全身心地投入新知识的学习\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}