{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"./data/tweet.js\"\n",
    "\n",
    "with open(path, \"r\") as f:\n",
    "    raw_text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_text = raw_text.replace(\"window.YTD.tweet.part0 = \", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from pprint import pprint\n",
    "from dateutil.parser import parse\n",
    "\n",
    "data = json.loads(raw_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for tweet in data:\n",
    "    tweet = tweet['tweet']\n",
    "    id = tweet[\"id_str\"]\n",
    "    date = str(parse(tweet['created_at'])).split(\"+\")[0]\n",
    "    text = tweet[\"full_text\"]\n",
    "    pprint(tweet)\n",
    "    print()\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_dict = {}\n",
    "date_list = []\n",
    "\n",
    "for tweet in data:\n",
    "    tweet = tweet['tweet']\n",
    "    id = tweet[\"id_str\"]\n",
    "    date = str(parse(tweet['created_at'])).split(\"+\")[0]\n",
    "    text = tweet[\"full_text\"]\n",
    "    if \"in_reply_to_status_id_str\" not in tweet:\n",
    "        a_tweet_dict = {\n",
    "            \"date\": date,\n",
    "            \"text\": text,\n",
    "        }\n",
    "        main_dict.update({id: a_tweet_dict})\n",
    "        date_list.append(date)\n",
    "\n",
    "for tweet in data:\n",
    "    tweet = tweet['tweet']\n",
    "    id = tweet[\"id_str\"]\n",
    "    date = str(parse(tweet['created_at'])).split(\"+\")[0]\n",
    "    text = tweet[\"full_text\"]\n",
    "    if \"in_reply_to_status_id_str\" in tweet:\n",
    "        reply_id = tweet[\"in_reply_to_status_id_str\"]\n",
    "        if reply_id in main_dict.keys():\n",
    "            main_dict[reply_id]['text'] += \"\\n\\n\\n\\n\" + text\n",
    "        else:\n",
    "            pass\n",
    "        \n",
    "main_list = list(main_dict.values())\n",
    "\n",
    "# take second element for sort\n",
    "def return_date(elem):\n",
    "    return elem[\"date\"]\n",
    "\n",
    "# sort list with key\n",
    "main_list.sort(key=return_date)\n",
    "\n",
    "pprint(main_list[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for oneday in main_list[-5:]:\n",
    "    date = oneday[\"date\"]\n",
    "    text = oneday[\"text\"]\n",
    "    print(text)\n",
    "    print(\"\\n\\n----------------\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm tweets.db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sqlite3\n",
    "\n",
    "class Self():\n",
    "    _root_path = \".\"\n",
    "    pass\n",
    "\n",
    "self = Self()\n",
    "self.SQL_DATA_FILE = os.path.join(self._root_path, \"tweets.db\")\n",
    "if not os.path.exists(self.SQL_DATA_FILE):\n",
    "    print(f\"SQL doesn't exist: {self.SQL_DATA_FILE}\")\n",
    "    print(f\"We'll create a new one!\")\n",
    "self._sql_conn = sqlite3.connect(self.SQL_DATA_FILE, check_same_thread=False)\n",
    "self._sql_cursor = self._sql_conn.cursor()\n",
    "self._sql_cursor.execute('''CREATE TABLE IF NOT EXISTS thoughts\n",
    "            (date TEXT, type TEXT, text TEXT, images TEXT)''')\n",
    "\n",
    "for oneday in main_list:\n",
    "    date = oneday[\"date\"]\n",
    "    text = oneday[\"text\"]\n",
    "    day = (date, \"twitter\", text, json.dumps([]))\n",
    "    self._sql_cursor.execute(\"INSERT INTO thoughts VALUES (?,?,?,?)\", day)\n",
    "    self._sql_conn.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}