{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KivyDiary  Mine  Qzone\tTwitter\n"
     ]
    }
   ],
   "source": [
    "!ls ../../2021/fetch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The Data Structure"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "{\n",
    "    type: 'freedom', # twitter and so on\n",
    "    date: '2022-11-28   09:30',\n",
    "    content: 'Morning',\n",
    "    images: [image_base64_string1,\n",
    "             image_base64_string2]\n",
    "}\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Common Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dateutil.parser import parse\n",
    "import json\n",
    "\n",
    "import base64\n",
    "import re\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KivyDiary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "diary.txt  Untitled.ipynb\n"
     ]
    }
   ],
   "source": [
    "!ls ../../2021/fetch/KivyDiary/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../2021/fetch/KivyDiary/diary.txt', 'r', encoding='utf-8') as f:\n",
    "    kivy_diary_raw_text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dec 01, 2017 \n",
      "\n",
      "I don't learn everything that I won't use again.\n",
      "\n",
      "——————————————\n",
      "\n",
      "Dec 14, 2017\n",
      "\n",
      "\n",
      "我们这一代人，只会跟着指令走，却从不思考指令背后的含义。\n",
      "\n",
      "——————————————\n",
      "\n",
      "Dec 16, 2017\n",
      "\n",
      "\n",
      "下了 spy camera, 准备长期奋战制作室友游戏粗口鬼畜合集\n",
      "\n",
      "不回击就和平，一\n"
     ]
    }
   ],
   "source": [
    "print(kivy_diary_raw_text[:200])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "parts = kivy_diary_raw_text.split(\"\\n\\n——————————————\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "kivy_diary_days = []\n",
    "for part in parts:\n",
    "    part = part.strip()\n",
    "    lines = part.split(\"\\n\")\n",
    "    date = lines[0]\n",
    "    date = str(parse(date))\n",
    "    text = \"\\n\".join(lines[1:])\n",
    "    kivy_diary_days.append((\"kivydiary\", date, text.strip(), []))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('kivydiary',\n",
       "  '2017-12-01 00:00:00',\n",
       "  \"I don't learn everything that I won't use again.\",\n",
       "  []),\n",
       " ('kivydiary', '2017-12-14 00:00:00', '我们这一代人，只会跟着指令走，却从不思考指令背后的含义。', []),\n",
       " ('kivydiary',\n",
       "  '2017-12-16 00:00:00',\n",
       "  '下了 spy camera, 准备长期奋战制作室友游戏粗口鬼畜合集\\n\\n不回击就和平，一回击我要让它们体验极客的厉害\\n\\n\\n同时也提醒我没能力就没引战',\n",
       "  [])]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kivy_diary_days[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "kivy_diary_messages = kivy_diary_days.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# QZone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MyQzoneData.json  Qzone.ipynb  raw\n"
     ]
    }
   ],
   "source": [
    "!ls ../../2021/fetch/Qzone/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../2021/fetch/Qzone/MyQzoneData.json', 'r', encoding='utf-8') as f:\n",
    "    qzone_raw_text_data = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\n",
      "    {\n",
      "        \"date\": \"2011-02-01 13:05:54\",\n",
      "        \"message\": \"Qiang\"\n",
      "    },\n",
      "    {\n",
      "        \"date\": \"2011-03-03 22:19:02\",\n",
      "        \"message\": \"\\u4f60\\u8981\\u662f\\u5bf9\\u6211\\u4e0d\\u597d\\uff0c\\u6211\n"
     ]
    }
   ],
   "source": [
    "print(qzone_raw_text_data[:200])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "qzone_raw_json_data = json.loads(qzone_raw_text_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'date': '2011-02-01 13:05:54', 'message': 'Qiang'},\n",
       " {'date': '2011-03-03 22:19:02', 'message': '你要是对我不好，我半夜就上你们家唱《忐忑》！！！'},\n",
       " {'date': '2011-03-03 22:22:40', 'message': '还是距离产生美，美人是用看的，不是泡的！'}]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qzone_raw_json_data[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def image_to_base64_string(path):\n",
    "    encoded = base64.b64encode(open(path, \"rb\").read())\n",
    "    return encoded.decode(\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "qzone_days = []\n",
    "for day in qzone_raw_json_data:\n",
    "    date = day[\"date\"]\n",
    "    \n",
    "    if \"images\" in day:\n",
    "        images = day[\"images\"]\n",
    "        images = [image_to_base64_string(os.path.join('../../2021/fetch/Qzone/raw', path)) for path in images]\n",
    "    else:\n",
    "        images = []\n",
    "        \n",
    "    if \"message\" in day:\n",
    "        message = day[\"message\"]\n",
    "    else:\n",
    "        message = \"\"\n",
    "    \n",
    "    if \"comments\" in day:\n",
    "        comments = day[\"comments\"]\n",
    "        comments = re.sub(r\"\\(http(.*?)\\)\", \"\", comments)\n",
    "        comments = re.sub(r\"\\s*\\!\\[\\]\\(图片\\/(.*?)\\)\", \"\", comments)\n",
    "        #comments = re.sub(r\"\\*\\s*\\[yingshaoxo\\]\\s\", \"\", comments)\n",
    "        #comments = re.sub(r\"\\*\\s*\\[.+\\]\\s\", \"\", comments)\n",
    "    else:\n",
    "        comments = \"\"\n",
    "    \n",
    "    text = message + \"\\n\"*4 + comments\n",
    "    text = text.strip()\n",
    "    dictionary = (\"qzone\", date, text, images)\n",
    "    qzone_days.append(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('qzone', '2011-02-01 13:05:54', 'Qiang', []),\n",
       " ('qzone', '2011-03-03 22:19:02', '你要是对我不好，我半夜就上你们家唱《忐忑》！！！', []),\n",
       " ('qzone', '2011-03-03 22:22:40', '还是距离产生美，美人是用看的，不是泡的！', []),\n",
       " ('qzone',\n",
       "  '2011-04-16 16:09:39',\n",
       "  '我坚持我的风格，我活在我的世界；唱反调是我的个行，出奇不意是我的个性；但我还是充满信心！',\n",
       "  []),\n",
       " ('qzone',\n",
       "  '2011-04-17 13:58:39',\n",
       "  '![](http://qzonestyle.gtimg.cn/qzone/em/e300.gif)6489',\n",
       "  [])]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qzone_days[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "qzone_messages = qzone_days.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Twitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " assets   data\t Untitled.ipynb  'Your archive.html'\n"
     ]
    }
   ],
   "source": [
    "!ls ../../2021/fetch/Twitter/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../2021/fetch/Twitter/data/tweet.js\"\n",
    "\n",
    "with open(path, \"r\") as f:\n",
    "    raw_twitter_text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'window.YTD.tweet.part0 = [ {\\n  \"tweet\" : {\\n    \"re'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_twitter_text[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_twitter_text = raw_twitter_text.replace(\"window.YTD.tweet.part0 = \", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[ {\\n  \"tweet\" : {\\n    \"retweeted\" : false,\\n    \"so'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_twitter_text[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_twitter_json_data = json.loads(raw_twitter_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'tweet': {'retweeted': False,\n",
       "   'source': '<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>',\n",
       "   'entities': {'hashtags': [],\n",
       "    'symbols': [],\n",
       "    'user_mentions': [{'name': 'tkasasagi🐻 cookie bear',\n",
       "      'screen_name': 'tkasasagi',\n",
       "      'indices': ['0', '10'],\n",
       "      'id_str': '892732157294030848',\n",
       "      'id': '892732157294030848'}],\n",
       "    'urls': []},\n",
       "   'display_text_range': ['0', '29'],\n",
       "   'favorite_count': '0',\n",
       "   'in_reply_to_status_id_str': '1250427941567094786',\n",
       "   'id_str': '1250504555005083649',\n",
       "   'in_reply_to_user_id': '892732157294030848',\n",
       "   'truncated': False,\n",
       "   'retweet_count': '0',\n",
       "   'id': '1250504555005083649',\n",
       "   'in_reply_to_status_id': '1250427941567094786',\n",
       "   'created_at': 'Wed Apr 15 19:21:33 +0000 2020',\n",
       "   'favorited': False,\n",
       "   'full_text': '@tkasasagi You looks so good!',\n",
       "   'lang': 'en',\n",
       "   'in_reply_to_screen_name': 'tkasasagi',\n",
       "   'in_reply_to_user_id_str': '892732157294030848'}}]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_twitter_json_data[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['../../2021/fetch/Twitter/data/tweet_media/1141165281756114944-D9Y7wkgVAAAFzr_.jpg',\n",
      " '../../2021/fetch/Twitter/data/tweet_media/1067056610579804160-Ds7yUk4X4AEMGOg.jpg',\n",
      " '../../2021/fetch/Twitter/data/tweet_media/1055856045371797504-DqcndoSXgAIJc7w.jpg']\n",
      "\n",
      "../../2021/fetch/Twitter/data/tweet_media/1141165281756114944-D9Y7wkgVAAAFzr_.jpg\n"
     ]
    }
   ],
   "source": [
    "the_twitter_image_folder = '../../2021/fetch/Twitter/data/tweet_media/'\n",
    "the_local_image_list = [os.path.join(the_twitter_image_folder, one) for one in os.listdir(the_twitter_image_folder)]\n",
    "pprint(the_local_image_list[:3])\n",
    "\n",
    "def convert_twitter_image_url_to_base64_string_from_local(url):\n",
    "    # url: http://pbs.twimg.com/media/CV05l-9U4AAXJIy.png\n",
    "    # local path: '../../2021/fetch/Twitter/data/tweet_media/1223675545612738561-EPteWcoVUAAGQiD.jpg'\n",
    "    one_part = url.split('/')[1]\n",
    "    for another_one in the_local_image_list:\n",
    "        if one_part in another_one:\n",
    "            return another_one\n",
    "\n",
    "print()\n",
    "print(convert_twitter_image_url_to_base64_string_from_local(\"http://pbs.twimg.com/media/CV05l-9U4AAXJIy.png\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_dict = {}\n",
    "date_list = []\n",
    "\n",
    "for tweet in raw_twitter_json_data:\n",
    "    tweet = tweet['tweet']\n",
    "    id = tweet[\"id_str\"]\n",
    "    date = str(parse(tweet['created_at'])).split(\"+\")[0]\n",
    "    text = tweet[\"full_text\"]\n",
    "    if \"in_reply_to_status_id_str\" not in tweet:\n",
    "        a_tweet_dict = {\n",
    "            \"date\": date,\n",
    "            \"text\": text,\n",
    "        }\n",
    "        main_dict.update({id: a_tweet_dict})\n",
    "        date_list.append(date)\n",
    "\n",
    "for tweet in raw_twitter_json_data:\n",
    "    tweet = tweet['tweet']\n",
    "    id = tweet[\"id_str\"]\n",
    "    date = str(parse(tweet['created_at'])).split(\"+\")[0]\n",
    "    text = tweet[\"full_text\"]\n",
    "    if \"in_reply_to_status_id_str\" in tweet:\n",
    "        reply_id = tweet[\"in_reply_to_status_id_str\"]\n",
    "        if reply_id in main_dict.keys():\n",
    "            main_dict[reply_id]['text'] += \"\\n\\n\\n\\n\" + text\n",
    "        else:\n",
    "            pass\n",
    "\n",
    "for tweet in raw_twitter_json_data:\n",
    "    tweet = tweet['tweet']\n",
    "    id = tweet[\"id_str\"]\n",
    "    entitie = tweet['entities']\n",
    "    images_within_a_message = []\n",
    "    if 'media' in entitie:\n",
    "        media_list = entitie['media']\n",
    "        for one in media_list:\n",
    "            if 'media_url' in one:\n",
    "                url = one['media_url']\n",
    "                images_within_a_message.append(url)\n",
    "                \n",
    "    images_within_a_message = [\n",
    "        image_to_base64_string(\n",
    "            convert_twitter_image_url_to_base64_string_from_local(one)\n",
    "        ) \n",
    "        for one in images_within_a_message\n",
    "    ]\n",
    "        \n",
    "    if \"in_reply_to_status_id_str\" not in tweet:\n",
    "        main_dict[id].update({\n",
    "            \"images\": images_within_a_message\n",
    "        })\n",
    "    else:\n",
    "        if reply_id in main_dict.keys():\n",
    "            reply_id = tweet[\"in_reply_to_status_id_str\"]\n",
    "            main_dict[reply_id].update({\n",
    "                \"images\": images_within_a_message\n",
    "            })\n",
    "\n",
    "main_list = list(main_dict.values())\n",
    "\n",
    "# take second element for sort\n",
    "def return_date(elem):\n",
    "    return elem[\"date\"]\n",
    "\n",
    "# sort list with key\n",
    "main_list.sort(key=return_date)\n",
    "\n",
    "#pprint(main_list[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_messages = []\n",
    "for item in main_list:\n",
    "    twitter_messages.append(('twitter', item['date'], item['text'], item['images']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('twitter',\n",
       "  '2017-03-05 06:54:06',\n",
       "  \"I just deployed a high performance cloud server on https://t.co/d8GymeVCoH ! It's cheaper anyway!\\n#ILoveVultr #Cloud https://t.co/5IJXDNu0cR\",\n",
       "  []),\n",
       " ('twitter', '2017-04-10 19:53:34', '我决定转战Twitter！', [])]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_messages[1:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Freedom (mine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " database.json\t'load old data.ipynb'   parseCurrentJsonFile.ipynb\n"
     ]
    }
   ],
   "source": [
    "!ls ../../2021/fetch/Mine/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../2021/fetch/Mine/database.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path, 'r', encoding=\"utf-8\") as f:\n",
    "    freedom_json_data = json.loads(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "freedom_json_data = [one for one in freedom_json_data if one['type'] == \"freedom\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'type': 'freedom', 'date': '2020-07-25 05:24:07', 'content': \"You're nothing if you can't stack up to your promise.\", 'images': []}\n"
     ]
    }
   ],
   "source": [
    "print(freedom_json_data[99])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'type': 'freedom',\n",
       " 'date': '2021-03-07 00:26:27',\n",
       " 'content': \"Do what you love to do.\\n\\nThat's the only way that could make you success.\"}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "one_item = freedom_json_data[527].copy()\n",
    "del one_item['images']\n",
    "one_item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "freedom_messages = [('freedom', one['date'], one['content'], one['images']) for one in freedom_json_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('freedom',\n",
       " '2021-05-19 19:55:12',\n",
       " '不小心就被老前辈说成是键盘侠了\\n\\n我不过就是文字功底强一点、表达能力强一点罢了\\n\\n难道非要变成一个个书呆子才行？\\n\\nLeader always loves expression, and people are willing to listen to him.',\n",
       " [])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "freedom_messages[669]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge togather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_messages = kivy_diary_messages + qzone_messages + twitter_messages + freedom_messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('kivydiary',\n",
       " '2020-02-24 00:00:00',\n",
       " 'An ideal company should only requires its employees to work 5-hours per day.\\n\\n理想中的公司，\\n\\n应该是5小时工作制。\\n\\n上午到公司上2小时的班，下午到公司上2小时的班，晚上回家利用业余时间为公司工作1小时。',\n",
       " [])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_messages[89]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_data = []\n",
    "for one in merged_messages:\n",
    "    final_data.append({\n",
    "        'type': one[0],\n",
    "        'date': one[1],\n",
    "        'content': one[2],\n",
    "        'images': one[3],\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('my_data.json', 'w', encoding=\"utf-8\") as f:\n",
    "    f.write(json.dumps(final_data, indent=4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert to txt file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "the_general_seperator = \"\\n\\n\\n__**__**__yingshaoxo_is_the_top_one__**__**__\\n\\n\\n\"\n",
    "\n",
    "the_text = \"\"\n",
    "for one in final_data:\n",
    "    the_text += one[\"content\"] + the_general_seperator\n",
    "\n",
    "with open(\"yingshaoxo_social_media_data.txt\", \"w\", encoding=\"utf-8\") as f:\n",
    "    f.write(the_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}