{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bbf5fc11-cd5b-41f8-90ab-f4a3162e264f",
   "metadata": {},
   "source": [
    "# Unobserved mentions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ac476b09-2a68-4031-bf94-801605aa961f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pathlib\n",
    "\n",
    "# wiki, version, langcode = 'simplewiki', '20211120', 'en'\n",
    "# wiki, version, langcode = 'tawiki', '20220301', 'ta'\n",
    "# wiki, version, langcode = 'fawiki', '20220301', 'fa'\n",
    "# wiki, version, langcode = 'trwiki', '20220301', 'tr'\n",
    "wiki, version, langcode = 'nlwiki', '20220301', 'nl'\n",
    "# wiki, version, langcode = 'arwiki', '20220301', 'ar'\n",
    "# wiki, version, langcode = 'srwiki', '20220301', 'sr'\n",
    "# wiki, version, langcode = 'eswiki', '20220301', 'es'\n",
    "# wiki, version, langcode = 'jawiki', '20220301', 'ja'\n",
    "# wiki, version, langcode = 'dewiki', '20220301', 'de'\n",
    "\n",
    "fgold = pathlib.Path(f\"evaluation/Mewsli-9/{langcode}.tsv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3a8e22a0-9da3-4bbf-83f0-2995c696532f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>surface</th>\n",
       "      <th>ref</th>\n",
       "      <th>count</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tangerang</td>\n",
       "      <td>10127</td>\n",
       "      <td>124</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>kortsluiting</td>\n",
       "      <td>206907</td>\n",
       "      <td>115</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bunq</td>\n",
       "      <td>24084541</td>\n",
       "      <td>13</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Mosoel</td>\n",
       "      <td>83317</td>\n",
       "      <td>250</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Abadi</td>\n",
       "      <td>5639013</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11341</th>\n",
       "      <td>VN-Veiligheidsraad</td>\n",
       "      <td>37470</td>\n",
       "      <td>1721</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11342</th>\n",
       "      <td>Verenigde Naties</td>\n",
       "      <td>1065</td>\n",
       "      <td>4826</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11343</th>\n",
       "      <td>Paramacaanse</td>\n",
       "      <td>2105235</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11344</th>\n",
       "      <td>Langatabbetje</td>\n",
       "      <td>2782195</td>\n",
       "      <td>14</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11345</th>\n",
       "      <td>Ceuta</td>\n",
       "      <td>5823</td>\n",
       "      <td>257</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11346 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  surface       ref  count  rank\n",
       "0               Tangerang     10127    124  -1.0\n",
       "1            kortsluiting    206907    115   0.0\n",
       "2                    Bunq  24084541     13  -1.0\n",
       "3                  Mosoel     83317    250  -1.0\n",
       "4                   Abadi   5639013      0   NaN\n",
       "...                   ...       ...    ...   ...\n",
       "11341  VN-Veiligheidsraad     37470   1721  -1.0\n",
       "11342    Verenigde Naties      1065   4826   0.0\n",
       "11343        Paramacaanse   2105235      0   NaN\n",
       "11344       Langatabbetje   2782195     14  -1.0\n",
       "11345               Ceuta      5823    257   0.0\n",
       "\n",
       "[11346 rows x 4 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import json, glob, dawg\n",
    "from minimel.normalize import normalize\n",
    "\n",
    "counts = json.load(open(f\"wiki/{wiki}-{version}/count.min2.json\"))\n",
    "disambig = set(pd.read_csv(\"data/wikidata-20211122-disambig.txt\", header=None)[0])\n",
    "index = glob.glob(f\"wiki/{wiki}-*/index_{wiki}-*.dawg\")[0]\n",
    "wm = dawg.IntDAWG().load(index)\n",
    "\n",
    "\n",
    "counts = {k:{a.replace('Q',''):b for a,b in v.items()} for k,v in counts.items()}\n",
    "\n",
    "def rank(wm, counts, surface, i, stem=None):\n",
    "    \"\"\"Return count and rank (rank = -1 means only in page title index)\"\"\"\n",
    "    for norm in normalize(surface, language=stem):\n",
    "        norm = norm.lower()\n",
    "        ent_count = counts.get(norm, {})\n",
    "        if len(ent_count) == 1 and ent_count.get(\"%s\" % i):\n",
    "            return ent_count[\"%s\" % i], -1\n",
    "        if ent_count.get(\"%s\" % i):\n",
    "            order = sorted(ent_count, key=lambda x: -ent_count[x])\n",
    "            return ent_count[\"%s\" % i], order.index(\"%s\" % i)\n",
    "    if wm.get(surface[0].upper() + surface[1:].replace(\" \", \"_\")) == i:\n",
    "        return 1, -1\n",
    "    return 0, None\n",
    "\n",
    "\n",
    "ranks = pd.DataFrame(\n",
    "    [\n",
    "        (surface, ref, *rank(wm, counts, surface, ref))\n",
    "        for _, links, text in (l.split(\"\\t\") for l in fgold.open())\n",
    "        for surface, ref in json.loads(links).items()\n",
    "        if ref not in disambig\n",
    "    ],\n",
    "    columns=[\"surface\", \"ref\", \"count\", \"rank\"],\n",
    ")\n",
    "ranks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d4787345-56cc-4c7a-b1f1-02824612da5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>surface</th>\n",
       "      <th>ref</th>\n",
       "      <th>count</th>\n",
       "      <th>rank</th>\n",
       "      <th>test_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>327</th>\n",
       "      <td>Nations League</td>\n",
       "      <td>29000951</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365</th>\n",
       "      <td>World Solar Challenge</td>\n",
       "      <td>14520244</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429</th>\n",
       "      <td>Abadi</td>\n",
       "      <td>5639013</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>457</th>\n",
       "      <td>Le Pen</td>\n",
       "      <td>12927</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>635</th>\n",
       "      <td>Trudeau</td>\n",
       "      <td>3099714</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>703</th>\n",
       "      <td>Wereldkampioenschap voetbal in Mexico</td>\n",
       "      <td>46938</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755</th>\n",
       "      <td>VK.com</td>\n",
       "      <td>116933</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>772</th>\n",
       "      <td>haat in Brussel</td>\n",
       "      <td>23365300</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>886</th>\n",
       "      <td>Mattarella</td>\n",
       "      <td>3956186</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1010</th>\n",
       "      <td>brug</td>\n",
       "      <td>2222042</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1034</th>\n",
       "      <td>Litvinenko</td>\n",
       "      <td>87015</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1058</th>\n",
       "      <td>Team BMC</td>\n",
       "      <td>787401</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1076</th>\n",
       "      <td>Tweede Kamerverkiezingen van 15 maart</td>\n",
       "      <td>16061881</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1082</th>\n",
       "      <td>getaserd</td>\n",
       "      <td>870114</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1149</th>\n",
       "      <td>TUI fly</td>\n",
       "      <td>664509</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1151</th>\n",
       "      <td>MOAS</td>\n",
       "      <td>18209263</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1194</th>\n",
       "      <td>tsunami</td>\n",
       "      <td>60042839</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1208</th>\n",
       "      <td>vale gieren</td>\n",
       "      <td>177856</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1217</th>\n",
       "      <td>Amerikaanse presidentschap</td>\n",
       "      <td>45578</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1281</th>\n",
       "      <td>NPB</td>\n",
       "      <td>2113973</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    surface       ref  count rank  test_count\n",
       "327                          Nations League  29000951      0   na           4\n",
       "365                   World Solar Challenge  14520244      0   na           4\n",
       "429                                   Abadi   5639013      0   na           3\n",
       "457                                  Le Pen     12927      0   na           3\n",
       "635                                 Trudeau   3099714      0   na           3\n",
       "703   Wereldkampioenschap voetbal in Mexico     46938      0   na           2\n",
       "755                                  VK.com    116933      0   na           2\n",
       "772                         haat in Brussel  23365300      0   na           2\n",
       "886                              Mattarella   3956186      0   na           2\n",
       "1010                                   brug   2222042      0   na           2\n",
       "1034                             Litvinenko     87015      0   na           2\n",
       "1058                               Team BMC    787401      0   na           2\n",
       "1076  Tweede Kamerverkiezingen van 15 maart  16061881      0   na           2\n",
       "1082                               getaserd    870114      0   na           2\n",
       "1149                                TUI fly    664509      0   na           2\n",
       "1151                                   MOAS  18209263      0   na           2\n",
       "1194                                tsunami  60042839      0   na           2\n",
       "1208                            vale gieren    177856      0   na           2\n",
       "1217             Amerikaanse presidentschap     45578      0   na           2\n",
       "1281                                    NPB   2113973      0   na           2"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_counts = ranks.fillna(\"na\").value_counts().rename(\"test_count\").reset_index()\n",
    "test_counts.query('(count == 0) & (rank == \"na\")').head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b9b3967-3a0b-4a07-80da-ffeb8e27adbc",
   "metadata": {},
   "source": [
    "## Hardest entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "cc5ac0f5-853f-4f14-8169-153d7940362c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ref</th>\n",
       "      <th>count</th>\n",
       "      <th>rank</th>\n",
       "      <th>test_count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>surface</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Openbaar Ministerie</th>\n",
       "      <td>2595790</td>\n",
       "      <td>553</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Openbaar Ministerie</th>\n",
       "      <td>11775750</td>\n",
       "      <td>312</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>China</th>\n",
       "      <td>148</td>\n",
       "      <td>6406</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>China</th>\n",
       "      <td>29520</td>\n",
       "      <td>8211</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Peter Madsen</th>\n",
       "      <td>2076849</td>\n",
       "      <td>10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Koninginnedag</th>\n",
       "      <td>2598945</td>\n",
       "      <td>10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Koninginnedag</th>\n",
       "      <td>333016</td>\n",
       "      <td>212</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kodiak</th>\n",
       "      <td>79985</td>\n",
       "      <td>21</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kos</th>\n",
       "      <td>568212</td>\n",
       "      <td>33</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kirkoek</th>\n",
       "      <td>193268</td>\n",
       "      <td>11</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>422 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          ref  count rank  test_count\n",
       "surface                                              \n",
       "Openbaar Ministerie   2595790    553  0.0          19\n",
       "Openbaar Ministerie  11775750    312  1.0           6\n",
       "China                     148   6406  1.0           5\n",
       "China                   29520   8211  0.0           5\n",
       "Peter Madsen          2076849     10  1.0           5\n",
       "...                       ...    ...  ...         ...\n",
       "Koninginnedag         2598945     10  1.0           1\n",
       "Koninginnedag          333016    212  0.0           1\n",
       "Kodiak                  79985     21  1.0           1\n",
       "Kos                    568212     33  1.0           1\n",
       "Kirkoek                193268     11  1.0           1\n",
       "\n",
       "[422 rows x 4 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hard_ents = test_counts.query(\"rank==1\")[\"surface\"]\n",
    "test_counts.set_index(\"surface\").loc[list(hard_ents)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b77b915-6a5e-4a68-ac4d-2ac8c168d7e1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}