author-disambiguation/author_disambiguation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creates text file with the considered criteria for the authors disambiguation\n",
    "import pandas as pd\n",
    "from unidecode import unidecode\n",
    "\n",
    "def normalize_name(name):\n",
    "    normalized_name = unidecode(name)\n",
    "    normalized_name = str.lower(normalized_name)\n",
    "    return normalized_name\n",
    "\n",
    "#Reads in all authors' names and associated dataset metadata\n",
    "df = pd.read_csv(\"/DSKG_BETA_DISAMBIGUATION.csv\")\n",
    "\n",
    "number_id = 10000\n",
    "author_id_names = []\n",
    "author_id_coauthors = []\n",
    "\n",
    "final_list = []\n",
    "\n",
    "i = 0\n",
    "while i < len(df):\n",
    "    authors_title = df['title'][i]\n",
    "    authors_years = df[\"issued\"][i]\n",
    "    authors_years = str(authors_years).split(\"-\")[0]\n",
    "    authors_dataset = df['dataset'][i] \n",
    "    authors_publisher = df['publisherName'][i] \n",
    "    authors_contributor = df['contributorName'][i]\n",
    "    authors_dataset_topic = df['theme'][i]\n",
    "    authors_string = df['creatorPersonName'][i]\n",
    "    authors = str(authors_string).split(\", \")\n",
    "    authors_dataset_lda_topic_distribution = df['LDA_Topic_Distribution'][i]\n",
    "\n",
    "    for author in authors:\n",
    "        if str(author).startswith(\" \"):\n",
    "            max_len = len(author)\n",
    "            author = author[1:max_len]\n",
    "                \n",
    "        author_id = str(number_id) + '\\t' + author\n",
    "        coauthors = str(authors_string).replace(author, \"\")\n",
    "        coauthors = coauthors.replace(\", ,\", \", \")\n",
    "        coauthors_normalized = normalize_name(coauthors)\n",
    "        author_name_normalized = normalize_name(author)\n",
    "        \n",
    "        authors_title_words = authors_title.split(\" \")\n",
    "        authors_title_words_usefull = \"\"\n",
    "        for word in authors_title_words:\n",
    "            if len(word) > 4:\n",
    "                authors_title_words_usefull = authors_title_words_usefull + word + \", \"\n",
    "        max_len = len(authors_title_words_usefull)\n",
    "        unwanted_cut_off = max_len - 2\n",
    "        authors_title_words_usefull = authors_title_words_usefull[0:unwanted_cut_off]\n",
    "  \n",
    "        if coauthors_normalized.startswith(\", \"):\n",
    "            max_len = len(coauthors)\n",
    "            coauthors_normalized = coauthors_normalized[2:max_len]\n",
    "        if coauthors_normalized.endswith(\", \"):\n",
    "            max_len = len(coauthors)\n",
    "            unwanted_cut_off = max_len - 2\n",
    "            coauthors_normalized = coauthors_normalized[0:unwanted_cut_off]\n",
    "        if str(authors_dataset_topic) == \"nan\":\n",
    "            authors_dataset_topic = \"\"\n",
    "        if str(coauthors) == \"nan\":\n",
    "            coauthors = \"\"\n",
    "        if str(authors_publisher) == \"nan\":\n",
    "            authors_publisher = \"\"\n",
    "        if str(authors_contributor) == \"nan\":\n",
    "            authors_contributor = \"\"\n",
    "        if str(authors_years) == \"nan\":\n",
    "            authors_years = \"\"\n",
    "\n",
    "        if str(author) != \"nan\":\n",
    "            final_list.append(author_name_normalized + '\\t' + str(author_id) + '\\t' + str(authors_dataset_topic) + '\\t' + str(authors_dataset) + '\\t' + str(coauthors_normalized) + '\\t' + str(authors_publisher) + '\\t' + str(authors_contributor) + '\\t' + str(authors_years) + '\\t' + str(authors_title_words_usefull) + '\\t' + str(authors_dataset_lda_topic_distribution) + '\\n')\n",
    "            number_id += 1\n",
    "    i += 1\n",
    "\n",
    "final_list.sort()\n",
    "\n",
    "#Saves required criteria for author disambiguation in text file\n",
    "with open(\"/Author_Disambiguation.txt\", \"w\") as outp:\n",
    "    for line in final_list:\n",
    "        outp.write(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Developed Author Disambiguation\n",
    "import pandas as pd\n",
    "from pyjarowinkler import distance\n",
    "import itertools\n",
    "import math\n",
    "import re\n",
    "from scipy import spatial\n",
    "\n",
    "def compare_first_names(author1, author2):\n",
    "    firstName1 = author1.split(\"\\t\")[0].strip().rsplit(' ', 1)[0]\n",
    "    firstName2 = author2.split(\"\\t\")[0].strip().rsplit(' ', 1)[0]\n",
    "    if len(str(firstName1)) >= 5:\n",
    "        if firstName1 == firstName2:\n",
    "            return 2\n",
    "        else:\n",
    "            return 0\n",
    "    else:\n",
    "        if firstName1 == firstName2:\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "        \n",
    "def compare_initials(author1, author2):\n",
    "    names1 = author1.split(\"\\t\")[0].strip().split(\" \")\n",
    "    names2 = author2.split(\"\\t\")[0].strip().split(\" \")\n",
    "    initials1 = \"\"\n",
    "    initials2 = \"\"\n",
    "    for name in names1:\n",
    "        initials1 += name[:1]\n",
    "    for name in names2:\n",
    "        initials2 += name[:1]\n",
    "    if len(initials1) == 1:\n",
    "        if initials1 == initials2:\n",
    "            return 1\n",
    "        else: \n",
    "            return 0\n",
    "    elif len(initials1) == 2:\n",
    "        if initials1 == initials2:\n",
    "            return 2\n",
    "        else:\n",
    "            return 0\n",
    "    else:\n",
    "        if initials1 == initials2:\n",
    "            return 3\n",
    "        else:\n",
    "            return 0\n",
    "\n",
    "def compare_titles(author1, author2):\n",
    "    titles1 = author1.split(\"\\t\")[9].strip().split(\", \")\n",
    "    titles2 = author2.split(\"\\t\")[9].strip().split(\", \")\n",
    "    if len(titles1) == 0 or len(titles2) == 0:\n",
    "        return 0\n",
    "    else:\n",
    "        titles1_set = set(titles1)\n",
    "        titles2_set = set(titles2)\n",
    "        return len(titles1_set.intersection(titles2_set))\n",
    "    \n",
    "def compare_years(author1, author2):\n",
    "    if author1.split(\"\\t\")[8].strip() == \"\" or author2.split(\"\\t\")[8].strip() == \"\":\n",
    "        return False\n",
    "    else:\n",
    "        year1 = author1.split(\"\\t\")[8].strip()\n",
    "        year2 = author2.split(\"\\t\")[8].strip()\n",
    "        return abs(int(year1) - int(year2)) < 10 \n",
    "      \n",
    "def compare_coauthors(author1, author2):\n",
    "    coauthors1 = set(author1.split(\"\\t\")[5].strip().split(\", \"))\n",
    "    coauthors2 = set(author2.split(\"\\t\")[5].strip().split(\", \"))\n",
    "    if len(coauthors1) == 0 or len(coauthors2) == 0:\n",
    "        return 0\n",
    "    else:\n",
    "        return len(coauthors1.intersection(coauthors2))\n",
    "    \n",
    "def compare_topic_datasets(author1, author2): \n",
    "    topics1 = set(author1.split(\"\\t\")[3].strip().split(\", \"))\n",
    "    topics2 = set(author2.split(\"\\t\")[3].strip().split(\", \"))\n",
    "    if len(topics1) == 0 or len(topics2) == 0:\n",
    "        return 0\n",
    "    else:\n",
    "        return len(topics1.intersection(topics2))\n",
    "    \n",
    "#Cosinus-Similarity LDA-Vectors\n",
    "def compare_lda_topic_distribution(author1, author2): \n",
    "    author1 = author1.split(\"\\t\")[10].strip()\n",
    "    author2 = author2.split(\"\\t\")[10].strip()\n",
    "    \n",
    "    author1 = re.sub(r'[0-9], ', '', author1)\n",
    "    author1 = author1.replace(\"[\", \"\").replace(\"]\", \"\").replace(\"(\", \"\").replace(\")\", \"\")\n",
    "    lda_topic_distribution1 = [float(percent) for percent in author1.split(', ')] \n",
    "    \n",
    "    author2 = re.sub(r'[0-9], ', '', author2)\n",
    "    author2 = author2.replace(\"[\", \"\").replace(\"]\", \"\").replace(\"(\", \"\").replace(\")\", \"\")\n",
    "    lda_topic_distribution2 = [float(percent) for percent in author2.split(', ')] \n",
    "    \n",
    "    cos_similarity = 1 - spatial.distance.cosine(lda_topic_distribution1, lda_topic_distribution2)\n",
    "    return cos_similarity\n",
    "\n",
    "def compare_publisher(author1, author2): \n",
    "    publisher1 = set(author1.split(\"\\t\")[6].strip().split(\", \"))\n",
    "    publisher2 = set(author2.split(\"\\t\")[6].strip().split(\", \"))\n",
    "    if len(publisher1) == 0 or len(publisher2) == 0:\n",
    "        return 0\n",
    "    else:\n",
    "        return len(publisher1.intersection(publisher2))\n",
    "    \n",
    "def compare_contributer(author1, author2):  \n",
    "    contributer1 = set(author1.split(\"\\t\")[7].strip().split(\", \"))\n",
    "    contributer2 = set(author2.split(\"\\t\")[7].strip().split(\", \"))\n",
    "    if len(contributer1) < 3 or len(contributer2) < 3:\n",
    "        return 0\n",
    "    else:\n",
    "        return len(contributer1.intersection(contributer2))\n",
    "    \n",
    "def compare_authors(author1, author2): \n",
    "    score = 0\n",
    "    \n",
    "    if compare_initials(author1, author2) == 2:\n",
    "        score += 3 \n",
    "    elif compare_initials(author1, author2) == 3:\n",
    "        score += 7\n",
    "    elif compare_initials(author1, author2) == 0:\n",
    "        score -= 5\n",
    "    \n",
    "    if compare_first_names(author1, author2) == 1:\n",
    "        score += 2\n",
    "    elif compare_first_names(author1, author2) == 2:\n",
    "        score += 5\n",
    "    \n",
    "    if compare_coauthors(author1, author2) == 1:\n",
    "        score += 4\n",
    "    elif compare_coauthors(author1, author2) == 2:\n",
    "        score += 7\n",
    "    elif compare_coauthors(author1, author2) > 2:\n",
    "        score += 10\n",
    "        \n",
    "    if compare_titles(author1, author2) == 1:\n",
    "        score += 2\n",
    "    elif compare_titles(author1, author2) == 2:\n",
    "        score += 4 \n",
    "    elif compare_titles(author1, author2) > 2:\n",
    "        score += 6 \n",
    "        \n",
    "    if compare_lda_topic_distribution(author1, author2) >= 0.99:\n",
    "        score += 4  \n",
    "    elif compare_lda_topic_distribution(author1, author2) >= 0.95:\n",
    "        score += 3 \n",
    "    elif compare_lda_topic_distribution(author1, author2) >= 0.75:\n",
    "        score += 2 \n",
    "\n",
    "    if compare_years(author1, author2):\n",
    "        score += 1\n",
    "        \n",
    "    if compare_topic_datasets(author1, author2) >= 1:\n",
    "        score += 1\n",
    " \n",
    "    if compare_publisher(author1, author2) >= 1:\n",
    "        score += 1\n",
    "      \n",
    "    if compare_contributer(author1, author2) >= 1:\n",
    "        score += 3\n",
    "\n",
    "    return score\n",
    "\n",
    "def get_id(author):\n",
    "    return author.split(\"\\t\")[1]\n",
    "\n",
    "def add_to_mapping(dict_of_maps, entry1, entry2):\n",
    "    if entry2 not in dict_of_maps:\n",
    "        dict_of_maps[entry1] = entry2\n",
    "        return dict_of_maps\n",
    "    else:\n",
    "        return add_to_mapping(dict_of_maps, entry1, dict_of_maps[entry2])\n",
    "    \n",
    "def merge_authors(tuple_of_authors):\n",
    "    author1 = tuple_of_authors[0].strip(\"\\n\").split(\"\\t\")\n",
    "    author2 = tuple_of_authors[1].strip(\"\\n\").split(\"\\t\")\n",
    "    output = \"\\t\".join(author1[0:11])\n",
    "    return output\n",
    "\n",
    "def disambiguate(list_of_authors, result, positive, negative):\n",
    "    author_dictionary = {get_id(author): author.strip(\"\\n\") for author in list_of_authors}\n",
    "    author_list = [get_id(author) for author in list_of_authors]\n",
    "    mapping = {}\n",
    "    result = result.copy()\n",
    "    #Creates tuples of two authors each \n",
    "    comparisons = list(itertools.combinations(author_list, 2)) \n",
    "    for item in comparisons:\n",
    "        try:\n",
    "            if compare_authors(author_dictionary[item[0]], author_dictionary[item[1]]) >= 11:\n",
    "                positive += 1\n",
    "                if item[0] not in mapping:\n",
    "                    mapping = add_to_mapping(mapping, item[1], item[0])\n",
    "                    result = add_to_mapping(result, item[1], item[0])\n",
    "                    author_dictionary[item[0]] = merge_authors((author_dictionary[item[0]], author_dictionary[item[1]]))\n",
    "                    del author_dictionary[item[1]] \n",
    "                else:\n",
    "                    author_dictionary[mapping[item[0]]] = merge_authors((author_dictionary[mapping[item[0]]], author_dictionary[item[1]]))\n",
    "                    mapping = add_to_mapping(mapping, item[1], item[0])\n",
    "                    result = add_to_mapping(result, item[1], item[0])\n",
    "                    del author_dictionary[item[1]]\n",
    "            else:\n",
    "                negative += 1\n",
    "        except KeyError:\n",
    "            pass \n",
    "    if not len(mapping) == 0:\n",
    "        return disambiguate([author_dictionary[author] for author in author_dictionary], result, positive, negative)\n",
    "    else:\n",
    "        return author_dictionary, result, positive, negative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Execute the author disambiguation\n",
    "with open(\"/Author_Disambiguation.txt\", \"r\") as inp: \n",
    "    with open(\"/results_summary.txt\", \"w\") as outp:\n",
    "        with open(\"/all_positives.txt\", \"w\") as outp2:\n",
    "            with open(\"/disambiguated_file.txt\", \"w\") as outp3:\n",
    "                positive = 0\n",
    "                negative = 0\n",
    "                previous_name = \"\"\n",
    "                current_authors = []\n",
    "                line_count = 1\n",
    "\n",
    "                for line in inp:\n",
    "                    name = line.split(\"\\t\")[0].strip()\n",
    "                    if previous_name == \"\":\n",
    "                        previous_name = name\n",
    "                        current_authors.append(line)\n",
    "                    elif distance.get_jaro_distance(str.lower(name), str.lower(previous_name), winkler=True, scaling=0.1) > 0.9:\n",
    "                        previous_name = name\n",
    "                        current_authors.append(line)\n",
    "                    else:\n",
    "                        result = {}\n",
    "                        authors, result, positive, negative = disambiguate(current_authors, result, positive, negative)\n",
    "                        previous_name = name\n",
    "                        current_authors = [line]\n",
    "                        for item in authors:\n",
    "                            outp3.write(authors[item] + \"\\n\")\n",
    "                        for item in result:\n",
    "                            outp2.write(item + \"\\t\" + result[item] + \"\\n\")\n",
    "                    line_count += 1\n",
    "                result = {} \n",
    "                authors, result, positive, negative = disambiguate(current_authors, result, positive, negative)\n",
    "                for item in authors:\n",
    "                    outp3.write(authors[item] + \"\\n\")\n",
    "                for item in result:\n",
    "                    outp2.write(item + \"\\t\" + result[item] + \"\\n\")\n",
    "\n",
    "        total_comparisons = positive + negative\n",
    "        outp.write(\"Total comparisons: \" + str(total_comparisons) + \"\\n\")\n",
    "        outp.write(\"Total positives: \" + str(positive) + \": \" + str(positive/total_comparisons) + \"\\n\")\n",
    "        outp.write(\"Total negatives: \" + str(negative) + \": \" + str(negative/total_comparisons))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}