3_fetch_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "watermark 1.8.1\n",
      "numpy     1.15.4\n",
      "pandas    0.23.4\n",
      "ELEANOR LUTZ 2019-06-03 \n",
      "\n",
      "CPython 3.7.1\n",
      "IPython 7.2.0\n",
      "\n",
      "compiler   : MSC v.1900 64 bit (AMD64)\n",
      "system     : Windows\n",
      "release    : 10\n",
      "machine    : AMD64\n",
      "processor  : Intel64 Family 6 Model 63 Stepping 2, GenuineIntel\n",
      "CPU cores  : 12\n",
      "interpreter: 64bit\n"
     ]
    }
   ],
   "source": [
    "# Watermark is not required for this code, but is included for information. \n",
    "import watermark\n",
    "%load_ext watermark\n",
    "%watermark -a \"ELEANOR LUTZ\" -d -v -iv -m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time, urllib, os.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_link(objid, start_time, stop_time, stepsize, center):\n",
    "    link = \"https://ssd.jpl.nasa.gov/horizons_batch.cgi?batch=1\"\n",
    "    link += \"&COMMAND='\"+ str(objid) +\"'\"\n",
    "    link += \"&MAKE_EPHEM='YES'\"\n",
    "    link += \"&TABLE_TYPE='VECTORS'\"\n",
    "    link += \"&START_TIME='\"+ start_time +\"'\"\n",
    "    link += \"&STOP_TIME='\"+ stop_time +\"'\"\n",
    "    link += \"&STEP_SIZE='\"+ stepsize + \"'\"\n",
    "    link += \"&OUT_UNITS='KM-S'\"\n",
    "    link += \"&REF_PLANE='ECLIPTIC'\"\n",
    "    link += \"&REF_SYSTEM='J2000'\"\n",
    "    link += \"&VEC_LABELS='YES'\"\n",
    "    link += \"&VEC_DELTA_T='NO'\"\n",
    "    link += \"&OBJ_DATA='YES'\"\n",
    "    link += \"&VEC_TABLE='1'\"\n",
    "    link += \"&VECT_CORR='NONE'\"\n",
    "    link += \"&CSV_FORMAT='YES'\"\n",
    "    link += \"&CENTER='\"+ center + \"'\"\n",
    "    return(link)\n",
    "\n",
    "def get_data(link, savename, objid):\n",
    "    values = {'name':'Eleanor Lutz', 'location':'Seattle WA USA', 'language':'Python 3.7.1'}\n",
    "    headers = { 'User-Agent':\"Atlas_of_Space_bot Eleanor Lutz contact with issues at tabletopwhale@outlook.com\" }\n",
    "    data = urllib.parse.urlencode(values)\n",
    "    \n",
    "    if (values.get(\"name\") == \"Eleanor Lutz\") or (\"Eleanor Lutz\" in headers.get(\"User-Agent\")): \n",
    "        raise ValueError(\"Please edit the scraper with your own contact information\")\n",
    "    \n",
    "    if not os.path.isfile(savename):\n",
    "        try:\n",
    "            output = urllib.request.urlopen(link, data, headers)\n",
    "            output = [x.decode('UTF-8') for x in output]\n",
    "            output = [x.strip() for x in output]\n",
    "\n",
    "            if '$$SOE' in output:\n",
    "                header = output[output.index('$$SOE')-2].split(',')\n",
    "                header = [x.lstrip() for x in header]\n",
    "\n",
    "                content = output[output.index('$$SOE')+1 : output.index('$$EOE')-1]\n",
    "                content = [x.split(',') for x in content]\n",
    "\n",
    "                df = pd.DataFrame(content, columns=header)\n",
    "                df.to_csv(savename, index=False)\n",
    "            else: \n",
    "                print(objid, 'request successful but output not expected format')\n",
    "                print(link)\n",
    "        except:\n",
    "            print(objid, 'request unsuccessful')\n",
    "            print(link)\n",
    "\n",
    "def query_horizons(readname, savename_head, stepsize='1d', center='@sun'):\n",
    "    if not os.path.isfile(savename_head[:-1]+\".zip\"):\n",
    "        print(\"---\\nNow analyzing\", readname)\n",
    "        df = pd.read_csv(readname, low_memory=False)\n",
    "\n",
    "        # Check for duplicated names before running script\n",
    "        dupl = df.duplicated('horizons')\n",
    "        print(sum(dupl), 'names in the series are duplicated, of', len(df), 'total')\n",
    "        if sum(dupl) > 1:\n",
    "            print(df[dupl == True])\n",
    "\n",
    "        # Check to see if any names are NaN values\n",
    "        print(df['horizons'].astype(str).replace(' ', '').isnull().sum(), \n",
    "                  'null values in horizons query list')\n",
    "\n",
    "        for index, row in df.iterrows():\n",
    "            if (index % 500 == 0) and (index != 0):\n",
    "                print(index, 'items analyzed!')\n",
    "            start_time = str(row['begin_time']).replace(' ', '')\n",
    "            stop_time = str(row['end_time']).replace(' ', '')\n",
    "            objid = str(row['horizons']).replace(' ', '')\n",
    "            savename = savename_head + objid + '.csv'\n",
    "\n",
    "            if not os.path.isfile(savename):\n",
    "                link = get_link(objid, start_time, stop_time, stepsize, center)\n",
    "                get_data(link, savename, objid)\n",
    "\n",
    "                # sleep to be polite to HORIZONS servers\n",
    "                sleeptime = min(20, max(1, np.random.normal(loc=10, scale=5)))\n",
    "                time.sleep(sleeptime) \n",
    "\n",
    "                if index == 0:\n",
    "                    # Save parameters of 1st item to txt file for later checking\n",
    "                    txtname = savename_head + 'PARAMETERS.txt'\n",
    "                    with open(txtname, \"w\") as f:\n",
    "                        f.write(link)\n",
    "    else: \n",
    "        raise ValueError(\"Please unzip the data files that already exist\")\n",
    "    print('ALL ITEMS ANALYZED!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\n",
      "Now analyzing ./data/moons.csv\n",
      "0 names in the series are duplicated, of 185 total\n",
      "0 null values in horizons query list\n",
      "2009s1 request unsuccessful\n",
      "https://ssd.jpl.nasa.gov/horizons_batch.cgi?batch=1&COMMAND='2009s1'&MAKE_EPHEM='YES'&TABLE_TYPE='VECTORS'&START_TIME='1970-07-30-00-00-23'&STOP_TIME='2000-01-01-00-00-00'&STEP_SIZE='1d'&OUT_UNITS='KM-S'&REF_PLANE='ECLIPTIC'&REF_SYSTEM='J2000'&VEC_LABELS='YES'&VEC_DELTA_T='NO'&OBJ_DATA='YES'&VEC_TABLE='1'&VECT_CORR='NONE'&CSV_FORMAT='YES'&CENTER='@sun'\n",
      "ALL ITEMS ANALYZED!\n",
      "---\n",
      "Now analyzing ./data/planets.csv\n",
      "0 names in the series are duplicated, of 8 total\n",
      "0 null values in horizons query list\n",
      "ALL ITEMS ANALYZED!\n",
      "---\n",
      "Now analyzing ./data/large_asteroids.csv\n",
      "0 names in the series are duplicated, of 2714 total\n",
      "0 null values in horizons query list\n",
      "500 items analyzed!\n",
      "1000 items analyzed!\n",
      "1500 items analyzed!\n",
      "2000 items analyzed!\n",
      "2500 items analyzed!\n",
      "ALL ITEMS ANALYZED!\n",
      "---\n",
      "Now analyzing ./data/large_comets.csv\n",
      "0 names in the series are duplicated, of 12 total\n",
      "0 null values in horizons query list\n",
      "ALL ITEMS ANALYZED!\n",
      "---\n",
      "Now analyzing ./data/small_asteroids.csv\n",
      "0 names in the series are duplicated, of 7468 total\n",
      "0 null values in horizons query list\n",
      "500 items analyzed!\n",
      "1000 items analyzed!\n",
      "1500 items analyzed!\n",
      "2000 items analyzed!\n",
      "2500 items analyzed!\n",
      "3000 items analyzed!\n",
      "3500 items analyzed!\n",
      "4000 items analyzed!\n",
      "4500 items analyzed!\n",
      "5000 items analyzed!\n",
      "5500 items analyzed!\n",
      "6000 items analyzed!\n",
      "6500 items analyzed!\n",
      "7000 items analyzed!\n",
      "ALL ITEMS ANALYZED!\n",
      "---\n",
      "Now analyzing ./data/any_outer_asteroids.csv\n",
      "0 names in the series are duplicated, of 5000 total\n",
      "0 null values in horizons query list\n",
      "500 items analyzed!\n",
      "1000 items analyzed!\n",
      "1500 items analyzed!\n",
      "2000 items analyzed!\n",
      "2500 items analyzed!\n",
      "3000 items analyzed!\n",
      "3500 items analyzed!\n",
      "4000 items analyzed!\n",
      "4500 items analyzed!\n",
      "ALL ITEMS ANALYZED!\n",
      "---\n",
      "Now analyzing ./data/any_inner_asteroids.csv\n",
      "0 names in the series are duplicated, of 3000 total\n",
      "0 null values in horizons query list\n",
      "500 items analyzed!\n",
      "1000 items analyzed!\n",
      "1500 items analyzed!\n",
      "2000 items analyzed!\n",
      "2500 items analyzed!\n",
      "ALL ITEMS ANALYZED!\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Get orbital coordinates from HORIZONS server\n",
    "'''\n",
    "center = '@sun'\n",
    "stepsize = '1d'\n",
    "\n",
    "readname = './data/moons.csv'\n",
    "savename_head = './data/moons/'\n",
    "query_horizons(readname, savename_head, stepsize=stepsize, center=center)\n",
    "\n",
    "readname = './data/planets.csv'\n",
    "savename_head = './data/planets/'\n",
    "query_horizons(readname, savename_head, stepsize=stepsize, center=center)\n",
    "\n",
    "readname = './data/large_asteroids.csv'\n",
    "savename_head = './data/large_asteroids/'\n",
    "query_horizons(readname, savename_head, stepsize=stepsize, center=center)\n",
    "\n",
    "readname = './data/large_comets.csv'\n",
    "savename_head = './data/large_comets/'\n",
    "query_horizons(readname, savename_head, stepsize=stepsize, center=center)\n",
    "\n",
    "readname = './data/small_asteroids.csv'\n",
    "savename_head = './data/small_asteroids/'\n",
    "query_horizons(readname, savename_head, stepsize=stepsize, center=center)\n",
    "\n",
    "readname = './data/any_outer_asteroids.csv'\n",
    "savename_head = './data/any_outer_asteroids/'\n",
    "query_horizons(readname, savename_head, stepsize=stepsize, center=center)\n",
    "\n",
    "readname = './data/any_inner_asteroids.csv'\n",
    "savename_head = './data/any_inner_asteroids/'\n",
    "query_horizons(readname, savename_head, stepsize=stepsize, center=center)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}