00_scrapers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| default_exp scrapers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrapers\n",
    "\n",
    "> A set of classes for scraping data about items, series, and agencies from the National Archives of Australia's online database, RecordSearch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "from nbdev.showdoc import *\n",
    "from IPython.display import display, JSON"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The main entities described within RecordSearch are **Items**, **Series**, and **Agencies**. Put simply, items are contained within series, and series are created and controlled by agencies. But the [Series System](https://www.naa.gov.au/help-your-research/getting-started/commonwealth-record-series-crs-system), on which RecordSearch is based, allows a much more complex range of relationships between entities to be documented."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "\n",
    "import requests\n",
    "from requests.adapters import HTTPAdapter\n",
    "from requests.packages.urllib3.util.retry import Retry\n",
    "import mechanicalsoup\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import datetime\n",
    "import math\n",
    "import arrow\n",
    "from sqlitedict import SqliteDict\n",
    "import time\n",
    "\n",
    "# We need separate browser/sessions so that we can nest searches -- eg series search -> series_details -> item search.\n",
    "# But we also want to reuse them as much as possible or we get misleading ConnectionErrors (actually due to too many open files)\n",
    "\n",
    "def make_session():\n",
    "    s = requests.Session()\n",
    "    retries = Retry(total=10, backoff_factor=1, status_forcelist=[ 502, 503, 504 ])\n",
    "    s.mount('https://', HTTPAdapter(max_retries=retries))\n",
    "    s.mount('http://', HTTPAdapter(max_retries=retries))\n",
    "    return s\n",
    "\n",
    "def make_browser():\n",
    "    s = make_session()\n",
    "    browser = mechanicalsoup.StatefulBrowser(soup_config={'features': 'lxml'}, session=s)\n",
    "    return browser\n",
    "\n",
    "s = make_session()\n",
    "\n",
    "browsers = {}\n",
    "browsers['item'] = make_browser()\n",
    "browsers['series'] = make_browser()\n",
    "browsers['agency'] = make_browser()\n",
    "browsers['item_search'] = make_browser()\n",
    "browsers['series_search'] = make_browser()\n",
    "browsers['agency_search'] = make_browser()\n",
    "\n",
    "cache_db = SqliteDict('./cache_db.sqlite', autocommit=True)\n",
    "\n",
    "RS_URLS = {\n",
    "    'item': 'https://recordsearch.naa.gov.au/scripts/AutoSearch.asp?O=I&Number=',\n",
    "    'series': 'https://recordsearch.naa.gov.au/scripts/AutoSearch.asp?Number=',\n",
    "    'agency': 'https://recordsearch.naa.gov.au/scripts/AutoSearch.asp?Number=',\n",
    "    'search_results': 'https://recordsearch.naa.gov.au/SearchNRetrieve/Interface/ListingReports/ItemsListing.aspx',\n",
    "    'ns_results': 'https://recordsearch.naa.gov.au/NameSearch/Interface/ItemsListing.aspx',\n",
    "    'series_results': 'https://recordsearch.naa.gov.au/SearchNRetrieve/Interface/ListingReports/SeriesListing.aspx',\n",
    "    'agency_results': 'https://recordsearch.naa.gov.au/SearchNRetrieve/Interface/ListingReports/AgencyListing.aspx'\n",
    "}\n",
    "\n",
    "ITEM_FORM = {\n",
    "    'kw': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbKeywords',\n",
    "        'type': 'input',\n",
    "    },\n",
    "    'kw_options': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlUsingKeywords',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'kw_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbExKeywords',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'kw_exclude_options': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlUsingExKwd',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    # Set to 'on' to search in item notes\n",
    "    # It's a checkbox, but uses Javascript to set text value.\n",
    "    # Pretend it's a select for validation purposes.\n",
    "    'search_notes': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$cbxKwdTitleNotes',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'series': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbSerNo',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'series_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbExSerNo',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'control': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbIteControlSymb',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'control_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbExIteControlSymb',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'item_id': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbIteBarcode',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'date_from': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbDateFrom',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'date_to': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbDateTo',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    # Select lists (options below)\n",
    "    'formats': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlPhysFormat',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'formats_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlExPhysFormat',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'locations': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlLocation',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'locations_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlExLocation',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'access': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlAccessStatus',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'access_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlExAccessStatus',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    # Checkbox\n",
    "    'digital': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$cbxDigitalCopies',\n",
    "        'type': 'checkbox'\n",
    "    }\n",
    "}\n",
    "\n",
    "KW_OPTIONS = [\n",
    "    'ALL',\n",
    "    'ANY',\n",
    "    'EXACT'\n",
    "]\n",
    "\n",
    "FORMATS = [\n",
    "    'Paper files and documents',\n",
    "    'Index cards',\n",
    "    'Bound volumes',\n",
    "    'Cartographic records',\n",
    "    'Photographs',\n",
    "    'Microforms',\n",
    "    'Audio-visual records',\n",
    "    'Audio records',\n",
    "    'Electronic records',\n",
    "    '3-dimensional records',\n",
    "    'Scientific specimens',\n",
    "    'Textiles'\n",
    "]\n",
    "\n",
    "LOCATIONS = [\n",
    "    'NAT,ACT',\n",
    "    'SA',\n",
    "    'AWM',\n",
    "    'QLD',\n",
    "    'NT',\n",
    "    'TAS',\n",
    "    'VIC',\n",
    "    'WA',\n",
    "    'NSW'\n",
    "]\n",
    "\n",
    "ACCESS = [\n",
    "    'OPEN',\n",
    "    'OWE',\n",
    "    'CLOSED',\n",
    "    'NYE'\n",
    "]\n",
    "\n",
    "AGENCY_FORM = {\n",
    "    'kw': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbKeywords',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'kw_options': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlUsingKeywords',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'kw_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbExcludeKeywords',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'kw_exclude_options': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlExcludeUsingKeywords',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'function': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txtFunctions',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'date_from': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbDateFrom',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'date_to': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbDateTo',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'locations': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlLocation',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'locations_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlExcludeLocation',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'agency_status': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlAgencyStatus',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'agency_status_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlExcludeAgencyStatus',\n",
    "        'type': 'select'\n",
    "    }\n",
    "}\n",
    "\n",
    "AGENCY_LOCATIONS = [\n",
    "    'NAT,ACT',\n",
    "    'COCOS OR CHRISTMAS ISLAND',\n",
    "    'NSW',\n",
    "    'NT',\n",
    "    'OVERSEAS',\n",
    "    'PNG', # Papua New Guinea\n",
    "    'QLD',\n",
    "    'SA',\n",
    "    'TAS',\n",
    "    'VIC',\n",
    "    'WA'\n",
    "]\n",
    "\n",
    "AGENCY_STATUS = [\n",
    "    'DOS', # Department of State\n",
    "    'HO', # Head Office\n",
    "    'RO', # Regional or State Office\n",
    "    'INTGOV', # Intergovernmental agency\n",
    "    'COURT', #Judicial Court or Tribunal\n",
    "    'LO', #Local Office\n",
    "    'NONEX' #Non-Executive government agency (Courts, Parliament)\n",
    "]\n",
    "\n",
    "SERIES_FORM = {\n",
    "    'kw': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbKeywords',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'kw_options': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlUsingKeywords',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'kw_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbExKeywords',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'kw_exclude_options': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlUsingExKwd',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'search_notes': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$cbxKwdTitleNotes',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'series_id': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbSerNo',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'agency_recording': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbAgencyRecording',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'agency_controlling': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbAgencyControlling',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'date_from': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbDateFrom',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'date_to': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$txbDateTo',\n",
    "        'type': 'input'\n",
    "    },\n",
    "    'formats': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlPhysFormat',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'formats_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlExPhysFormat',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'locations': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlLocation',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    'locations_exclude': {\n",
    "        'id': 'ctl00$ContentPlaceHolderSNR$ddlExLocation',\n",
    "        'type': 'select'\n",
    "    },\n",
    "    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "\n",
    "class RSBase(object):\n",
    "    '''\n",
    "    Base class with utility methods.\n",
    "    ''' \n",
    "\n",
    "    def parse_date(self, date_string):\n",
    "        '''\n",
    "        Tests a date string against a list of expected formats.\n",
    "        If a known format is found, the date is refromatted as an ISO date.\n",
    "        '''\n",
    "        date_formats = [\n",
    "            ['D MMMM YYYY', 'YYYY-MM-DD'],\n",
    "            ['D MMM YYYY', 'YYYY-MM-DD'],\n",
    "            ['MMMM YYYY', 'YYYY-MM'],\n",
    "            ['MMM YYYY', 'YYYY-MM'],\n",
    "            ['YYYY', 'YYYY']\n",
    "        ]\n",
    "        for df in date_formats:\n",
    "            try:\n",
    "                iso_date = arrow.get(date_string, df[0]).format(df[1])\n",
    "                return iso_date\n",
    "            # No matching formats\n",
    "            except (arrow.parser.ParserMatchError, ValueError):\n",
    "                pass\n",
    "        return\n",
    "\n",
    "    def process_date_string(self, date_string, prefix=''):\n",
    "        '''\n",
    "        Takes a date range in a string and returns ISO formatted start and end dates.\n",
    "        '''\n",
    "        if not date_string:\n",
    "            start_date = None\n",
    "            end_date = None\n",
    "        else:\n",
    "            dates = date_string.strip().replace(u'\\u2013', '-').split('-')\n",
    "            if dates:\n",
    "                start_date = self.parse_date(dates[0].strip())\n",
    "                try:\n",
    "                    end_date = self.parse_date(dates[1].strip())\n",
    "                except IndexError:\n",
    "                    end_date = None\n",
    "        return {f'{prefix}date_str': date_string, f'{prefix}start_date': start_date, f'{prefix}end_date': end_date}\n",
    "\n",
    "    def get_url(self, url):\n",
    "        '''\n",
    "        Get a RecordSearch page, setting up a session and handling the weird redirects.\n",
    "        '''\n",
    "        self.browser.open(url, timeout=30)\n",
    "        # Note: As of Jan 2023 the following lines are not necessary as the intermediary page seems to have been removed.\n",
    "        # Instead of going directly to the page, you get this weird intermediary page.\n",
    "        # We have to select and submit the form to redirect to the page we want.\n",
    "        # self.browser.select_form('form[name=\"t\"]')\n",
    "        # return self.browser.submit_selected()\n",
    "        return self.browser\n",
    "\n",
    "class RSEntity(RSBase):\n",
    "    '''\n",
    "    Base class for individual RecordSearch entities – item, series, or agency.\n",
    "    \n",
    "    Not for direct use – use the appropriate subclass instead.\n",
    "    '''\n",
    "        \n",
    "    def __init__(self, identifier=None, cache=True, **kwargs):\n",
    "        self.identifier = identifier\n",
    "        self.cache = cache\n",
    "        self.browser = browsers[self.entity_type]\n",
    "        \n",
    "    def get_entity_page(self):\n",
    "        item_url = f'{RS_URLS[self.entity_type]}{self.identifier}'\n",
    "        response = self.get_url(item_url)\n",
    "        return response\n",
    "    \n",
    "    def refresh_cache(self):\n",
    "        '''\n",
    "        Delete data for this entity from the cache, then extract a fresh version from RecordSearch.\n",
    "        '''\n",
    "        del cache_db[f'{self.entity_type}_{self.identifier}']\n",
    "        self.__init__(self.identifier, self.cache)\n",
    "        \n",
    "    def get_cell(self, label):\n",
    "        '''\n",
    "        RecordSearch results for individual entities are formatted in tables, with labels and values.\n",
    "        This retrieves the cell immediately after the cell containing the supplied label.\n",
    "        '''\n",
    "        try:\n",
    "            cell = (\n",
    "                self.details.find(['td', 'div'], string=re.compile(label)).parent.find_next_siblings('td')[0]\n",
    "            )\n",
    "        except (IndexError, AttributeError):\n",
    "            # Sometimes the cell labels are inside an enclosing div,\n",
    "            # but sometimes not. Try again assuming no div.\n",
    "            try:\n",
    "                cell = (\n",
    "                    self.details.find(['td', 'div'], string=re.compile(label)).find_next_siblings('td')[0]\n",
    "                )\n",
    "            except (IndexError, AttributeError):\n",
    "                cell = None\n",
    "        return cell\n",
    "\n",
    "    def get_value(self, label):\n",
    "        '''\n",
    "        Get the value associated with the supplied label from the RecordSearch entity description.\n",
    "        '''\n",
    "        # Get the table cell containing the value\n",
    "        cell = self.get_cell(label)\n",
    "        try:\n",
    "            # Get the text from the cell\n",
    "            value = ' '.join([string for string in cell.stripped_strings])\n",
    "        except AttributeError:\n",
    "            value = None\n",
    "        return value\n",
    "    \n",
    "    def get_note(self, label):\n",
    "        cell = self.get_cell(label)\n",
    "        if cell:\n",
    "            note = cell.find('div', style='display: none;')\n",
    "            if note is None:\n",
    "                note = cell.find('div', id='notes')\n",
    "                note_text = note.get_text(' ', strip=True).replace('\\n', ' ')\n",
    "        else:\n",
    "            note_text = ''\n",
    "        return note_text\n",
    "\n",
    "    def get_details(self):\n",
    "        '''\n",
    "        Extract the main results table from the HTML page.\n",
    "        '''\n",
    "        details = self.browser.page.find('div', 'mainDetailsTable')\n",
    "        return details\n",
    "    \n",
    "    def get_formatted_dates(self, label, prefix=''):\n",
    "        '''\n",
    "        Extract dates from a date range string, try to parse them and format as ISO dates.\n",
    "        Return the original date string, the start date and the end date.\n",
    "        '''\n",
    "        date_str = self.get_value(label)\n",
    "        if date_str:\n",
    "            formatted_dates = self.process_date_string(date_str, prefix)\n",
    "        else:\n",
    "            formatted_dates = {f'{prefix}date_str': '', f'{prefix}start_date': None, f'{prefix}end_date': None}\n",
    "        return formatted_dates\n",
    "    \n",
    "    def get_formatted_date(self, label, prefix=''):\n",
    "        date_str = self.get_value(label)\n",
    "        if date_str:\n",
    "            formatted_date = {f'{prefix}date_str': date_str, f'{prefix}date': self.parse_date(date_str)}\n",
    "        else:\n",
    "            formatted_date = {f'{prefix}date_str': '', f'{prefix}date': None}\n",
    "        return formatted_date\n",
    "    \n",
    "    def get_relations(self, label):\n",
    "        cell = self.get_cell(label)\n",
    "        relations = []\n",
    "        if cell is not None:\n",
    "            for relation in cell.find_all('li'):\n",
    "                try:\n",
    "                    date_str = relation.find('div', 'dates').string.strip()\n",
    "                except AttributeError:\n",
    "                    formatted_dates = {'date_str': '', 'start_date': None, 'end_date': None}\n",
    "                else:\n",
    "                    formatted_dates = self.process_date_string(date_str)\n",
    "                details = [string for string in relation.find('div', 'linkagesInfo').stripped_strings]\n",
    "                try:\n",
    "                    identifier = details[0]\n",
    "                    title = details[1][2:]\n",
    "                except IndexError:\n",
    "                    identifier = details[0]\n",
    "                    title = details[0]\n",
    "                relations.append({\n",
    "                    'date_str': formatted_dates['date_str'],\n",
    "                    'start_date': formatted_dates['start_date'],\n",
    "                    'end_date': formatted_dates['end_date'],\n",
    "                    'identifier': identifier,\n",
    "                    'title': title\n",
    "                })\n",
    "                relation.decompose()\n",
    "            cell.decompose()\n",
    "        return relations\n",
    "    \n",
    "class RSSearch(RSBase):\n",
    "    '''\n",
    "    Base class for an advanced search in RecordSearch. There are different search forms for the different\n",
    "    RecordSearch entities, so don't use this directly. Use one of the subclasses.\n",
    "    '''\n",
    "    \n",
    "    def __init__(self, results_per_page=20, sort=None, record_detail='brief', **kwargs):\n",
    "        self.browser = browsers[f'{self.entity_type}_search']\n",
    "        self.page = 0\n",
    "        params = locals().copy()\n",
    "        params.pop('self')\n",
    "        params.pop('kwargs')\n",
    "        self.params = params\n",
    "        self.kwargs = kwargs\n",
    "        self.initialise_search()     \n",
    "    \n",
    "    def get_full_details(self, results):\n",
    "        full_results = []\n",
    "        for result in results:\n",
    "            full_results.append(self.entity(result['identifier'], cache=False).data)\n",
    "            time.sleep(0.5)\n",
    "        return full_results\n",
    "\n",
    "    def get_results(self, page=None):\n",
    "        '''\n",
    "        Return a list of results from a search results page.\n",
    "        \n",
    "        The `page` value is incremented with each request, so you can call this method\n",
    "        in a loop to retrieve the complete results set. When you reach then of the results, \n",
    "        this method will return an empty list.\n",
    "        \n",
    "        Optional parameter:\n",
    "        \n",
    "        * `page` – request a specific page from the results set\n",
    "        '''\n",
    "        if self.total_results == '20,000+':\n",
    "            return {'total_results': self.total_results, 'page': None, 'number_of_results': 0, 'results': [], 'error': 'Your search returns too many results.'}\n",
    "        if page:\n",
    "            self.page = page\n",
    "        else:\n",
    "            self.page += 1\n",
    "        if self.page <= self.total_pages:\n",
    "            results = self.search(**self.params, **self.kwargs)\n",
    "        else:\n",
    "            # No more data, so return an empty list\n",
    "            results = {'total_results': self.total_results, 'page': None, 'number_of_results': 0, 'results': []}\n",
    "        return results\n",
    "    \n",
    "    def process_list(self, details):\n",
    "        results = []\n",
    "        retrieved = arrow.now(tz='Australia/Sydney').isoformat()\n",
    "        for row in details.find_all('tr')[1:]:\n",
    "            record = self.process_row(row)\n",
    "            record['retrieved'] = retrieved\n",
    "            results.append(record)\n",
    "        return results\n",
    "    \n",
    "    def process_page(self, soup, record_detail):\n",
    "        # Do something\n",
    "        return []\n",
    "    \n",
    "    def get_total_results(self, soup):\n",
    "        total = 0\n",
    "        if soup.find(id='ContentPlaceHolderSNR_lblToManyRecordsError') is not None:\n",
    "            total = '20,000+'\n",
    "        elif soup.find('span', attrs={'id': re.compile('lblDisplaying$')}) is not None:\n",
    "            total_text = soup.find('span', attrs={'id': re.compile('lblDisplaying$')}).text\n",
    "            total = int(re.search(r'of (\\d+)', total_text).group(1))\n",
    "        elif soup.find('span', text='Displaying 1 of 1'):\n",
    "            total = 1\n",
    "        return total\n",
    "    \n",
    "    def refresh_cache(self):\n",
    "        '''\n",
    "        Delete data for this search from the cache, then retrieve a fresh version from RecordSearch.\n",
    "        '''\n",
    "        cache_key = self.generate_cache_key()\n",
    "        cache_key = re.sub('_page_\\d+$', '', cache_key)\n",
    "        for key in cache_db.keys():\n",
    "            if cache_key in key:\n",
    "                del cache_db[key]\n",
    "        self.page = 0\n",
    "        self.initialise_search()\n",
    "    \n",
    "    def generate_cache_key(self):\n",
    "        '''\n",
    "        Use the search parameters to generate a key to use in storing the cached results.\n",
    "        '''\n",
    "        params = self.params.copy()\n",
    "        params.update(self.kwargs)\n",
    "        search_key = '_'.join(sorted([f'{k}_{v}' for k, v in params.items() if v is not None]))\n",
    "        search_key = f'{self.entity_type}_{search_key}_page_{self.page}'\n",
    "        return search_key\n",
    "\n",
    "    def search(self, results_per_page=None, sort=None, record_detail='brief', **kwargs):\n",
    "        # Generate key to use with cache\n",
    "        cache_key = self.generate_cache_key()\n",
    "        # Try to get results from cache first\n",
    "        try:\n",
    "            results = cache_db[cache_key]\n",
    "        except KeyError:\n",
    "            # Set the number of results per page\n",
    "            if results_per_page != 20:\n",
    "                search_form = self.browser.select_form('#formSNRMaster')\n",
    "                search_form.set('ctl00$ContentPlaceHolderSNR$ddlResultsPerPage', results_per_page)\n",
    "                submit_button = self.browser.page.find(id='ContentPlaceHolderSNR_btnSearch')\n",
    "                self.browser.submit_selected()\n",
    "            # Apply sort\n",
    "            if sort:\n",
    "                r = self.browser.open(f'{self.browser.url}?sort={sort}')\n",
    "            # Retrieve a specific page in the results set\n",
    "            if self.page > 1:\n",
    "                url = self.browser.url.split('?')[0]   \n",
    "                self.browser.open(f'{url}?page={self.page-1}')\n",
    "            # Get item details from list of search results\n",
    "            data = self.process_page(self.browser.page, record_detail)\n",
    "            results = {\n",
    "                'total_results': self.total_results, \n",
    "                'page': self.page, \n",
    "                'number_of_results': len(data), \n",
    "                'results': data,\n",
    "                'retrieved': arrow.now(tz='Australia/Sydney').isoformat()\n",
    "            }\n",
    "            cache_db[cache_key] = results\n",
    "        #self.page = results['page']\n",
    "        return results\n",
    "\n",
    "    def initialise_search(self):\n",
    "        '''\n",
    "        Populates the search form and retrieves the total number of results.\n",
    "        '''\n",
    "        # Start a session\n",
    "        self.get_url('https://recordsearch.naa.gov.au/scripts/Logon.asp?N=guest')\n",
    "        # Find the main advanced search link\n",
    "        main_link = self.browser.find_link(url_regex='AdvSearchMain.aspx')\n",
    "        self.browser.follow_link(main_link)\n",
    "        # Find the advanced search link for this entity\n",
    "        search_link = self.browser.find_link(url_regex=self.search_page)\n",
    "        self.browser.follow_link(search_link)\n",
    "        # Get the submit button for the search form\n",
    "        submit_button = self.browser.page.find(id='ContentPlaceHolderSNR_btnSearch')\n",
    "        # Get the search form\n",
    "        search_form = self.browser.select_form('#formSNRMaster')\n",
    "        # Populate the search form with the supplied params\n",
    "        for key, value in self.kwargs.items():\n",
    "            search_form.set(self.search_params[key]['id'], value)\n",
    "        # Submit the form\n",
    "        search_form.choose_submit(submit_button)\n",
    "        self.browser.submit_selected()\n",
    "        # There's a 'search is running' page that has a form that needs to be submitted.\n",
    "        running_form = self.browser.select_form('#Form1')\n",
    "        self.browser.submit_selected()\n",
    "        # Save the total number of results\n",
    "        self.total_results = self.get_total_results(self.browser.page)\n",
    "        # Calcuate the number of pages in the results set\n",
    "        try:\n",
    "            self.total_pages = math.ceil(self.total_results / self.params['results_per_page'])\n",
    "        except TypeError:\n",
    "            # More than 20,000 results\n",
    "            pass\n",
    "   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export    \n",
    "\n",
    "class RSItem(RSEntity):\n",
    "    '''\n",
    "    Class used for extracting data about an individual item (usually a file, but can be a volume, box, photograph etc) from RecordSearch.\n",
    "    \n",
    "    You need to supply one of the following parameters:\n",
    "    \n",
    "    * `identifier` – the Item ID (aka barcode)\n",
    "    * `details` – the BeautifulSoup HTML element containing the item details\n",
    "    \n",
    "    You'd only use `details` if you already have a RecordSearch page and want to extract item data from it. \n",
    "    (There's an example of this in the `RSItemSearch` class.)\n",
    "    \n",
    "    The item data is obtained by accessing the item's `.data` attribute.\n",
    "    '''\n",
    "    entity_type = 'item'\n",
    "    \n",
    "    def __init__(self, identifier=None, cache=True, details=None):\n",
    "        super(RSItem, self).__init__(identifier, cache)\n",
    "        self.details = details\n",
    "        if details:\n",
    "            self.identifier = self.get_value('Item ID')\n",
    "        self.data = self.get_item()\n",
    "        \n",
    "    def get_series(self):\n",
    "        cell = self.get_cell('Series number')\n",
    "        return cell.find('a').string.strip()\n",
    "\n",
    "    def get_access_reasons(self):\n",
    "        '''\n",
    "        Extract the list of reasons why material has been withheld after access examination.\n",
    "        '''\n",
    "        cell = self.get_cell('Reason for restriction')\n",
    "        reasons = []\n",
    "        if cell:\n",
    "            for link in cell.find_all('a'):\n",
    "                reasons.append(link.string.strip())\n",
    "        return reasons\n",
    "    \n",
    "    def get_digitised_pages(self):\n",
    "        '''\n",
    "        Returns the number of pages (images) in a digitised file.\n",
    "        This is scraped from the RecordSearch digitised file viewer.\n",
    "        The file viewer is outside of RecordSearch's session system, so it can be requested directly.\n",
    "        '''\n",
    "        url = f'https://recordsearch.naa.gov.au/SearchNRetrieve/Interface/ViewImage.aspx?B={self.identifier}'\n",
    "        response = s.get(url, timeout=30)\n",
    "        soup = BeautifulSoup(response.text, features='lxml')\n",
    "        try:\n",
    "            # The last page number from the navigation will be the total number of pages\n",
    "            pages = int(soup.find('span', attrs={'id': \"lblEndPage\"}).string)\n",
    "        except AttributeError:\n",
    "            # If there's no navigation it might be a single page\n",
    "            if soup.find('span', attrs={'id': \"lblCitation\"}):\n",
    "                pages = 1\n",
    "            # Or something else...\n",
    "            else:\n",
    "                pages = 0\n",
    "        return pages\n",
    "\n",
    "    def check_if_digitised(self):\n",
    "        '''\n",
    "        Check to see if the file is digitised, by looking for a link to the digital copy.\n",
    "        '''\n",
    "        if self.details.find(text=re.compile(\"View digital copy\")):\n",
    "            return True\n",
    "        else:\n",
    "            return False\n",
    "\n",
    "    def get_item(self, date_format='iso'):\n",
    "        # Try to retrieve from cache first\n",
    "        try:\n",
    "            item = cache_db[f'item_{self.identifier}']\n",
    "        # If not in the cache and the details are not supplied, get it from RS.\n",
    "        except KeyError:\n",
    "            if not self.details:\n",
    "                self.get_entity_page()\n",
    "                self.details = self.get_details()\n",
    "            if self.details:\n",
    "                item = {\n",
    "                    'title': self.get_value('Title'),\n",
    "                    'identifier': self.identifier,\n",
    "                    'series': self.get_series(),\n",
    "                    'control_symbol': self.get_value('Control symbol'),\n",
    "                    'note': self.get_note('Item note'),\n",
    "                    'digitised_status': self.check_if_digitised(),\n",
    "                    'digitised_pages': self.get_digitised_pages(),\n",
    "                    'access_status': self.get_value('Access status'),\n",
    "                    'access_decision_reasons': self.get_access_reasons(),\n",
    "                    'location': self.get_value('Location'),\n",
    "                    'retrieved': arrow.now(tz='Australia/Sydney').isoformat()\n",
    "                }\n",
    "                item.update(self.get_formatted_dates('Contents date range', 'contents_'))\n",
    "                item.update(self.get_formatted_date('Date of decision', 'access_decision_'))\n",
    "                if self.cache:\n",
    "                    # Add to the cache\n",
    "                    cache_db[f'item_{self.identifier}'] = item\n",
    "            else:\n",
    "                item = {'identifier': self.identifier, 'error': 'Item not found'}\n",
    "        return item\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f'NAA: {self.data[\"series\"]}, {self.data[\"control_symbol\"]}'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Items in RecordSearch are usually paper files, but can be other things like volumes, boxes, videos, or digital objects. Items have a unique identifier described as the 'Item ID', this was previously referred to as the item's 'barcode'. The `RSItem` class extracts information about an individual item from RecordSearch, using it's Item ID.\n",
    "\n",
    "Here are the fields returned:\n",
    "\n",
    "* `title` (string)\n",
    "* `identifier` (string)\n",
    "* `series` (string)\n",
    "* `control_symbol` (string)\n",
    "* `digitised_status` (boolean) – `True` if the item has been digitised\n",
    "* `digitised_pages` (integer) – number of pages in the digitised file\n",
    "* `access_status` (string) – one of 'Open', 'OWE', 'Closed', 'NYE' (see [access status options](#Access-status-options))\n",
    "* `access_decision_reasons` (list) – a list of reasons why material has been been witheld from public access (if CLOSED, or OWE)\n",
    "* `location` (string)\n",
    "* `contents_date_str` (ISO formatted date)\n",
    "* `contents_start_date` (ISO formatted date)\n",
    "* `contents_end_date` (ISO formatted date)\n",
    "* `access_decision_date_str` (ISO formatted date)\n",
    "* `access_decision_date` (ISO formatted date)\n",
    "* `retrieved` (ISO formatted datetime) - when this record was scraped\n",
    "\n",
    "Note that `digitised_pages` is not part of the metadata presented in RecordSearch's item description. This value is obtained from the digitised file viewer using `get_digitised_pages()`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To retrieve information about an item, just give `RSItem()` the Item ID (also known as the barcode)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get an item\n",
    "item = RSItem('3445411')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can then access the item data using the `.data` attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': 'WRAGGE Clement Lionel Egerton : SERN 647 : POB Cheadle England : POE Enoggera QLD : NOK  (Father) WRAGGE Clement Lindley',\n",
       " 'identifier': '3445411',\n",
       " 'series': 'B2455',\n",
       " 'control_symbol': 'WRAGGE C L E',\n",
       " 'note': '',\n",
       " 'digitised_status': True,\n",
       " 'digitised_pages': 47,\n",
       " 'access_status': 'Open',\n",
       " 'access_decision_reasons': [],\n",
       " 'location': 'Canberra',\n",
       " 'retrieved': '2023-01-20T10:47:44.524487+11:00',\n",
       " 'contents_date_str': '1914 - 1920',\n",
       " 'contents_start_date': '1914',\n",
       " 'contents_end_date': '1920',\n",
       " 'access_decision_date_str': '12 Apr 2001',\n",
       " 'access_decision_date': '2001-04-12'}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(item.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use `item.data[FIELD NAME]` to access individual fields. The `series` value of this item should be 'B2455'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert item.data['series'] == 'B2455'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As an added bonus, the string representation of the item is also its brief citation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'NAA: B2455, WRAGGE C L E'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(item)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not all items have notes, but if they do, they should be included."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Summary heading Negative of photograph of Poon Gooey available Descriptive note Photograph of Poon Gooey used in National Archives of Australia exhibition – Alien Edwardians – Chinese Immigrants and Commonwealth Government. Duplicating negative of photograph available – please see National Archives staff [ B6416 b/c #10551458]'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This item has a note\n",
    "item = RSItem(324717)\n",
    "\n",
    "assert 'note' in item.data\n",
    "\n",
    "# Display the note\n",
    "item.data['note']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The extracted data is saved into a simple key-value cache to speed up repeat requests. If you want to scrape a fresh version, use `.refresh_cache()`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "### RSItem.refresh_cache\n",
       "\n",
       ">      RSItem.refresh_cache ()\n",
       "\n",
       "Delete data for this entity from the cache, then extract a fresh version from RecordSearch."
      ],
      "text/plain": [
       "---\n",
       "\n",
       "### RSItem.refresh_cache\n",
       "\n",
       ">      RSItem.refresh_cache ()\n",
       "\n",
       "Delete data for this entity from the cache, then extract a fresh version from RecordSearch."
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(RSItem.refresh_cache, name=\"RSItem.refresh_cache\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can check that this has worked by comparing the value of `retrieved`, which is the date/time the data was scraped."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_retrieved_date = item.data['retrieved']\n",
    "\n",
    "item.refresh_cache()\n",
    "\n",
    "new_retrieved_date = item.data['retrieved']\n",
    "\n",
    "assert old_retrieved_date != new_retrieved_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export \n",
    "    \n",
    "class RSItemSearch(RSSearch):\n",
    "    '''\n",
    "    Search for items in RecordSearch.\n",
    "    \n",
    "    Supply any of the item search parameters as kwargs to initialise the search.\n",
    "    \n",
    "    Optional parameters:\n",
    "    \n",
    "    * `results_per_page` (default: 20)\n",
    "    * `sort` (default: 1 – order by id)\n",
    "    * `page` – to retrieve a specific page of results\n",
    "    * `record_detail` – amount of detail to include, options are:\n",
    "        * 'brief' (default) – just the info in the search results\n",
    "        * `digitised` – add the number of pages if the file is digitised (slower)\n",
    "        * 'full' – get the full individual record for each result (slowest)\n",
    "    \n",
    "    To access a page of results, use the `.get_results()` method. \n",
    "    This method increments the results page, so you can call it in a loop\n",
    "    to retrieve the complete result set.\n",
    "    \n",
    "    Useful attributes:\n",
    "    \n",
    "    * `.total_results` – the total number of results in the results set\n",
    "    * `.total_pages` – the total number of result pages\n",
    "    * `.kwargs` – a dict containing the supplied search parameters\n",
    "    * `.params` – a dict containing the values of the optional parameters\n",
    "    '''\n",
    "    \n",
    "    entity_type = 'item'\n",
    "    search_params = ITEM_FORM\n",
    "    search_page = 'AdvSearchItems.aspx'\n",
    "    entity = RSItem\n",
    "    \n",
    "    def __init__(self, results_per_page=20, sort=9, record_detail='brief', **kwargs):\n",
    "        super(RSItemSearch, self).__init__(results_per_page=results_per_page, sort=sort, record_detail=record_detail, **kwargs)\n",
    "\n",
    "    def process_row(self, row):\n",
    "        cells = row.find_all('td')\n",
    "        item = {\n",
    "            'series': cells[1].string.strip(),\n",
    "            'control_symbol': cells[2].string.strip(),\n",
    "            'title': cells[3].contents[0].string.strip(),\n",
    "            'identifier': cells[6].string.strip()\n",
    "        }\n",
    "        access_string = cells[3].find('div', 'CombinedTitleBottomLeft').string\n",
    "        item['access_status'] = re.search(r'Access status: ([\\w ]+)', access_string).group(1).strip()\n",
    "        location_string = cells[3].find('div', 'CombinedTitleBottomRight').string\n",
    "        item['location'] = re.search(r'Location: ([\\w ]+)', location_string).group(1).strip()\n",
    "        date_str = cells[4].string.strip()\n",
    "        item.update(self.process_date_string(date_str, 'contents_'))\n",
    "        if cells[5].find('a') is not None:\n",
    "            item['digitised_status'] = True\n",
    "        else:\n",
    "            item['digitised_status'] = False\n",
    "        return item\n",
    "\n",
    "    def get_digitised_page_counts(self, items):\n",
    "        for item in items:\n",
    "            if item['digitised_status'] == True:\n",
    "                item['digitised_pages'] = RSItem(item['identifier']).get_digitised_pages()\n",
    "            else:\n",
    "                item['digitised_pages'] = 0\n",
    "        return items\n",
    "    \n",
    "    def process_page(self, soup, record_detail):\n",
    "        '''\n",
    "        Extract item data from a search results page.\n",
    "        Level of item data can be varied using the `record_detail` parameter:\n",
    "            * 'brief' - just the data in the search results\n",
    "            * 'digitised' - if the file is digitised, get the number of pages\n",
    "            * 'full': retrieve the individual item record to get extra fields\n",
    "        '''\n",
    "        # There's a list of items\n",
    "        if details := soup.find(id=re.compile('tblItemDetails$')):\n",
    "            items = self.process_list(details)\n",
    "        # There's a single item\n",
    "        elif soup.find(id=re.compile('ContentPlaceHolderSNR_ucItemDetails_phDetailsView')) is not None:\n",
    "            details = soup.find('div', 'mainDetailsTable')\n",
    "            items = [self.entity(details=details, cache=False).data]\n",
    "        # No items?\n",
    "        else:\n",
    "            raise Exception('No results found on page!')\n",
    "        # Add number of pages in digitised files\n",
    "        if record_detail == 'digitised':\n",
    "            items = self.get_digitised_page_counts(items)\n",
    "        # Get full item information\n",
    "        if record_detail == 'full':\n",
    "            items = self.get_full_details(items)\n",
    "        return items\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Item search parameters\n",
    "\n",
    "These are the parameters you can supply as keyword arguments to `RSItemSearch`.\n",
    "\n",
    "| Parameter | Input type | Values |\n",
    "| :--- | :--- | :--- |\n",
    "| **kw** | Text | Keywords or phrase to search for|\n",
    "| **kw_options** | Select | How to combine the keywords – see [keyword options](#Keyword-options) |\n",
    "| **kw_exclude** | Text | Keywords or phrase to exclude |\n",
    "| **kw_exclude_options** | Select | How to combine the keywords – see [keyword options](#Keyword-options) |\n",
    "| **search_notes** | Checkbox | Set to `True` to search notes as well as titles |\n",
    "| **series** | Text | Limit to items from this series – eg 'A1' |\n",
    "| **series_exclude** | Text | Exclude items from this series |\n",
    "| **control** | Text | Limit to items with this control symbol (use * for wildcards) – eg '1947/2*' |\n",
    "| **control_exclude** | Text | Exclude items with this control symbol |\n",
    "| **item_id** | Text | Get the item with this identifier (no wildcards allowed)|\n",
    "| **date_from** | Text | Include items with content after this date (year only) – eg '1925' |\n",
    "| **date_to** | Text | Include items with content before this date (year only) – eg '1945' |\n",
    "| **formats** | Select | Limit to items in this format – see [format options](#Format-options) |\n",
    "| **formats_exclude** | Select | Exclude items in this format  – see [format options](#Format-options) |\n",
    "| **locations** | Select | Limit to items held in this location – see list of [locations](#Location-options) |\n",
    "| **locations_exclude** | Select | Exclude items held in this location – see list of [locations](#Location-options) |\n",
    "| **access** | Select | Limit to items with this access status – see [access status options](#Access-status-options) |\n",
    "| **access_exclude** | Select | Exclude items with this access status – see [access status options](#Access-status-options) |\n",
    "| **digital** | Checkbox | Limit to digitised items – set to `True` |\n",
    "\n",
    "### Keyword options\n",
    "\n",
    "Use one of the following values to specify how keywords or phrases should be treated using the `kw_options` parameter. The default is 'ALL'.\n",
    "\n",
    "* **'ALL'** (default) – must include all keywords\n",
    "* **'ANY'** – must include at least one of the keywords\n",
    "* **'EXACT'** – treat the keywords as a phrase\n",
    "\n",
    "### Format options\n",
    "\n",
    "Use one of the following values with the `formats` and `formats_exclude` parameters to limit your results to items in that format. The default is to include all formats.\n",
    "\n",
    "* **'Paper files and documents'**\n",
    "* **'Index cards'**\n",
    "* **'Bound volumes'**\n",
    "* **'Cartographic records'**\n",
    "* **'Photographs'**\n",
    "* **'Microforms'**\n",
    "* **'Audio-visual records'**\n",
    "* **'Audio records'**\n",
    "* **'Electronic records'**\n",
    "* **'3-dimensional records'**\n",
    "* **'Scientific specimens'**\n",
    "* **'Textiles'**\n",
    "\n",
    "### Location options\n",
    "\n",
    "Use one of the following values with the `locations` and `locations_exclude` parameters to limit your results to items held in that location. The default is to include all locations.\n",
    "\n",
    "* **'NAT,ACT'** – National office (ACT)\n",
    "* **'AWM'** – Australian War Memorial\n",
    "* **'NSW'**\n",
    "* **'NT'**\n",
    "* **'QLD'**\n",
    "* **'SA'**\n",
    "* **'TAS'**\n",
    "* **'VIC'**\n",
    "* **'WA'**\n",
    "\n",
    "\n",
    "### Access status options\n",
    "\n",
    "Use one of the following values with the `access` and `access_exclude` parameters to limit your results to items with this access examination status. The default is to include all.\n",
    "\n",
    "* **'OPEN'** – available for public access\n",
    "* **'OWE'** – open with exceptions (eg it might have pages withheld or redactions applied)\n",
    "* **'CLOSED'** – withheld completely from public access\n",
    "* **'NYE'** – not yet examined (no access decision has been made)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's a basic keyword search for items."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_results = RSItemSearch(kw='wragge')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialising the `RSItemSearch` class sets up the search and retrieves some information about the results set. For example, to see the total number of results, we just access the `.total_results` attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "212"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_results.total_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "### RSItemSearch.get_results\n",
       "\n",
       ">      RSItemSearch.get_results (page=None)\n",
       "\n",
       "Return a list of results from a search results page.\n",
       "\n",
       "The `page` value is incremented with each request, so you can call this method\n",
       "in a loop to retrieve the complete results set. When you reach then of the results, \n",
       "this method will return an empty list.\n",
       "\n",
       "Optional parameter:\n",
       "\n",
       "* `page` – request a specific page from the results set"
      ],
      "text/plain": [
       "---\n",
       "\n",
       "### RSItemSearch.get_results\n",
       "\n",
       ">      RSItemSearch.get_results (page=None)\n",
       "\n",
       "Return a list of results from a search results page.\n",
       "\n",
       "The `page` value is incremented with each request, so you can call this method\n",
       "in a loop to retrieve the complete results set. When you reach then of the results, \n",
       "this method will return an empty list.\n",
       "\n",
       "Optional parameter:\n",
       "\n",
       "* `page` – request a specific page from the results set"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(RSItemSearch.get_results, name=\"RSItemSearch.get_results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'total_results': 212,\n",
       " 'page': 1,\n",
       " 'number_of_results': 20,\n",
       " 'results': [{'series': 'A2479',\n",
       "   'control_symbol': '17/1306',\n",
       "   'title': 'The Wragge Estate. Property for sale.',\n",
       "   'identifier': '149309',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': '1917 - 1917',\n",
       "   'contents_start_date': '1917',\n",
       "   'contents_end_date': '1917',\n",
       "   'digitised_status': True,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'A2487',\n",
       "   'control_symbol': '1919/8962',\n",
       "   'title': '[Application for free passage - Rupert Lindley Wragge]',\n",
       "   'identifier': '156686',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': '1918 - 1919',\n",
       "   'contents_start_date': '1918',\n",
       "   'contents_end_date': '1919',\n",
       "   'digitised_status': True,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'B1535',\n",
       "   'control_symbol': '736/23/341',\n",
       "   'title': '[Applications for Commissions: K S Wragge, R A Fry, V J T Sharpe, J G Cameron]',\n",
       "   'identifier': '377317',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Melbourne',\n",
       "   'contents_date_str': '1939 - 1939',\n",
       "   'contents_start_date': '1939',\n",
       "   'contents_end_date': '1939',\n",
       "   'digitised_status': False,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'AWM54',\n",
       "   'control_symbol': '1010/4/150',\n",
       "   'title': '[War Crimes and Trials - Affidavits and Sworn Statements:] Statements by No number RW Woodhouse; VX39749 Pte RC Woodman; 23663 Able Seaman CD Woodman; NX58655 WO/1 WJ Woodward; VX37644 Pte CA Woodward; QX15620 Pte HA Woodward; TX3959 Pte BJ Woodward; No number Capt EJ Wooldridge; No number AN Wooton; NX501953 Spr GA Worland; QX259 Capt NA Worthington; QX14395 Gnr RL Wragge; NX12233 Lt-Col JW Wright; NX70664 Capt RG Wright; VX35065 L/Bdr JH Wright; SX8474 Pte RR Wright; NX56234 Cpl AG Wright; NX27630 L/Sgt L Wrightson; NX69819 Dvr RP Wyatt; WX12593 Pte RW Wyllie',\n",
       "   'identifier': '479150',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Australian War Memorial',\n",
       "   'contents_date_str': '1945 - 1947',\n",
       "   'contents_start_date': '1945',\n",
       "   'contents_end_date': '1947',\n",
       "   'digitised_status': False,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'A1716',\n",
       "   'control_symbol': '240',\n",
       "   'title': '\"Wragge\" [NOTE: Registration and exhibit]',\n",
       "   'identifier': '659953',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': '1902 - 1902',\n",
       "   'contents_start_date': '1902',\n",
       "   'contents_end_date': '1902',\n",
       "   'digitised_status': True,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'A1716',\n",
       "   'control_symbol': '241',\n",
       "   'title': '\"Wragge\" [NOTE: Registration and exhibit]',\n",
       "   'identifier': '659955',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': '1902 - 1902',\n",
       "   'contents_start_date': '1902',\n",
       "   'contents_end_date': '1902',\n",
       "   'digitised_status': True,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'A1716',\n",
       "   'control_symbol': '243',\n",
       "   'title': '\"Wragge\" [NOTE: Registration and exhibit]',\n",
       "   'identifier': '659958',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': '1902 - 1902',\n",
       "   'contents_start_date': '1902',\n",
       "   'contents_end_date': '1902',\n",
       "   'digitised_status': True,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'AWM93',\n",
       "   'control_symbol': '22/2/83',\n",
       "   'title': '[Australian War Memorial registry files, first series] Staff - Personal files: Mr W G Wragge',\n",
       "   'identifier': '1015137',\n",
       "   'access_status': 'Not yet examined',\n",
       "   'location': 'Australian War Memorial',\n",
       "   'contents_date_str': '1924 - 1925',\n",
       "   'contents_start_date': '1924',\n",
       "   'contents_end_date': '1925',\n",
       "   'digitised_status': False,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'B503',\n",
       "   'control_symbol': 'Q2018',\n",
       "   'title': 'Wragge, Raymond Lindley QX14395 [Prisoners of War Trust Fund application]',\n",
       "   'identifier': '1031372',\n",
       "   'access_status': 'Not yet examined',\n",
       "   'location': 'Melbourne',\n",
       "   'contents_date_str': '1955 - 1977',\n",
       "   'contents_start_date': '1955',\n",
       "   'contents_end_date': '1977',\n",
       "   'digitised_status': False,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'B503',\n",
       "   'control_symbol': 'V3600',\n",
       "   'title': 'Wragge, Leslie James VX25877 [Prisoners of War Trust Fund application]',\n",
       "   'identifier': '1033806',\n",
       "   'access_status': 'Not yet examined',\n",
       "   'location': 'Melbourne',\n",
       "   'contents_date_str': '1955 - 1977',\n",
       "   'contents_start_date': '1955',\n",
       "   'contents_end_date': '1977',\n",
       "   'digitised_status': False,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'SP459/1',\n",
       "   'control_symbol': '429/8/5434',\n",
       "   'title': 'Injuries - VX112202 Captain WRAGGE, H S [Box 64]',\n",
       "   'identifier': '1365125',\n",
       "   'access_status': 'Not yet examined',\n",
       "   'location': 'Sydney',\n",
       "   'contents_date_str': '1947 - 1947',\n",
       "   'contents_start_date': '1947',\n",
       "   'contents_end_date': '1947',\n",
       "   'digitised_status': False,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'A2478',\n",
       "   'control_symbol': 'TRZECIAK H',\n",
       "   'title': 'TRZECIAK Helmut born 17 February 1920; frida (nee Wragge) born 13 November 1921; Monika born 9 January 1944; Rosemarie born 30 June 1945; Hartmut born 19 January 1952 - German - travelled per ship SKAUBRYN departing in 1954',\n",
       "   'identifier': '1438793',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': '1954 - 1954',\n",
       "   'contents_start_date': '1954',\n",
       "   'contents_end_date': '1954',\n",
       "   'digitised_status': False,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'A9951',\n",
       "   'control_symbol': '71',\n",
       "   'title': 'Nominal Roll - Original 3501 - 3542. Wragge, Keith Clement - Zusman, Solomon',\n",
       "   'identifier': '1751559',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': '1949 - 1949',\n",
       "   'contents_start_date': '1949',\n",
       "   'contents_end_date': '1949',\n",
       "   'digitised_status': True,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'A9951',\n",
       "   'control_symbol': '72',\n",
       "   'title': 'Nominal Roll - Duplicate 3501 - 3542. Wragge, Keith Clement - Zusman, Solomon',\n",
       "   'identifier': '1751560',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': '1949 - 1949',\n",
       "   'contents_start_date': '1949',\n",
       "   'contents_end_date': '1949',\n",
       "   'digitised_status': True,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'J1193',\n",
       "   'control_symbol': 'QX14395',\n",
       "   'title': 'Members folders, Second World War Queensland army personnel - Wragge Raymond Lindley',\n",
       "   'identifier': '1908313',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Brisbane',\n",
       "   'contents_date_str': '1940 - 1949',\n",
       "   'contents_start_date': '1940',\n",
       "   'contents_end_date': '1949',\n",
       "   'digitised_status': False,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'A10074',\n",
       "   'control_symbol': '1926/13',\n",
       "   'title': 'WRAGGE Thomas William Eric versus COLLINS James; COLLINS Edward',\n",
       "   'identifier': '3141698',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': '1926 - 1926',\n",
       "   'contents_start_date': '1926',\n",
       "   'contents_end_date': '1926',\n",
       "   'digitised_status': False,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'B2455',\n",
       "   'control_symbol': 'WRAGGE A C P',\n",
       "   'title': 'Wragge Alfred Charles Peter : SERN Depot : POB Rockhampton QLD : POE Brisbane QLD : NOK W Wragge F E',\n",
       "   'identifier': '3445406',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': 'circa1914 - circa1920',\n",
       "   'contents_start_date': None,\n",
       "   'contents_end_date': None,\n",
       "   'digitised_status': True,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'B2455',\n",
       "   'control_symbol': 'WRAGGE C L E',\n",
       "   'title': 'WRAGGE Clement Lionel Egerton : SERN 647 : POB Cheadle England : POE Enoggera QLD : NOK  (Father) WRAGGE Clement Lindley',\n",
       "   'identifier': '3445411',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': 'circa1914 - circa1920',\n",
       "   'contents_start_date': None,\n",
       "   'contents_end_date': None,\n",
       "   'digitised_status': True,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'B2455',\n",
       "   'control_symbol': 'WRAGGE G S',\n",
       "   'title': 'Wragge George Stanley : SERN 5580 : POB Croydon QLD : POE Cairns QLD : NOK Rivers Mrs Elizabeth',\n",
       "   'identifier': '3445416',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': 'circa1914 - circa1920',\n",
       "   'contents_start_date': None,\n",
       "   'contents_end_date': None,\n",
       "   'digitised_status': True,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'},\n",
       "  {'series': 'B2455',\n",
       "   'control_symbol': 'WRAGGE J H',\n",
       "   'title': 'Wragge John Henry : SERN 6922 : POB Richmond VIC : POE Melbourne VIC : NOK W Wragge Lillian Maud',\n",
       "   'identifier': '3445422',\n",
       "   'access_status': 'Open',\n",
       "   'location': 'Canberra',\n",
       "   'contents_date_str': 'circa1914 - circa1920',\n",
       "   'contents_start_date': None,\n",
       "   'contents_end_date': None,\n",
       "   'digitised_status': True,\n",
       "   'retrieved': '2023-01-20T10:47:46.524370+11:00'}],\n",
       " 'retrieved': '2023-01-20T10:47:46.530565+11:00'}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_results.get_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'results_per_page': 20, 'sort': 9, 'record_detail': 'brief'}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_results.params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This example returns a result set with a single item. In this case full data will be retrieved even if `record_detail` is set to 'brief'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_results = RSItemSearch(series='B2455', control='HART G H*', record_detail='brief')\n",
    "assert item_results.total_results == 1\n",
    "data = item_results.get_results()\n",
    "\n",
    "# Page layout is different for a single result, so check it's being parsed\n",
    "assert data['results'][0]['digitised_status'] == True\n",
    "\n",
    "# The number of digitised pages is included even though record_detail is set to brief\n",
    "assert data['results'][0]['digitised_pages'] > 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calling `.refresh_cache` will remove all of the data for this search from the cache, and set the results page back to 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "### RSItemSearch.refresh_cache\n",
       "\n",
       ">      RSItemSearch.refresh_cache ()\n",
       "\n",
       "Delete data for this search from the cache, then retrieve a fresh version from RecordSearch."
      ],
      "text/plain": [
       "---\n",
       "\n",
       "### RSItemSearch.refresh_cache\n",
       "\n",
       ">      RSItemSearch.refresh_cache ()\n",
       "\n",
       "Delete data for this search from the cache, then retrieve a fresh version from RecordSearch."
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(RSItemSearch.refresh_cache, name=\"RSItemSearch.refresh_cache\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "\n",
    "class RSSeries(RSEntity):\n",
    "    '''\n",
    "    Class used for extracting data about an individual series.\n",
    "    You need to supply the following parameter:\n",
    "    \n",
    "    * `identifier` – the series number, eg 'A1', 'B2455' \n",
    "    \n",
    "    Optional parameters:\n",
    "    \n",
    "    * `include_number_digitised` (boolean, default: True) – \n",
    "      include the number of items in this series that have been digitised.\n",
    "    * `include_access_status` (boolean, default: True) – \n",
    "      include the number of items in this series in each of the access status categories.\n",
    "    \n",
    "    The series data is obtained by accessing the series' `.data` attribute.\n",
    "    '''\n",
    "    entity_type = 'series'\n",
    "    \n",
    "    def __init__(self, identifier=None, cache=True, details=None, include_number_digitised=True, include_access_status=True):\n",
    "        super(RSSeries, self).__init__(identifier, cache)\n",
    "        self.digitised = include_number_digitised\n",
    "        self.access_status = include_access_status\n",
    "        self.details = details\n",
    "        if details:\n",
    "            self.identifier = self.get_value('Series number')\n",
    "        self.data = self.get_series()\n",
    "          \n",
    "    def get_number_described(self):\n",
    "        described = self.get_value('Items in this series on RecordSearch')\n",
    "        try:\n",
    "            described_number, described_note = re.search(r'(\\d+)(.*)', described).groups()\n",
    "            try:\n",
    "                described_number = int(described_number)\n",
    "            except ValueError:\n",
    "                pass\n",
    "        except AttributeError:\n",
    "            described_number = 0\n",
    "            described_note = described\n",
    "        except TypeError:\n",
    "            described_number = 0\n",
    "            described_note = ''\n",
    "        return {'items_described': described_number, 'items_described_note': described_note.strip()}\n",
    "    \n",
    "    def get_quantity_locations(self):\n",
    "        cell = self.get_cell('Quantity and location')\n",
    "        locations = []\n",
    "        if cell:\n",
    "            for location in cell.findAll('li'):\n",
    "                try:\n",
    "                    quantity, location = re.search(r'(\\d+\\.*\\d*) metres held in ([A-Z,a-z]+)', location.string).groups()\n",
    "                    quantity = float(quantity)\n",
    "                except AttributeError:\n",
    "                    quantity = None\n",
    "                    location = None\n",
    "                locations.append({\n",
    "                    'quantity': quantity,\n",
    "                    'location': location\n",
    "                })\n",
    "        return locations\n",
    "    \n",
    "    def get_access_status_totals(self):\n",
    "        totals = {}\n",
    "        for status in ACCESS:\n",
    "            totals[status] = RSItemSearch(series=self.identifier, access=status).total_results\n",
    "            time.sleep(0.5)\n",
    "        return totals\n",
    "    \n",
    "    def generate_cache_key(self):\n",
    "        cache_key = f'series_{self.identifier}'\n",
    "        if self.digitised:\n",
    "            cache_key = f'{cache_key}_digitised'\n",
    "        if self.access_status:\n",
    "            cache_key = f'{cache_key}_access'\n",
    "        return cache_key\n",
    "    \n",
    "    def refresh_cache(self):\n",
    "        '''\n",
    "        Delete data for this item from the cache, then extract a fresh version from RecordSearch.\n",
    "        '''\n",
    "        cache_key = self.generate_cache_key()\n",
    "        del cache_db[cache_key]\n",
    "        self.__init__(self.identifier, self.cache, self.digitised, self.access_status)\n",
    "        \n",
    "    def get_series(self):\n",
    "        # Try to retrieve from cache first\n",
    "        cache_key = self.generate_cache_key()\n",
    "        try:\n",
    "            series = cache_db[cache_key]\n",
    "        # If not in the cache and the details are not supplied, get it from RS.\n",
    "        except KeyError:\n",
    "            if not self.details:\n",
    "                self.get_entity_page()\n",
    "                self.details = self.get_details()\n",
    "            if self.details:\n",
    "                series = {\n",
    "                    'identifier': self.identifier,\n",
    "                    'title': self.get_value('Title'),\n",
    "                    'physical_format': self.get_value('Predominant physical format'),\n",
    "                    'arrangement': self.get_value('System of arrangement/ control'),\n",
    "                    'control_symbols': self.get_value('Range of control symbols'),\n",
    "                    'locations': self.get_quantity_locations(),\n",
    "                    'note': self.get_note('Series note'),\n",
    "                    'recording_agencies': self.get_relations('recording'),\n",
    "                    'controlling_agencies': self.get_relations('controlling'),\n",
    "                    'previous_series': self.get_relations('Previous'),\n",
    "                    'subsequent_series': self.get_relations('Subsequent'),\n",
    "                    'controlling_series': self.get_relations('Controlling'),\n",
    "                    'related_series': self.get_relations('Related'),\n",
    "                    'availability': self.get_value('Visibility & availability indicator'),\n",
    "                    'retrieved': arrow.now(tz='Australia/Sydney').isoformat()\n",
    "                }\n",
    "                series.update(self.get_number_described())\n",
    "                series.update(self.get_formatted_dates('Contents dates', 'contents_'))\n",
    "                series.update(self.get_formatted_dates('Accumulation dates', 'accumulation_'))\n",
    "                if self.digitised:\n",
    "                    series['items_digitised'] = RSItemSearch(series=self.identifier, digital=True).total_results\n",
    "                if self.access_status:\n",
    "                    series['access_status_totals'] = self.get_access_status_totals()\n",
    "                if self.cache:\n",
    "                    # Add to the cache\n",
    "                    cache_db[cache_key] = series\n",
    "            else:\n",
    "                series = {'identifier': self.identifier, 'error': 'Series not found'}\n",
    "        return series\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f'NAA: {self.identifier}'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A series is a group of records that have something in common, for example, they might have been part of the same filing system. Series can be related to other series, and to agencies. A single series can also be held across multiple locations. All this means the data can be quite complex.\n",
    "\n",
    "Not that as well as the standard RecordSearch metadata, the scraper can also extract some extra information about the series, such as the number of items digitised, and the access status of items in the series. \n",
    "\n",
    "Here are the fields returned:\n",
    "\n",
    "* `identifier` (string)\n",
    "* `title` (string)\n",
    "* `physical_format` (string)\n",
    "* `arrangement` (string)\n",
    "* `control_symbols` (string)\n",
    "* `locations` (list) – a list of locations, each with the fields:\n",
    "    * `quantity` (string)\n",
    "    * `location` (string)\n",
    "* `recording_agencies` – a list of agencies, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `controlling_agencies` – a list of agencies, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `previous_series` – a list of series, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `subsequent_series` – a list of series, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `controlling_series` – a list of series, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `related_series` (list) – a list of series, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `items_described` (integer)\n",
    "* `items_described_note` (string)\n",
    "* `contents_date_str` (string)\n",
    "* `contents_start_date` (ISO formatted date)\n",
    "* `contents_end_date` (ISO formatted date)\n",
    "* `accumulation_date_str` (string)\n",
    "* `accumulation_start_date` (ISO formatted date)\n",
    "* `accumulation_end_date` (ISO formatted date)\n",
    "* `items_digitised` (integer) – the number of items in this series that have been digitised\n",
    "* `access_status_totals` (dict) – the number of items in each of the access status categories, OPEN, OWE, CLOSED, and NYE.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To retrieve information about a series, just give `RSSeries()` the series number."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series = RSSeries('A863')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can then access the series data using the `.data` attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'identifier': 'A863',\n",
       " 'title': 'Correspondence files, single number series relating to Civil Defence matters',\n",
       " 'physical_format': 'PAPER FILES AND DOCUMENTS',\n",
       " 'arrangement': 'Single number',\n",
       " 'control_symbols': '1 - 468 (with gaps)',\n",
       " 'locations': [{'quantity': 0.72, 'location': 'ACT'}],\n",
       " 'note': 'Function and purpose This series consists of correspondence files covering a wide number of topics generally relating to Civil Defence matters. Included are reports from a wide range of sources, discussions on shelters, blood transfusion and medical services to mention a few. System of arrangement and control This series is controlled by a single number system. There are many gaps in the numbering. Finding aids All items in this series have been entered onto the National Archives of Australia item level database. Series history National Archives staff reviewed this series in 2001. Some records transferred into the National Archives of Australia’s custody were assessed as no longer justifying retention in the collection. These records were disposed of in accordance with records disposal authorities approved by the National Archives of Australia and by the Commonwealth agency responsible for the functions to which the records related. Disposal History A863/3 was withdrawn by the agency on 21 September 2016.',\n",
       " 'recording_agencies': [{'date_str': '01 Jan 1944 - 31 Dec 1948',\n",
       "   'start_date': '1944-01-01',\n",
       "   'end_date': '1948-12-31',\n",
       "   'identifier': 'CA 31',\n",
       "   'title': 'Department of the Interior [II], Central Office'},\n",
       "  {'date_str': '01 Jan 1949 - 31 Dec 1961',\n",
       "   'start_date': '1949-01-01',\n",
       "   'end_date': '1961-12-31',\n",
       "   'identifier': 'CA 541',\n",
       "   'title': 'Directorate of Civil Defence'}],\n",
       " 'controlling_agencies': [{'date_str': '1944 -',\n",
       "   'start_date': '1944',\n",
       "   'end_date': None,\n",
       "   'identifier': 'CA 46',\n",
       "   'title': 'Department of Defence [III], Central Office'}],\n",
       " 'previous_series': [],\n",
       " 'subsequent_series': [{'date_str': '31 Dec 1961',\n",
       "   'start_date': '1961-12-31',\n",
       "   'end_date': None,\n",
       "   'identifier': 'A5518',\n",
       "   'title': 'General correspondence files, annual single number series with \"CD\" or \"NDO\" prefix'}],\n",
       " 'controlling_series': [],\n",
       " 'related_series': [],\n",
       " 'availability': None,\n",
       " 'retrieved': '2023-01-20T10:47:48.175830+11:00',\n",
       " 'items_described': 50,\n",
       " 'items_described_note': \"Click to see items listed on RecordSearch. Please contact the National Reference Service if you can't find the record you want as not all items from the series may be on RecordSearch.\",\n",
       " 'contents_date_str': '1944 - 1961',\n",
       " 'contents_start_date': '1944',\n",
       " 'contents_end_date': '1961',\n",
       " 'accumulation_date_str': '1944 - 1961',\n",
       " 'accumulation_start_date': '1944',\n",
       " 'accumulation_end_date': '1961',\n",
       " 'items_digitised': 0,\n",
       " 'access_status_totals': {'OPEN': 50, 'OWE': 0, 'CLOSED': 0, 'NYE': 0}}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(series.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can find out how many items within the series are closed to public access. In this case, it should be none."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert series.data['access_status_totals']['CLOSED'] == 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can access both the number of items described and digitised within each series. We'd expect the number described to be greater than or equal to the number digitised."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert series.data['items_described'] >= series.data['items_digitised']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, the scraper adds some extra information to the basic metadata – `items_digitised` and `access_status_totals`. To obtain these values, the scraper runs item searches – one to find digitised files, and another four to find all the access status values. This can slow things down considerably. If you want a quick response and don't care about these values, you can set `include_number_digitised` and/or `include_access_status` to `False`.\n",
    "\n",
    "In the case below, the series data should **not** include a value for `items_digitised`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series = RSSeries('A3', include_number_digitised=False)\n",
    "\n",
    "assert 'items_digitised' not in series.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The extracted data is saved into a simple key-value cache to speed up repeat requests. If you want to scrape a fresh version, use `.refresh_cache()`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "### RSSeries.refresh_cache\n",
       "\n",
       ">      RSSeries.refresh_cache ()\n",
       "\n",
       "Delete data for this item from the cache, then extract a fresh version from RecordSearch."
      ],
      "text/plain": [
       "---\n",
       "\n",
       "### RSSeries.refresh_cache\n",
       "\n",
       ">      RSSeries.refresh_cache ()\n",
       "\n",
       "Delete data for this item from the cache, then extract a fresh version from RecordSearch."
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(RSSeries.refresh_cache, name=\"RSSeries.refresh_cache\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can check that this has worked by comparing the value of `retrieved`, which is the date/time the data was scraped."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_retrieved_date = series.data['retrieved']\n",
    "\n",
    "series.refresh_cache()\n",
    "\n",
    "new_retrieved_date = series.data['retrieved']\n",
    "\n",
    "assert old_retrieved_date != new_retrieved_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "\n",
    "class RSSeriesSearch(RSSearch):\n",
    "    '''\n",
    "    Search for series in RecordSearch.\n",
    "    \n",
    "    Supply any of the series search parameters as kwargs to initialise the search.\n",
    "    \n",
    "    Optional parameters:\n",
    "    \n",
    "    * `results_per_page` (default: 20)\n",
    "    * `sort` (default: 1 – order by id)\n",
    "    * `page` – to retrieve a specific page of results\n",
    "    * `record_detail` – amount of detail to include, options are:\n",
    "        * 'brief' (default) – just the info in the search results\n",
    "        * 'full' – get the full individual record for each result (slow)\n",
    "    \n",
    "    To access a page of results, use the `.get_results()` method. \n",
    "    This method increments the results page, so you can call it in a loop\n",
    "    to retrieve the complete result set.\n",
    "    \n",
    "    Useful attributes:\n",
    "    \n",
    "    * `.total_results` – the total number of results in the results set\n",
    "    * `.total_pages` – the total number of result pages\n",
    "    * `.kwargs` – a dict containing the supplied search parameters\n",
    "    * `.params` – a dict containing the values of the optional parameters\n",
    "    '''\n",
    "    \n",
    "    entity_type = 'series'\n",
    "    search_params = SERIES_FORM\n",
    "    search_page = 'AdvSearchSeries.aspx'\n",
    "    entity = RSSeries\n",
    "    \n",
    "    def __init__(self, results_per_page=20, sort=1, record_detail='brief', **kwargs):\n",
    "        super(RSSeriesSearch, self).__init__(results_per_page=results_per_page, sort=sort, record_detail=record_detail, **kwargs)\n",
    "    \n",
    "    def get_locations(self, cell):\n",
    "        locations = []\n",
    "        for row in cell.stripped_strings:\n",
    "            location, quantity = row.split(': ')\n",
    "            locations.append({'location': location, 'quantity': quantity})\n",
    "        return locations\n",
    "    \n",
    "    def get_items_described(self, cell):\n",
    "        if link := cell.find('a'):\n",
    "            return int(link.string.strip())\n",
    "        else:\n",
    "            return int(cell.string.strip())   \n",
    "\n",
    "    def process_row(self, row):\n",
    "        cells = row.find_all('td')\n",
    "        series = {\n",
    "            'identifier': cells[1].string.strip(),\n",
    "            'title': cells[2].contents[0].string.strip(),\n",
    "            'locations': self.get_locations(cells[5]),\n",
    "            'items_described': self.get_items_described(cells[6])\n",
    "        }\n",
    "        accumulated_date_str = cells[3].string\n",
    "        series.update(self.process_date_string(accumulated_date_str, 'accumulation_'))\n",
    "        contents_date_str = cells[4].string\n",
    "        series.update(self.process_date_string(contents_date_str, 'contents_'))\n",
    "        return series\n",
    "    \n",
    "    def process_page(self, soup, record_detail):\n",
    "        '''\n",
    "        Extract item data from a search results page.\n",
    "        Level of item data can be varied using the `record_detail` parameter:\n",
    "            * 'brief' - just the data in the search results\n",
    "            * 'full': retrieve the individual item record to get extra fields\n",
    "        '''\n",
    "        # There's a list of items\n",
    "        if details := soup.find(id=re.compile('tblSeriesDetails$')):\n",
    "            series = self.process_list(details)\n",
    "        # There's a single item\n",
    "        elif soup.find(id=re.compile('ContentPlaceHolderSNR_ucSeriesDetails_phDetailsView')) is not None:\n",
    "            # If there's only one item & you have a 'sort' param, RS returns an empty details page\n",
    "            # So we'll remove any sort parameter & resubmit the url\n",
    "            details_url = re.sub(r'(\\?|&)sort=\\d', '', self.browser.url)\n",
    "            self.browser.open(details_url)\n",
    "            details = self.browser.page.find('div', 'mainDetailsTable')\n",
    "            series = [self.entity(details=details, cache=False).data]\n",
    "        # No items?\n",
    "        else:\n",
    "            raise Exception('No results found on page!')\n",
    "        # Get full record information\n",
    "        if record_detail == 'full':\n",
    "            series = self.get_full_details(series)\n",
    "        return series"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Series search parameters\n",
    "\n",
    "These are the parameters you can supply as keyword arguments to `RSSeriesSearch`.\n",
    "\n",
    "| Parameter | Input type | Values |\n",
    "| :--- | :--- | :--- |\n",
    "| **kw** | Text | Keywords or phrase to search for|\n",
    "| **kw_options** | Select | How to combine the keywords – see [keyword options](#Series-keyword-options) |\n",
    "| **kw_exclude** | Text | Keywords or phrase to exclude |\n",
    "| **kw_exclude_options** | Select | How to combine the keywords – see [keyword options](#Series-keyword-options) |\n",
    "| **search_notes** | Checkbox | Set to `True` to search notes as well as titles |\n",
    "| **series_id** | Text | Search for this series identifier |\n",
    "| **date_from** | Text | Include series with content after this date (year only) – eg '1925' |\n",
    "| **date_to** | Text | Include series with content before this date (year only) – eg '1945' |\n",
    "| **formats** | Select | Limit to series with items in this format – see [format options](#Series-format-options) |\n",
    "| **formats_exclude** | Select | Exclude series with items in this format  – see [format options](#Series-format-options) |\n",
    "| **locations** | Select | Limit to series held in this location – see list of [locations](#Series-location-options) |\n",
    "| **locations_exclude** | Select | Exclude series held in this location – see list of [locations](#Series-location-options) |\n",
    "| **agency_recording** | Select | Limit to series created by this agency or person |\n",
    "| **agency_controlling** | Select | Limit to series controlled by this agency or person |\n",
    "\n",
    "### Series keyword options\n",
    "\n",
    "Use one of the following values to specify how keywords or phrases should be treated using the `kw_options` parameter. The default is 'ALL'.\n",
    "\n",
    "* **'ALL'** (default) – must include all keywords\n",
    "* **'ANY'** – must include at least one of the keywords\n",
    "* **'EXACT'** – treat the keywords as a phrase\n",
    "\n",
    "### Series location options\n",
    "\n",
    "Use one of the following values with the `locations` and `locations_exclude` parameters to limit your results to items held in that location. The default is to include all locations.\n",
    "\n",
    "* **'NAT,ACT'** – National office (ACT)\n",
    "* **'AWM'** – Australian War Memorial\n",
    "* **'NSW'**\n",
    "* **'NT'**\n",
    "* **'QLD'**\n",
    "* **'SA'**\n",
    "* **'TAS'**\n",
    "* **'VIC'**\n",
    "* **'WA'**\n",
    "\n",
    "### Series format options\n",
    "\n",
    "Use one of the following values with the `formats` and `formats_exclude` parameters to limit your results to series containing that format. The default is to include all formats.\n",
    "\n",
    "* **'Paper files and documents'**\n",
    "* **'Index cards'**\n",
    "* **'Bound volumes'**\n",
    "* **'Cartographic records'**\n",
    "* **'Photographs'**\n",
    "* **'Microforms'**\n",
    "* **'Audio-visual records'**\n",
    "* **'Audio records'**\n",
    "* **'Electronic records'**\n",
    "* **'3-dimensional records'**\n",
    "* **'Scientific specimens'**\n",
    "* **'Textiles'**\n",
    "\n",
    "### Examples\n",
    "\n",
    "Initialise a search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_results = RSSeriesSearch(agency_recording='CA 1196')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can access the `.total_results` attribute to find out how many results there are."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "series_results.total_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Naturally enough, the `.total_results` value should be an integer, as should `.total_pages`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert isinstance(series_results.total_results, int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert isinstance(series_results.total_pages, int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'results_per_page': 20, 'sort': 1, 'record_detail': 'brief'}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "series_results.params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "### RSSeriesSearch.get_results\n",
       "\n",
       ">      RSSeriesSearch.get_results (page=None)\n",
       "\n",
       "Return a list of results from a search results page.\n",
       "\n",
       "The `page` value is incremented with each request, so you can call this method\n",
       "in a loop to retrieve the complete results set. When you reach then of the results, \n",
       "this method will return an empty list.\n",
       "\n",
       "Optional parameter:\n",
       "\n",
       "* `page` – request a specific page from the results set"
      ],
      "text/plain": [
       "---\n",
       "\n",
       "### RSSeriesSearch.get_results\n",
       "\n",
       ">      RSSeriesSearch.get_results (page=None)\n",
       "\n",
       "Return a list of results from a search results page.\n",
       "\n",
       "The `page` value is incremented with each request, so you can call this method\n",
       "in a loop to retrieve the complete results set. When you reach then of the results, \n",
       "this method will return an empty list.\n",
       "\n",
       "Optional parameter:\n",
       "\n",
       "* `page` – request a specific page from the results set"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(RSSeriesSearch.get_results, name=\"RSSeriesSearch.get_results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'total_results': 100,\n",
       " 'page': 2,\n",
       " 'number_of_results': 20,\n",
       " 'results': [{'identifier': 'A1644',\n",
       "   'title': \"'[Master] subject index' [list of indexable headings] for correspondence files, annual single number series\",\n",
       "   'locations': [],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': 'circa1967 - circa1973',\n",
       "   'accumulation_start_date': None,\n",
       "   'accumulation_end_date': None,\n",
       "   'contents_date_str': 'circa1967 - circa1973',\n",
       "   'contents_start_date': None,\n",
       "   'contents_end_date': None,\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1645',\n",
       "   'title': 'Subject index cards for correspondence files, annual single number series',\n",
       "   'locations': [{'location': 'ACT', 'quantity': '1.35m'}],\n",
       "   'items_described': 6,\n",
       "   'accumulation_date_str': '1967 - 1973',\n",
       "   'accumulation_start_date': '1967',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1967 - 1973',\n",
       "   'contents_start_date': '1967',\n",
       "   'contents_end_date': '1973',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1646',\n",
       "   'title': 'Movement cards for annual single number series',\n",
       "   'locations': [],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': 'circa1967 - circa1970',\n",
       "   'accumulation_start_date': None,\n",
       "   'accumulation_end_date': None,\n",
       "   'contents_date_str': 'circa1967 - circa1970',\n",
       "   'contents_start_date': None,\n",
       "   'contents_end_date': None,\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1647',\n",
       "   'title': 'Correspondence files, DES series (Cabinet)',\n",
       "   'locations': [],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1967 - ',\n",
       "   'accumulation_start_date': '1967',\n",
       "   'accumulation_end_date': None,\n",
       "   'contents_date_str': '1967 - ',\n",
       "   'contents_start_date': '1967',\n",
       "   'contents_end_date': None,\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1648',\n",
       "   'title': 'Cabinet file register',\n",
       "   'locations': [],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1967 - ',\n",
       "   'accumulation_start_date': '1967',\n",
       "   'accumulation_end_date': None,\n",
       "   'contents_date_str': '1967 - ',\n",
       "   'contents_start_date': '1967',\n",
       "   'contents_end_date': None,\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1649',\n",
       "   'title': 'Subject index cards, Cabinet files',\n",
       "   'locations': [],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1967 - ',\n",
       "   'accumulation_start_date': '1967',\n",
       "   'accumulation_end_date': None,\n",
       "   'contents_date_str': '1967 - ',\n",
       "   'contents_start_date': '1967',\n",
       "   'contents_end_date': None,\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1873',\n",
       "   'title': \"Staffing files [individual staff files], single number series with 'S' prefix\",\n",
       "   'locations': [],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1949 - 1972',\n",
       "   'accumulation_start_date': '1949',\n",
       "   'accumulation_end_date': '1972',\n",
       "   'contents_date_str': '1930 - 1983',\n",
       "   'contents_start_date': '1930',\n",
       "   'contents_end_date': '1983',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1904',\n",
       "   'title': \"Name index cards, DES [Department of Education and Science] State Offices (formerly 'COE [Commonwealth Office of Education] All States')\",\n",
       "   'locations': [{'location': 'ACT', 'quantity': '0.18m'}],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1965 - 1973',\n",
       "   'accumulation_start_date': '1965',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1961 - 1972',\n",
       "   'contents_start_date': '1961',\n",
       "   'contents_end_date': '1972',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1905',\n",
       "   'title': 'Movement Cards for Interior Files',\n",
       "   'locations': [],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1968 - 1968',\n",
       "   'accumulation_start_date': '1968',\n",
       "   'accumulation_end_date': '1968',\n",
       "   'contents_date_str': '1963 - 1968',\n",
       "   'contents_start_date': '1963',\n",
       "   'contents_end_date': '1968',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1906',\n",
       "   'title': 'Name index cards, Commonwealth Co-operation in Education, alphabetical series',\n",
       "   'locations': [{'location': 'ACT', 'quantity': '0.18m'}],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1968 - 1973',\n",
       "   'accumulation_start_date': '1968',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1966 - 1972',\n",
       "   'contents_start_date': '1966',\n",
       "   'contents_end_date': '1972',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1907',\n",
       "   'title': 'Name index cards, Anzac Fellowship, alphabetical series',\n",
       "   'locations': [{'location': 'ACT', 'quantity': '0.18m'}],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1967 - 1973',\n",
       "   'accumulation_start_date': '1967',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1967 - 1972',\n",
       "   'contents_start_date': '1967',\n",
       "   'contents_end_date': '1972',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1908',\n",
       "   'title': 'Name index cards, Commonwealth Scholarship and Fellowship Plan',\n",
       "   'locations': [{'location': 'ACT', 'quantity': '0.18m'}],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1968 - 1973',\n",
       "   'accumulation_start_date': '1968',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1966 - 1972',\n",
       "   'contents_start_date': '1966',\n",
       "   'contents_end_date': '1972',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1909',\n",
       "   'title': 'Name index cards, Scholarships by Foreign Governments, alphabetical series',\n",
       "   'locations': [{'location': 'ACT', 'quantity': '0.18m'}],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1967 - 1973',\n",
       "   'accumulation_start_date': '1967',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1966 - 1972',\n",
       "   'contents_start_date': '1966',\n",
       "   'contents_end_date': '1972',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1910',\n",
       "   'title': 'Name index cards, CBI Scholarships (Confederation of British Industries) alphabetical series',\n",
       "   'locations': [{'location': 'ACT', 'quantity': '0.09m'}],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1968 - 1973',\n",
       "   'accumulation_start_date': '1968',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1968 - 1972',\n",
       "   'contents_start_date': '1968',\n",
       "   'contents_end_date': '1972',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1911',\n",
       "   'title': 'Name index cards, Australian American Educational Foundation, alphabetical series',\n",
       "   'locations': [{'location': 'ACT', 'quantity': '0.18m'}],\n",
       "   'items_described': 1,\n",
       "   'accumulation_date_str': '1968 - 1973',\n",
       "   'accumulation_start_date': '1968',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1965 - 1972',\n",
       "   'contents_start_date': '1965',\n",
       "   'contents_end_date': '1972',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1912',\n",
       "   'title': \"Correspondence files, single number series with '68' prefix (12, 000 block) (School Libraries)\",\n",
       "   'locations': [],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1968 - 1973',\n",
       "   'accumulation_start_date': '1968',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1968 - 1973',\n",
       "   'contents_start_date': '1968',\n",
       "   'contents_end_date': '1973',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1913',\n",
       "   'title': \"Register book for correspondence files, single number series with '68' prefix (12,000 block) ('School Libraries')\",\n",
       "   'locations': [],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1968 - 1973',\n",
       "   'accumulation_start_date': '1968',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1968 - 1973',\n",
       "   'contents_start_date': '1968',\n",
       "   'contents_end_date': '1973',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1914',\n",
       "   'title': 'Movement cards for correspondence files (School Libraries)',\n",
       "   'locations': [],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1968 - 1973',\n",
       "   'accumulation_start_date': '1968',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1968 - 1973',\n",
       "   'contents_start_date': '1968',\n",
       "   'contents_end_date': '1973',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A1915',\n",
       "   'title': 'Name index cards, School Libraries',\n",
       "   'locations': [{'location': 'ACT', 'quantity': '0.18m'}],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1968 - 1973',\n",
       "   'accumulation_start_date': '1968',\n",
       "   'accumulation_end_date': '1973',\n",
       "   'contents_date_str': '1968 - 1972',\n",
       "   'contents_start_date': '1968',\n",
       "   'contents_end_date': '1972',\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'},\n",
       "  {'identifier': 'A2102',\n",
       "   'title': 'Paid Claims (Treasury Form 12)',\n",
       "   'locations': [],\n",
       "   'items_described': 0,\n",
       "   'accumulation_date_str': '1967 - ',\n",
       "   'accumulation_start_date': '1967',\n",
       "   'accumulation_end_date': None,\n",
       "   'contents_date_str': None,\n",
       "   'contents_start_date': None,\n",
       "   'contents_end_date': None,\n",
       "   'retrieved': '2023-01-20T10:48:04.337177+11:00'}],\n",
       " 'retrieved': '2023-01-20T10:48:04.347385+11:00'}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "series_results.get_results(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calling `.refresh_cache` will remove all of the data for this search from the cache, and set the results page back to 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "### RSSeriesSearch.refresh_cache\n",
       "\n",
       ">      RSSeriesSearch.refresh_cache ()\n",
       "\n",
       "Delete data for this search from the cache, then retrieve a fresh version from RecordSearch."
      ],
      "text/plain": [
       "---\n",
       "\n",
       "### RSSeriesSearch.refresh_cache\n",
       "\n",
       ">      RSSeriesSearch.refresh_cache ()\n",
       "\n",
       "Delete data for this search from the cache, then retrieve a fresh version from RecordSearch."
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(RSSeriesSearch.refresh_cache, name=\"RSSeriesSearch.refresh_cache\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "\n",
    "class RSAgency(RSEntity):\n",
    "    '''\n",
    "    Class used for extracting data about an individual agency.\n",
    "    You need to supply the following parameter:\n",
    "    \n",
    "    * `identifier` – the series number, eg 'A1', 'B2455' \n",
    "    \n",
    "    The series data is obtained by accessing the series' `.data` attribute.\n",
    "    '''\n",
    "    entity_type = 'agency'\n",
    "    \n",
    "    def __init__(self, identifier=None, cache=True, details=None, include_series_count=True):\n",
    "        super(RSAgency, self).__init__(identifier, cache)\n",
    "        self.series = include_series_count\n",
    "        self.details = details\n",
    "        if details:\n",
    "            self.identifier = self.get_value('Agency number')\n",
    "        self.data = self.get_agency()\n",
    "    \n",
    "    def get_series_count(self):\n",
    "        series_count = RSSeriesSearch(agency_recording=self.identifier).total_results\n",
    "        return series_count\n",
    "\n",
    "    def get_agency(self):\n",
    "        # Try to retrieve from cache first\n",
    "        cache_key = f'{self.entity_type}_{self.identifier}'\n",
    "        try:\n",
    "            agency = cache_db[cache_key]\n",
    "        # If not in the cache and the details are not supplied, get it from RS.\n",
    "        except KeyError:\n",
    "            if not self.details:\n",
    "                self.get_entity_page()\n",
    "                self.details = self.get_details()\n",
    "            if self.details:\n",
    "                agency = {\n",
    "                    'identifier': self.identifier,\n",
    "                    'title': self.get_value('Title'),\n",
    "                    'institution_title': self.get_value('Institution title'),\n",
    "                    'agency_status': self.get_value('Agency status'),\n",
    "                    'location': self.get_value('Location'),\n",
    "                    'functions': self.get_relations('Function'),\n",
    "                    'note': self.get_note('Agency note'),\n",
    "                    'controlling_organisation': self.get_relations('Organisation controlling'),\n",
    "                    'previous_agencies': self.get_relations('Previous'),\n",
    "                    'subsequent_agencies': self.get_relations('Subsequent'),\n",
    "                    'superior_agencies': self.get_relations('Superior'),\n",
    "                    'controlled_agencies': self.get_relations('Controlled'),\n",
    "                    'associated_people': self.get_relations('Persons'),\n",
    "                    'retrieved': arrow.now(tz='Australia/Sydney').isoformat()\n",
    "                }\n",
    "                # series.update(self.get_number_described())\n",
    "                agency.update(self.get_formatted_dates('Date range'))\n",
    "                if self.series:\n",
    "                    agency['number_of_series'] = self.get_series_count()\n",
    "                if self.cache:\n",
    "                    # Add to the cache\n",
    "                    cache_db[cache_key] = agency\n",
    "            else:\n",
    "                agency = {'identifier': self.identifier, 'error': 'Agency not found'}\n",
    "        return agency\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f'{self.identifier}, self.data[\"title\"]'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here are the the fields returned:\n",
    "    \n",
    "* `identifier` (string)\n",
    "* `title` (string)\n",
    "* `location` (string)\n",
    "* `functions` – a list of functions performed by this agency, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `controlling_organisation` – a list of organisations, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `previous_agencies` – a list of agencies, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `subsequent_agencies` – a list of agencies, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `superior_agencies` – a list of agencies, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `controlled_agencies` – a list of agencies, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `associated_people` (list) – a list of people, each with the fields:\n",
    "    * `identifier` (string)\n",
    "    * `title` (string)\n",
    "    * `date_str` (string)\n",
    "    * `start_date` (ISO formatted date)\n",
    "    * `end_date` (ISO formatted date)\n",
    "* `date_str` (string)\n",
    "* `start_date` (ISO formatted date)\n",
    "* `end_date` (ISO formatted date)\n",
    "* `number_of_series` (integer) – number of series created by this agency\n",
    "* `retrieved` (ISO formatted date)\n",
    "\n",
    "### Examples\n",
    "\n",
    "To retrieve information about an agency, just give `RSAgency` the agency identifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agency = RSAgency('CA 343')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can then access the agency data using the `.data` attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'identifier': 'CA 343',\n",
       " 'title': 'Industrial Atomic Energy Policy Committee',\n",
       " 'institution_title': None,\n",
       " 'agency_status': 'Head Office',\n",
       " 'location': 'New South Wales',\n",
       " 'functions': [{'date_str': '01 Jan 1949 - 30 Apr 1952',\n",
       "   'start_date': '1949-01-01',\n",
       "   'end_date': '1952-04-30',\n",
       "   'identifier': 'ENERGY',\n",
       "   'title': 'ENERGY'}],\n",
       " 'note': 'On 19 August, 1949 the Commonwealth government decided to establish an Industrial Atomic Energy Policy Committee. The Committee was set up before the end of the year. Its function was to study possible industrial applications of atomic energy and to recommend a possible programme of development. The Committee consisted of Professor Marcus Oliphant (Chairman), Dr F.W.G. White, Chief Executive Officer of the Commonwealth Scientific and Industrial Research Organization (Deputy Chairman); Professor J.P. Baxter, Professor of Chemical Engineering, New South Wales University of Technology, Mr H.P. Breen, Secretary, Department of Supply and Development, Mr H.J. Goodes, representing the Treasury, Professor L.H. Martin, Professor of Physics, University of Melbourne and Commonwealth Defence Scientific Adviser and Dr H.G. Raggett, Director of the Bureau of Mineral Resources, Geology and Geophysics. (Australian Atomic Energy Commission, First Annual Report, 1953, p.7) The Committee held a succession of meetings collecting and studying information on technical aspects of atomic energy and arrangements overseas for atomic energy development and in the light of these studies, prepared basic recommendations for an Australian programme. The Committee also prepared basic proposals for the training of staff for a research and development programme. By 1952 the Committee came to the conclusion that the problems referred to it required consideration under wider terms of reference. Accordingly it made a recommendation that it be disbanded and replaced by a new committee with revised functions. This recommendation was accepted and in April 1952 the Atomic Energy Policy Committee (CA 332) was established to replace the Industrial Atomic Energy Policy Committee. Historical agency address Subject to research Sydney, NSW',\n",
       " 'controlling_organisation': [{'date_str': '01 Jan 1949 - 30 Apr 1952',\n",
       "   'start_date': '1949-01-01',\n",
       "   'end_date': '1952-04-30',\n",
       "   'identifier': 'CO 1',\n",
       "   'title': 'COMMONWEALTH OF AUSTRALIA'}],\n",
       " 'previous_agencies': [],\n",
       " 'subsequent_agencies': [{'date_str': '',\n",
       "   'start_date': None,\n",
       "   'end_date': None,\n",
       "   'identifier': 'CA 332',\n",
       "   'title': 'Atomic Energy Policy Committee'}],\n",
       " 'superior_agencies': [{'date_str': '01 Jan 1949 - 16 Mar 1950',\n",
       "   'start_date': '1949-01-01',\n",
       "   'end_date': '1950-03-16',\n",
       "   'identifier': 'CA 54',\n",
       "   'title': 'Department of Supply and Development [II]'},\n",
       "  {'date_str': '16 Mar 1950 - 30 Apr 1952',\n",
       "   'start_date': '1950-03-16',\n",
       "   'end_date': '1952-04-30',\n",
       "   'identifier': 'CA 57',\n",
       "   'title': 'Department of Supply, Central Office'}],\n",
       " 'controlled_agencies': [],\n",
       " 'associated_people': [],\n",
       " 'retrieved': '2023-01-20T10:48:05.099605+11:00',\n",
       " 'date_str': '01 Jan 1949 -  30 Apr 1952',\n",
       " 'start_date': '1949-01-01',\n",
       " 'end_date': '1952-04-30',\n",
       " 'number_of_series': 0}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agency.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use `agency.data[FIELD NAME]` to access individual fields. The `agency_status` value of this agency should be 'Head Office'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert agency.data['agency_status'] == 'Head Office'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The extracted data is saved into a simple key-value cache to speed up repeat requests. If you want to scrape a fresh version, use `.refresh_cache()`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "### RSAgency.refresh_cache\n",
       "\n",
       ">      RSAgency.refresh_cache ()\n",
       "\n",
       "Delete data for this entity from the cache, then extract a fresh version from RecordSearch."
      ],
      "text/plain": [
       "---\n",
       "\n",
       "### RSAgency.refresh_cache\n",
       "\n",
       ">      RSAgency.refresh_cache ()\n",
       "\n",
       "Delete data for this entity from the cache, then extract a fresh version from RecordSearch."
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(RSAgency.refresh_cache, name=\"RSAgency.refresh_cache\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "\n",
    "class RSAgencySearch(RSSearch):\n",
    "    '''\n",
    "    Search for agencies in RecordSearch.\n",
    "    \n",
    "    Supply any of the agency search parameters as kwargs to initialise the search.\n",
    "    \n",
    "    Optional parameters:\n",
    "    \n",
    "    * `results_per_page` (default: 20)\n",
    "    * `sort` (default: 1 – order by id)\n",
    "    * `page` – to retrieve a specific page of results\n",
    "    * `record_detail` – amount of detail to include, options are:\n",
    "        * 'brief' (default) – just the info in the search results\n",
    "        * 'full' – get the full individual record for each result (slow)\n",
    "    \n",
    "    To access a page of results, use the `.get_results()` method. \n",
    "    This method increments the results page, so you can call it in a loop\n",
    "    to retrieve the complete result set.\n",
    "    \n",
    "    Useful attributes:\n",
    "    \n",
    "    * `.total_results` – the total number of results in the results set\n",
    "    * `.total_pages` – the total number of result pages\n",
    "    * `.kwargs` – a dict containing the supplied search parameters\n",
    "    * `.params` – a dict containing the values of the optional parameters\n",
    "    '''\n",
    "    \n",
    "    entity_type = 'agency'\n",
    "    search_params = AGENCY_FORM\n",
    "    search_page = 'AdvSearchAgencies.aspx'\n",
    "    entity = RSAgency\n",
    "    \n",
    "    def __init__(self, results_per_page=20, sort=1, record_detail='brief', **kwargs):\n",
    "        super(RSAgencySearch, self).__init__(results_per_page=results_per_page, sort=sort, record_detail=record_detail, **kwargs)  \n",
    "\n",
    "    def process_row(self, row):\n",
    "        cells = row.find_all('td')\n",
    "        agency = {\n",
    "            'identifier': cells[1].string.strip(),\n",
    "            'title': cells[2].contents[0].string.strip()\n",
    "        }\n",
    "        date_str = cells[3].string\n",
    "        agency.update(self.process_date_string(date_str))\n",
    "        return agency\n",
    "    \n",
    "    def process_page(self, soup, record_detail):\n",
    "        '''\n",
    "        Extract item data from a search results page.\n",
    "        Level of record data can be varied using the `record_detail` parameter.\n",
    "        '''\n",
    "        # There's a list of items\n",
    "        if results := soup.find(id=re.compile('tblProvDetails$')):\n",
    "            agencies = self.process_list(results)\n",
    "        # There's a single item\n",
    "        elif soup.find(id=re.compile('ucAgencyDetails_phDetailsView')) is not None:\n",
    "            # If there's only one item & you have a 'sort' param, RS returns an empty details page\n",
    "            # So we'll remove any sort parameter & resubmit the url\n",
    "            details_url = re.sub(r'(\\?|&)sort=\\d', '', self.browser.url)\n",
    "            self.browser.open(details_url)\n",
    "            details = self.browser.page.find('div', 'mainDetailsTable')\n",
    "            agencies = [self.entity(details=details, cache=False).data]\n",
    "        # No items?\n",
    "        else:\n",
    "            raise Exception('No results found on page!')\n",
    "        # Get full item information\n",
    "        if record_detail == 'full':\n",
    "            agencies = self.get_full_details(agencies)\n",
    "        return agencies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Agency search parameters\n",
    "\n",
    "These are the parameters you can supply as keyword arguments to `RSAgencySearch`.\n",
    "\n",
    "| Parameter | Input type | Values |\n",
    "| :--- | :--- | :--- |\n",
    "| **kw** | Text | Keywords or phrase to search for|\n",
    "| **kw_options** | Select | How to combine the keywords – see [keyword options](#Agency-keyword-options) |\n",
    "| **kw_exclude** | Text | Keywords or phrase to exclude |\n",
    "| **kw_exclude_options** | Select | How to combine the keywords – see [keyword options](#Agency-keyword-options) |\n",
    "| **function** | Text | Limit to agencies that performed this function – see [note](#Agency-function-note) |\n",
    "| **date_from** | Text | Include agencies that existed after this date (year only) – eg '1925' |\n",
    "| **date_to** | Text | Include agencies that existed before this date (year only) – eg '1945' |\n",
    "| **locations** | Select | Limit to agencies in this location – see list of [locations](#Agency-location-options) |\n",
    "| **locations_exclude** | Select | Exclude agencies in this location – see list of [locations](#Agency-location-options) |\n",
    "| **agency_status** | Select | Limit to agencies with this status – see [list of possible values](#Agency-status-options) |\n",
    "| **agency_status_exclude** | Select | Exclude agencies with this status – see [list of possible values](#Agency-status-options) |\n",
    "\n",
    "### Agency keyword options\n",
    "\n",
    "Use one of the following values to specify how keywords or phrases should be treated using the `kw_options` parameter. The default is 'ALL'.\n",
    "\n",
    "* **'ALL'** (default) – must include all keywords\n",
    "* **'ANY'** – must include at least one of the keywords\n",
    "* **'EXACT'** – treat the keywords as a phrase\n",
    "\n",
    "### Agency function note\n",
    "\n",
    "In theory, functions are a controlled, hierarchical list, but previous examinations have shown that the use of functions in RecordSearch can be inconsistent. Here's a [list of functions](https://github.com/GLAM-Workbench/recordsearch/blob/master/data/functions.txt) extracted from the RecordSearch interface that you can use as values with the `function` parameter.\n",
    "\n",
    "### Agency location options\n",
    "\n",
    "Use one of the following values with the `locations` and `locations_exclude` parameters to limit your results to agencies in that location. The default is to include all locations.\n",
    "\n",
    "* **'NAT,ACT'**\n",
    "* **'COCOS OR CHRISTMAS ISLAND'**\n",
    "* **'NSW'**\n",
    "* **'NT'**\n",
    "* **'OVERSEAS'**\n",
    "* **'PNG'** – Papua New Guinea\n",
    "* **'QLD'**\n",
    "* **'SA'**\n",
    "* **'TAS'**\n",
    "* **'VIC'**\n",
    "* **'WA'**\n",
    "\n",
    "### Agency status options\n",
    "\n",
    "Use one of the following values with the `agency_status` and `agency_status_exclude` parameters to limit your results to agencies with that status. The default is to include all status values.\n",
    "\n",
    "* **'DOS'** – Department of State\n",
    "* **'HO'** – Head Office\n",
    "* **'RO'** – Regional or State Office\n",
    "* **'INTGOV'** – Intergovernmental agency\n",
    "* **'COURT'** – Judicial Court or Tribunal\n",
    "* **'LO'** – Local Office\n",
    "* **'NONEX'** – Non-Executive government agency (Courts, Parliament)\n",
    "\n",
    "### Examples\n",
    "\n",
    "Search for all agencies that have performed the 'SCIENCE' function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agency_search = RSAgencySearch(function='science')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialising the `RSAgencySearch` class sets up the search and retrieves some information about the results set. For example, to see the total number of results, we just access the `.total_results` attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agency_search.total_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "### RSAgencySearch.get_results\n",
       "\n",
       ">      RSAgencySearch.get_results (page=None)\n",
       "\n",
       "Return a list of results from a search results page.\n",
       "\n",
       "The `page` value is incremented with each request, so you can call this method\n",
       "in a loop to retrieve the complete results set. When you reach then of the results, \n",
       "this method will return an empty list.\n",
       "\n",
       "Optional parameter:\n",
       "\n",
       "* `page` – request a specific page from the results set"
      ],
      "text/plain": [
       "---\n",
       "\n",
       "### RSAgencySearch.get_results\n",
       "\n",
       ">      RSAgencySearch.get_results (page=None)\n",
       "\n",
       "Return a list of results from a search results page.\n",
       "\n",
       "The `page` value is incremented with each request, so you can call this method\n",
       "in a loop to retrieve the complete results set. When you reach then of the results, \n",
       "this method will return an empty list.\n",
       "\n",
       "Optional parameter:\n",
       "\n",
       "* `page` – request a specific page from the results set"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(RSAgencySearch.get_results, name=\"RSAgencySearch.get_results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'total_results': 60,\n",
       " 'page': 1,\n",
       " 'number_of_results': 20,\n",
       " 'results': [{'identifier': 'CA 49',\n",
       "   'title': 'Department of Post-War Reconstruction, Central Office',\n",
       "   'date_str': '1942 - 1950',\n",
       "   'start_date': '1942',\n",
       "   'end_date': '1950',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 333',\n",
       "   'title': 'Defence Scientific Advisory Committee',\n",
       "   'date_str': '1947 - 1948',\n",
       "   'start_date': '1947',\n",
       "   'end_date': '1948',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 429',\n",
       "   'title': 'Scientific Advisory Committee, Foodstuffs',\n",
       "   'date_str': '1943 - 1947',\n",
       "   'start_date': '1943',\n",
       "   'end_date': '1947',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 1936',\n",
       "   'title': 'Senate Standing Committee on Education, Science and the Arts',\n",
       "   'date_str': '1971 - 1976',\n",
       "   'start_date': '1971',\n",
       "   'end_date': '1976',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 2423',\n",
       "   'title': 'Australian Science, Technology and Engineering Council (ASTEC)',\n",
       "   'date_str': '1977 - 1989',\n",
       "   'start_date': '1977',\n",
       "   'end_date': '1989',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 3277',\n",
       "   'title': 'Senate Standing Committee on Science and the Environment/ (from 1983) Senate Standing Committee on Science, Technology and the Environment',\n",
       "   'date_str': '1976 - 1987',\n",
       "   'start_date': '1976',\n",
       "   'end_date': '1987',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 4136',\n",
       "   'title': 'Department of Science [III], Central Office',\n",
       "   'date_str': '1984 - 1987',\n",
       "   'start_date': '1984',\n",
       "   'end_date': '1987',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 6703',\n",
       "   'title': 'Senate Standing Committee on Industry, Science, Technology, Transport, Communications and Infrastructure',\n",
       "   'date_str': '1993 - 1994',\n",
       "   'start_date': '1993',\n",
       "   'end_date': '1994',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 6944',\n",
       "   'title': 'House of Representatives Standing Committee on Industry, Science and Resources',\n",
       "   'date_str': '1987 - ',\n",
       "   'start_date': '1987',\n",
       "   'end_date': None,\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 7780',\n",
       "   'title': 'Questacon – The National Science and Technology Centre',\n",
       "   'date_str': '1985 - ',\n",
       "   'start_date': '1985',\n",
       "   'end_date': None,\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 7902',\n",
       "   'title': 'Department of Industry, Science and Technology, Central Office',\n",
       "   'date_str': '1994 - 1996',\n",
       "   'start_date': '1994',\n",
       "   'end_date': '1996',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 7923',\n",
       "   'title': 'Department of Industry, Science and Technology, State Office, New South Wales',\n",
       "   'date_str': '1994 - 1996',\n",
       "   'start_date': '1994',\n",
       "   'end_date': '1996',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 7924',\n",
       "   'title': 'Department of Industry, Science and Technology, State Office, Western Australia',\n",
       "   'date_str': '1994 - 1996',\n",
       "   'start_date': '1994',\n",
       "   'end_date': '1996',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 7925',\n",
       "   'title': 'Department of Industry, Science and Technology, State Office, Queensland',\n",
       "   'date_str': '1994 - 1996',\n",
       "   'start_date': '1994',\n",
       "   'end_date': '1996',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 7926',\n",
       "   'title': 'Department of Industry, Science and Technology, State Office, South Australia',\n",
       "   'date_str': '1994 - 1996',\n",
       "   'start_date': '1994',\n",
       "   'end_date': '1996',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 7927',\n",
       "   'title': 'Department of Industry, Science and Technology, State Office, Tasmania',\n",
       "   'date_str': '1994 - 1996',\n",
       "   'start_date': '1994',\n",
       "   'end_date': '1996',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 7930',\n",
       "   'title': 'Department of Industry, Science and Technology, State Office, Victoria',\n",
       "   'date_str': '1994 - 1996',\n",
       "   'start_date': '1994',\n",
       "   'end_date': '1996',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 8247',\n",
       "   'title': 'Department of Industry, Science and Tourism, Central Office',\n",
       "   'date_str': '1996 - 1998',\n",
       "   'start_date': '1996',\n",
       "   'end_date': '1998',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 8267',\n",
       "   'title': 'Department of Industry, Science and Tourism, State Office, Queensland',\n",
       "   'date_str': '1996 - 1998',\n",
       "   'start_date': '1996',\n",
       "   'end_date': '1998',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'},\n",
       "  {'identifier': 'CA 8268',\n",
       "   'title': 'Department of Industry, Science and Tourism, State Office, New South Wales',\n",
       "   'date_str': '1996 - 1998',\n",
       "   'start_date': '1996',\n",
       "   'end_date': '1998',\n",
       "   'retrieved': '2023-01-20T10:48:06.515238+11:00'}],\n",
       " 'retrieved': '2023-01-20T10:48:06.520278+11:00'}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agency_search.get_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'total_results': 1,\n",
       " 'page': 1,\n",
       " 'number_of_results': 1,\n",
       " 'results': [{'identifier': 'CA 9343',\n",
       "   'title': 'Australian Embassy, East Timor [Dili]',\n",
       "   'institution_title': None,\n",
       "   'agency_status': 'Head Office',\n",
       "   'location': 'Overseas',\n",
       "   'functions': [{'date_str': '25 Oct 1999 -',\n",
       "     'start_date': '1999-10-25',\n",
       "     'end_date': None,\n",
       "     'identifier': 'CONSULAR SERVICES',\n",
       "     'title': 'CONSULAR SERVICES'},\n",
       "    {'date_str': '25 Oct 1999 -',\n",
       "     'start_date': '1999-10-25',\n",
       "     'end_date': None,\n",
       "     'identifier': 'GOVERNMENT REPRESENTATION OVERSEAS',\n",
       "     'title': 'GOVERNMENT REPRESENTATION OVERSEAS'},\n",
       "    {'date_str': '25 Oct 1999 -',\n",
       "     'start_date': '1999-10-25',\n",
       "     'end_date': None,\n",
       "     'identifier': 'INTERNATIONAL RELATIONS',\n",
       "     'title': 'INTERNATIONAL RELATIONS'}],\n",
       "   'note': 'Summary heading Agency registration notes Abolition Creation Functions and activities An Embassy, headed by an Ambassador, is the highest form of diplomatic mission. An Australian Ambassador carries Letters of Credence from the Governor-General addressed to the Head of State of the country to which Australia is represented. Accreditation of an Ambassador comprises the formal presentation and acceptance of these credentials. An Australian Embassy’s broad function is to represent to the country to which it is accredited the interests of the Commonwealth of Australia. This involves a variety of tasks including - negotiating with governments and international organisations on issues of concern to Australia; - observing and reporting on events and developments in the host country and other countries or organisations for which it has representational, visiting, reporting or consular responsibilities; - carrying out various duties on behalf of other Australian government agencies; - promoting and facilitating trade relations with Australia; - disseminating information about Australia and its policies; and - assisting Australian citizens residing in or visiting the country to which the embassy is accredited. The Australian Embassy to East Timor was officially opened as an Embassy on 20 May 2002, following the granting of independence to East Timor from Indonesia on 20 May 2002. Prior to this, Australia was officially represented in East Timor by an Australian Mission from 25 October 1999, headed by James F Batley, (PSM 2000) until the opening of the embassy. The date of Presentation of the Credentials was 9 July 2002. The Australian Government had previously been represented in Timor by an Australian Consulate, Dili , which commenced on 1 January 1946 with the appointment of Mr Charles Eaton, OBE as Consul. The last Consul was Mr M F Berman, who closed the post on 31 August 1971 (see CA 2766 – Australian Consulate, Dili [Portuguese Timor]. Ambassadors 21 May 2002 James F Batley, PSM, Amb . 08 July 2002 G Paul Foley, Amb . 13 August 2004 Margaret E Twomey , Amb . 09 January 2008 Peter M Heyward, Amb Legislation administered Administrative structure Historical agency address 1999 - present Address: Avenida dos Martires da Patria, Dili PO Box 332 Ph : (670) 3322111 Fax: (670) 3322247 State/regional structure An Australian Embassy’s broad function is to represent to the country to which it is accredited the interests of the Commonwealth of Australia, as directed by the Department of Foreign Affairs and Trade Central Office. Records created by the agency The Australian Embassy in East Timor is scheduled to transition from primarily using a paper-based recordkeeping system, managed in TRIM Captura , to using an electronic document and records management system (TRIM Context) in 2011. Additional information End notes Sources Department of Foreign Affairs and Trade (2009). Statement of Service: appointments and biographies – 15 th edition. Commonwealth Government of Australia.',\n",
       "   'controlling_organisation': [{'date_str': '25 Oct 1999 -',\n",
       "     'start_date': '1999-10-25',\n",
       "     'end_date': None,\n",
       "     'identifier': 'CO 1',\n",
       "     'title': 'COMMONWEALTH OF AUSTRALIA'}],\n",
       "   'previous_agencies': [{'date_str': '',\n",
       "     'start_date': None,\n",
       "     'end_date': None,\n",
       "     'identifier': 'CA 3217',\n",
       "     'title': 'Official Representative, Portuguese Timor [Dili]'},\n",
       "    {'date_str': '',\n",
       "     'start_date': None,\n",
       "     'end_date': None,\n",
       "     'identifier': 'CA 2766',\n",
       "     'title': 'Australian Consulate, Dili [Portuguese Timor]'}],\n",
       "   'subsequent_agencies': [],\n",
       "   'superior_agencies': [{'date_str': '25 Oct 1999 -',\n",
       "     'start_date': '1999-10-25',\n",
       "     'end_date': None,\n",
       "     'identifier': 'CA 5987',\n",
       "     'title': 'Department of Foreign Affairs and Trade, Central Office'}],\n",
       "   'controlled_agencies': [],\n",
       "   'associated_people': [],\n",
       "   'retrieved': '2023-01-20T10:48:07.708766+11:00',\n",
       "   'date_str': '25 Oct 1999 -',\n",
       "   'start_date': '1999-10-25',\n",
       "   'end_date': None,\n",
       "   'number_of_series': 1}],\n",
       " 'retrieved': '2023-01-20T10:48:08.597801+11:00'}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This returns a single agency that created a single series\n",
    "# It should not cause any exceptions\n",
    "agency_search = RSAgencySearch(function='consular services')\n",
    "agency_search.get_results()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calling `.refresh_cache` will remove all of the data for this search from the cache, and set the results page back to 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "### RSSeriesSearch.refresh_cache\n",
       "\n",
       ">      RSSeriesSearch.refresh_cache ()\n",
       "\n",
       "Delete data for this search from the cache, then retrieve a fresh version from RecordSearch."
      ],
      "text/plain": [
       "---\n",
       "\n",
       "### RSSeriesSearch.refresh_cache\n",
       "\n",
       ">      RSSeriesSearch.refresh_cache ()\n",
       "\n",
       "Delete data for this search from the cache, then retrieve a fresh version from RecordSearch."
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(RSAgencySearch.refresh_cache, name=\"RSSeriesSearch.refresh_cache\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "import nbdev\n",
    "\n",
    "nbdev.nbdev_export()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}