tutorials.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SExpr(head=Tokenizer(name='term', value='+', lineno=0, colno=1), tail=[Tokenizer(name='term', value='1', lineno=0, colno=3), SExpr(head=Tokenizer(name='term', value='*', lineno=0, colno=6), tail=[Tokenizer(name='term', value='2', lineno=0, colno=8), Tokenizer(name='term', value='3', lineno=0, colno=10)])])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# write raw rbnf source code.\n",
    "from rbnf.easy import build_parser, Language, build_language\n",
    "from typing import NamedTuple, List\n",
    "\n",
    "my_lisp_definition = \"\"\"\n",
    "ignore [space]\n",
    "space := R'\\s'\n",
    "term  := R'[^\\(\\)\\s]'\n",
    "sexpr ::= '(' [sexpr as head sexpr* as tail] ')' | term as atom\n",
    "          rewrite\n",
    "            if atom:\n",
    "                return atom \n",
    "            if not head:\n",
    "                return ()\n",
    "            return SExpr(head, tail)\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "class SExpr(NamedTuple):\n",
    "    head: 'SExpr'\n",
    "    tail: List['SExpr']\n",
    "\n",
    "\n",
    "lisp = Language('lisp')\n",
    "\n",
    "lisp.namespace['SExpr'] = SExpr\n",
    "\n",
    "build_language(my_lisp_definition, lisp, '<tutorials>')\n",
    "\n",
    "parse = build_parser(lisp)\n",
    "\n",
    "parse(\"(+ 1 (* 2 3))\").result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SExpr(head=Tokenizer(name='term', value='+', lineno=0, colno=1), tail=[Tokenizer(name='term', value='1', lineno=0, colno=3), SExpr(head=Tokenizer(name='term', value='*', lineno=0, colno=6), tail=[Tokenizer(name='term', value='2', lineno=0, colno=8), Tokenizer(name='term', value='3', lineno=0, colno=10)])])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use Python to write rbnf\n",
    "from rbnf.easy import Parser, Lexer, Language, auto_context, C, build_parser\n",
    "from rbnf.core.State import State\n",
    "from rbnf.core.Tokenizer import Tokenizer\n",
    "from typing import NamedTuple, List, Optional\n",
    "\n",
    "lisp = Language('lisp')\n",
    "lisp.ignore('space')\n",
    "\n",
    "\n",
    "@lisp\n",
    "class space(Lexer):\n",
    "    @staticmethod\n",
    "    def regex():\n",
    "        return '\\s'\n",
    "\n",
    "\n",
    "@lisp\n",
    "class term(Lexer):\n",
    "    @staticmethod\n",
    "    def regex():\n",
    "        return '[^\\(\\)\\s]'\n",
    "\n",
    "\n",
    "@lisp\n",
    "class sexpr(Parser):\n",
    "    @staticmethod\n",
    "    def bnf():\n",
    "        return C('(') + (sexpr @ \"head\" + sexpr.unlimited @ \"tail\").optional + C(\")\") | term @ \"term\"\n",
    "    @staticmethod\n",
    "    @auto_context\n",
    "    def rewrite(state: State):\n",
    "        head: Optional[SExpr]\n",
    "        tail: Optional[List[SExpr]]\n",
    "        term: Optional[Tokenizer]\n",
    "        if term:\n",
    "            return term\n",
    "        return SExpr(head, tail)\n",
    "\n",
    "\n",
    "class SExpr(NamedTuple):\n",
    "    head: 'SExpr'\n",
    "    tail: List['SExpr']\n",
    "\n",
    "\n",
    "lisp.build()\n",
    "parse = build_parser(lisp, use_parser='sexpr')\n",
    "parse(\"(+ 1 (* 2 3))\").result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# About literal parsers\n",
    "from rbnf.core.Tokenizer import Tokenizer\n",
    "from rbnf.core.CachingPool import ConstStrPool\n",
    "from rbnf.core.State import State \n",
    "from rbnf.easy import N, NC, C, R, V \n",
    "\n",
    "const_value = ConstStrPool.cast_to_const('<value>')\n",
    "name = ConstStrPool.cast_to_const('<name>')\n",
    "\n",
    "tokens0 = [Tokenizer('<name>x'[:-1], const_value, lineno=0, colno=0)]\n",
    "tokens1 = [Tokenizer(name, '<value>x'[:-1], lineno=0, colno=0)]\n",
    "tokens2 = [Tokenizer(name, const_value, lineno=0, colno=0)]\n",
    "\n",
    "make_state = lambda : State({}, '<playground>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parse tokens0:  \u001b[31mUnmatched\u001b[39m\nparse tokens1: \u001b[32mTokenizer(name='<name>', value='<value>', lineno=0, colno=0)\u001b[39m\nparse tokens2: \u001b[32mTokenizer(name='<name>', value='<value>', lineno=0, colno=0)\u001b[39m\n"
     ]
    }
   ],
   "source": [
    "name_parser = N('<name>')\n",
    "\n",
    "print('parse tokens0: ', name_parser.match(tokens0, make_state()))\n",
    "print('parse tokens1:', name_parser.match(tokens1, make_state()))\n",
    "print('parse tokens2:', name_parser.match(tokens2, make_state()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parse tokens0:  \u001b[32mTokenizer(name='<name>', value='<value>', lineno=0, colno=0)\u001b[39m\nparse tokens1: \u001b[32mTokenizer(name='<name>', value='<value>', lineno=0, colno=0)\u001b[39m\nparse tokens2: \u001b[32mTokenizer(name='<name>', value='<value>', lineno=0, colno=0)\u001b[39m\n"
     ]
    }
   ],
   "source": [
    "value_parser = V('<value>')\n",
    "\n",
    "print('parse tokens0: ', value_parser.match(tokens0, make_state()))\n",
    "print('parse tokens1:', value_parser.match(tokens1, make_state()))\n",
    "print('parse tokens2:', value_parser.match(tokens2, make_state()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parse tokens0:  \u001b[32mTokenizer(name='<name>', value='<value>', lineno=0, colno=0)\u001b[39m\nparse tokens1: \u001b[31mUnmatched\u001b[39m\nparse tokens2: \u001b[32mTokenizer(name='<name>', value='<value>', lineno=0, colno=0)\u001b[39m\n"
     ]
    }
   ],
   "source": [
    "value_parser = C('<value>')\n",
    "\n",
    "print('parse tokens0: ', value_parser.match(tokens0, make_state()))\n",
    "print('parse tokens1:', value_parser.match(tokens1, make_state()))\n",
    "print('parse tokens2:', value_parser.match(tokens2, make_state()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parse tokens0:  \u001b[32mTokenizer(name='<name>', value='<value>', lineno=0, colno=0)\u001b[39m\nparse tokens1: \u001b[32mTokenizer(name='<name>', value='<value>', lineno=0, colno=0)\u001b[39m\nparse tokens2: \u001b[32mTokenizer(name='<name>', value='<value>', lineno=0, colno=0)\u001b[39m\n"
     ]
    }
   ],
   "source": [
    "regex_parser = R(\"\\S+\")  \n",
    "# Not recommend to use regex parser when parsing. \n",
    "# Lexers use regex to tokenize raw input into tokenizers \n",
    "#   with constant names which could be compared by memory address\n",
    "print('parse tokens0: ', regex_parser.match(tokens0, make_state()))\n",
    "print('parse tokens1:', regex_parser.match(tokens1, make_state()))\n",
    "print('parse tokens2:', regex_parser.match(tokens2, make_state()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parse tokens0:  \u001b[31mUnmatched\u001b[39m\nparse tokens1: \u001b[31mUnmatched\u001b[39m\nparse tokens2: \u001b[32mTokenizer(name='<name>', value='<value>', lineno=0, colno=0)\u001b[39m\n"
     ]
    }
   ],
   "source": [
    "name_and_value = NC(name, const_value)\n",
    "\n",
    "print('parse tokens0: ', name_and_value.match(tokens0, make_state()))\n",
    "print('parse tokens1:', name_and_value.match(tokens1, make_state()))\n",
    "print('parse tokens2:', name_and_value.match(tokens2, make_state()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TO BE CONTINUE"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}