% Copyright 2012-2014, Alexander Shibakov
% This file is part of SPLinT
%
% SPLinT is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% SPLinT is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with SPLinT.  If not, see <http://www.gnu.org/licenses/>.

% `data structure access' macros': picking the n-th undelimited parameter
% in a parameter list inside a token register
% it is assumed that none of the arguments is \end, and that there are enough
% parameters to pick the desired one

\def\getfirst#1\to#2{%
    #2\expandafter\expandafter\expandafter{\expandafter\g@tfirst\the#1\end}%
}

\def\g@tfirst#1#2\end{#1}

\def\getsecond#1\to#2{%
    #2\expandafter\expandafter\expandafter{\expandafter\g@tsecond\the#1\end}%
}

\def\g@tsecond#1#2#3\end{#2}

\def\getthird#1\to#2{%
    #2\expandafter\expandafter\expandafter{\expandafter\g@tthird\the#1\end}%
}

\def\g@tthird#1#2#3#4\end{#3}

\def\getfourth#1\to#2{%
    #2\expandafter\expandafter\expandafter{\expandafter\g@tfourth\the#1\end}%
}

\def\g@tfourth#1#2#3#4#5\end{#4}

\def\getfifth#1\to#2{%
    #2\expandafter\expandafter\expandafter{\expandafter\g@tfifth\the#1\end}%
}

\def\g@tfifth#1#2#3#4#5#6\end{#5}

\def\getsixth#1\to#2{%
    #2\expandafter\expandafter\expandafter{\expandafter\g@tsixth\the#1\end}%
}

\def\g@tsixth#1#2#3#4#5#6#7\end{#6}

\def\getseventh#1\to#2{%
    #2\expandafter\expandafter\expandafter{\expandafter\g@tseventh\the#1\end}%
}

\def\g@tseventh#1#2#3#4#5#6#7#8\end{#7}

\def\geteightth#1\to#2{%
    #2\expandafter\expandafter\expandafter{\expandafter\g@teightth\the#1\end}%
}

\def\g@teightth#1#2#3#4#5#6#7#8#9\end{#8}

\def\getninth#1\to#2{% the `.' is necessary since \TeX's scanning mechanism will
                     % strip any potential braces surrounding the last parameter
                     % (assuming there are exactly nine) twice:
                     %  o first, expanding \g@tninth,
                     %  o second, expanding \g@tfirst
    #2\expandafter\expandafter\expandafter{\expandafter\g@tninth\the#1.\end}%
    #2\expandafter\expandafter\expandafter{\the#2}%
}

\def\g@tninth#1#2#3#4#5#6#7#8#9\end{\g@tfirst#9\end}

\def\gettenth#1\to#2{% no need for `.' (or any other placeholder) here,
                     % since the #9-th parameter to \g@ttenth is a list of
                     % at least two parameters itself, thus any existing braces
                     % have survived intact
    #2\expandafter\expandafter\expandafter{\expandafter\g@ttenth\the#1\end}%
    #2\expandafter\expandafter\expandafter{\the#2}%
}

\def\g@ttenth#1#2#3#4#5#6#7#8#9\end{\g@tsecond#9\end}

% string replacement: all arguments are registers, nothing is expanded, no \next is defined
% note that this is not a greedy replacement: this could be arranged with a more sophisticated macro
% also note that the string being replaced cannot have any braces in it

\newif\ifyytracereplacements
\newif\ifyyreplaced

\yytracereplacementstrue

\def\yyreplacestring#1\in#2\with#3{%
      \expandafter\def\expandafter\r@placestring\expandafter##\expandafter1\the#1##2\end{%
          \def\r@placestring{##2}% is this the string at the very end?
          \ifx\r@placestring\empty % then it is the one we inserted,
                                   % report
              \yyreplacedfalse
              \ifyytracereplacements
                  \errmessage{string <\the#1> not present in \the#2}% do not change the register if the string is not there
              \fi
          \else % remove the extra copy of #1\end at the end
              \yyreplacedtrue
              \expandafter#2\expandafter\expandafter\expandafter
                  {\expandafter\r@plac@string\expandafter{\the#3}{##1}##2\end}%
      \fi}% end of \r@placestring definition
      \expandafter\def\expandafter\r@plac@string
          \expandafter##\expandafter1%
          \expandafter##\expandafter2%
          \expandafter##\expandafter3%
          \the#1\end{##2##1##3}%
      \expandafter\expandafter\expandafter\r@placestring\expandafter\the\expandafter#2\the#1\end
}

% creating a sequence containing all pairs from the two given sequences;
% the (long) string produced by the \diagprod macro
% lists all the elements so that each ordered pair ab where a and b are
% different elements from each of the two sets appears exactly once;
% a simple strategy for creating such strings is to build them recursively so that if
% S is a string that lists n values in this manner and s is a new item type,
% start with sSs and then add each symbol from S on either side 
% in such a way that among any two consecutive symbols exactly one is s

% these macros are used to create switch statements that use such ordered pairs
% and are not particularly general or robust; 
% they are supposed to be used once in the setup stage;
% the assumption made by these mactos is that `.' or `;' never appear as elements 
% of the two sets

\def\gnxtelem#1\to#2\and#3{%
    \expandafter\gnxtel@m\expandafter#2\expandafter#3\the#1.;%
}

\def\gnxtel@m#1#2#3#4;{%
    \def\next{#4}%
    \ifx\next\empty
        #1{}%
    \else
        #1{#3}%
        \gnxt@l@m#2#4%
    \fi
}

\def\gnxt@l@m#1#2.{%
    #1{#2}%
}

\def\pairup{%
    \gnxtelem\toksa\to\toksc\and\toksa % get the first remaining element of set A (\toksa)
    \edef\elemofA{\the\toksc}%
    \ifx\elemofA\empty % no more elements in A
         \let\next\relax
    \else
        \ifx\elemofA\lastelemofB % the current element of A is the same as the last element of B
            \edef\next{\the\toksa}%
            \ifx\next\empty % it is the last remaining element of A
                \expandafter\p@ir@p\the\toksb.% form all pair of \the\toksc with elements in B except the first one
            \else
                \concat\toksa\toksc % move it to the end of A
            \fi
            \let\next\pairup
        \else
            \expandafter\pair@p\the\toksb.% form all pairs of \the\toksc with elements in B
            \let\next\pairup
        \fi
    \fi
    \next
}

\def\dotcontainer{.}

\def\pair@p#1{%
    \def\next{#1}%
    \ifx\next\elemofA % the next element of B is the current element of A
        \ifx\next\lastelemofB % it is the last element of B
                              % we can arrive here only if 1) B has more than one element and 
                              % 2) \elemofA is the last remaining element of A
            \edef\next{\toksd{\the\toksd\the\tokse}}\next
            \let\next\eatone % eat the remaining dot     
        \else
            \removeelem\elemofA\from\toksb % remove it from B
            \let\next\pair@p
        \fi
    \else
        \ifx\next\dotcontainer % no more elements in B
            \let\next\relax
        \else
            \toksf{#1}%
            \edef\next{\toksd{\the\toksd\the\toksc\the\toksf}}\next
            %\showthe\toksd
            \let\next\pair@p
        \fi
    \fi
    \next
}

\def\p@ir@p#1{%
    \def\next{#1}%
    \ifx\next\elemofA % the next element of B is the current element of A
                      % this can only happen if B consists of a single element
        \let\next\eatone
    \else
        \toksf{#1}%
        \edef\next{\toksd{\the\toksd\the\toksf}}\next
        \let\next\pair@p
    \fi
    \next
}

\def\removeelem#1\from#2{% #1 should be a sequence containing the element
                         % #2 is the token register
    \expandafter\def\expandafter\r@moveelem\expandafter##\expandafter1#1##2.{%
        \def\next{##2}%
        \ifx\next\empty % there was no such element
            \errmessage{Could not find \expandafter\string#1 in \the#2}%
        \else
            \expandafter\def\expandafter\r@mov@elem\expandafter####\expandafter1#1{%
                #2{##1####1}%
            }\r@mov@elem##2%
       \fi
    }%
    \expandafter\expandafter\expandafter\r@moveelem\expandafter\the\expandafter#2#1.%
}

% the intersection of A and B should be at the end of A:

\def\pushintersect{%
    \gnxtelem\toksf\to\toksc\and\toksf % get the first remaining element of set A (\toksa)
    \edef\next{\the\toksc}%
    \ifx\next\empty % there are no elements left in A
        \let\next\relax
    \else
        \expandafter\def\expandafter\p@shintersect\expandafter##\expandafter1\the\toksc##2.{%
            \def\next{##2}%
            \ifx\next\empty % there was no such element in the other set
                \edef\next{\toksa{\the\toksc\the\toksa}}\next % append it to the front
            \else
                \edef\next{\toksa{\the\toksa\the\toksc}}\next % append it to the back
            \fi
        }%
        \expandafter\expandafter\expandafter\p@shintersect\expandafter\the\expandafter\toksb\the\toksc.%
        \let\next\pushintersect
    \fi
    \next
}

\def\diagprod#1#2\in#3{%
    \toksa\expandafter{#1}%
    \toksb\expandafter{#2}%
    \gnxtelem\toksb\to\tokse\and\toksb % \tokse is a selected element in set B (\toksb)
    \concat\toksb\tokse % now \tokse is the last element of B 
    \toksf\toksa\toksa{}%
    \pushintersect % prepare the sequence representing set A so that the intersection elements are in the tail
    \edef\lastelemofB{\the\tokse}% the last element of B
    \toksd\expandafter{\the\tokse}% future sequence pairs
    \pairup
    \edef#3{\the\toksd}%
}

% namespace management for macros
% note that if one of the sequences were made \let\name. the macros would break;
% this can be fixed by using a more sophisticated comparison but was decided against
% in the interest of efficiency; the old version (that used a control sequence
% as a `stop marker') was more prone to this bug.

\def\savecs#1#2{%
    \def\next##1{%
         \ifx##1.%
             \let\next\relax
         \else
             \expandafter\let\csname '#1'[\expandafter\eatone\string##1]\endcsname##1%
         \fi
         \next
    }%
    \next#2.%
}

\def\restorecs#1#2{%
    \def\next##1{%
         \ifx##1.%
             \let\next\relax
         \else
             \expandafter\let\expandafter##1\csname '#1'[\expandafter\eatone\string##1]\endcsname
         \fi
         \next
    }%
    \next#2.%
}

\def\hidecs#1{\h@decs#1.}

\def\h@decs#1{%
  \ifx#1.%
      \let\next\relax
  \else
      \let#1\relax
      \let\next\h@decs
  \fi
  \next
}

\def\savehcs#1#2{\savecs{#1}{#2}\hidecs{#2}}

\def\savecslist#1#2{%
    \expandafter\s@vecslist\expandafter{#2}{#1}%
}

\def\s@vecslist#1#2{%
    \savecs{#2}{#1}%
}

\def\restorecslist#1#2{%
    \expandafter\r@storecslist\expandafter{#2}{#1}%
}

\def\r@storecslist#1#2{%
    \restorecs{#2}{#1}%
}

% a twist on the macros above: save control sequences with a postfix

\def\savecsx#1#2{\toksa{#1}\s@vecsx#2.}

\def\s@vecsx#1{%
  \ifx#1.%
      \let\next\relax
  \else
      \expandafter\s@vecsxlet\expandafter#1\csname\expandafter\defprefix\expandafter\eatone\string#1\defpostfix\endcsname
      \let\next\s@vecsx
  \fi
  \next
}

\def\s@vecsxlet#1{%
  \expandafter\let\csname '\the\toksa'[\expandafter\defprefix\expandafter\eatone\string#1\defpostfix]\endcsname
}

\def\restorecsx#1#2{\toksa{#1}\r@storecsx#2.}

\def\r@storecsx#1{%
  \ifx#1.%
      \let\next\relax
  \else
      \expandafter\r@storecsxlet\expandafter#1\csname '\the\toksa'[\expandafter\defprefix\expandafter\eatone\string#1\defpostfix]\endcsname
      \let\next\r@storecsx
  \fi
  \next
}

\def\r@storecsxlet#1{%
  \expandafter\let\csname\expandafter\defprefix\expandafter\eatone\string#1\defpostfix\endcsname
}

\def\hidecsx#1{\h@decsx#1.}

\def\h@decsx#1{%
  \ifx#1.%
      \let\next\relax
  \else
      \expandafter\let\csname\expandafter\defprefix\expandafter\eatone\string#1\defpostfix\endcsname\relax
      \let\next\h@decsx
  \fi
  \next
}

\def\savehcsx#1#2{\savecsx{#1}{#2}\hidecsx{#2}}

\def\defx#1{% defining sequences as above
  \toksa\expandafter{%
  \csname\expandafter\defprefix\expandafter\eatone\string#1\defpostfix\endcsname}%
  \afterassignment\d@fx
  \expandafter\def\the\toksa
}

\def\d@fx#1{%
  \toksb{#1}\edef\next{\noexpand\savehcs{\the\toksb}\the\toksa}\next
}

\def\defy#1#2#{% in addition to the definition of the contorl sequence
               % in the appropriate namespace, this macro adds a
               % preamble, a postamble and a `this' type macro; this
               % will mostly be used with indexing \TeX\ control sequences
  \tokse{\def\thisname{#1}\edef\thisnamex{\expandafter\eatone\string#1}}%
  \toksa\expandafter{%
      \csname\expandafter\defprefix\expandafter\eatone\string#1\defpostfix\endcsname}%
  \toksc\expandafter{\expandafter\def\the\toksa#2}%
  \afterassignment\d@f@
  \toksd=%
}

\def\d@f@{%
   \appendl\toksd{\the\tokse}%
   \tokse\expandafter{\defypreamble}%
   \appendl\toksd{\the\tokse}%
   \tokse\expandafter{\defypostamble}%
   \appendr\toksd{\the\tokse}
   \toksd\expandafter{\expandafter{\the\toksd}}%
   \concat\toksc\toksd
   \afterassignment\d@fy
   \the\toksc
}

\def\d@fy#1{%
  \toksb{#1}\edef\next{\noexpand\savehcs{\the\toksb}\the\toksa}\next
}

{\catcode`\ =13 \aftergroup\def\aftergroup\activespace\aftergroup{\aftergroup \aftergroup}}% active space
{\catcode`\% =12 \aftergroup\def\aftergroup\harmlesscomment\aftergroup{\aftergroup%\aftergroup}}% not really a comment
{\catcode`\{ =12 \catcode`\[=1 \aftergroup\def\aftergroup\lbchar\aftergroup[\aftergroup{\aftergroup}}% not really a brace
{\catcode`\} =12 \catcode`\]=2 \aftergroup\def\aftergroup\rbchar\aftergroup{\aftergroup}\aftergroup]]% not really a brace
{\catcode`\_=12 \aftergroup\def\aftergroup\uscoreletter\aftergroup{\aftergroup_\aftergroup}}
{\catcode`\^^M=12 \aftergroup\def\aftergroup\eolletter\aftergroup{\aftergroup^^M\aftergroup}}% end of line, ... not really
{\catcode`\|=0\catcode`\\=12 |aftergroup|def|aftergroup|benignescape|aftergroup{|aftergroup\|aftergroup}}% not an escape
{\catcode`\#=12 \aftergroup\def\aftergroup\hashletter\aftergroup{\aftergroup#\aftergroup}}% not a parameter token

% token name input

\def\doascii#1{%
  \tempca=\z@ \tempcb=\@cclvi
  \let\yyasciicc\empty
  \let\sts\relax
  \loop
      \edef\yyasciicc{\sts{\the\catcode\tempca}\yyasciicc}%
      \catcode\tempca=#1
      \advance\tempca\@ne
  \ifnum\tempca<\tempcb\relax
  \repeat
  \catcode`\ =10 % keep normal spaces (otherwise \string will convert the category to 10)
  %\catcode`\^^M=5 % if newline is to be preserved
  \catcode`\^^M=12 % to match \eolletter
}

\def\undoascii{%
  \tempca=\@cclv
  \let\sts\undoacatcode
  \yyasciicc\def\yyasciicc{}%
}

\def\undoacatcode#1{\catcode\tempca=#1\advance\tempca\m@ne}

% yytname macros

\def\endcontainer{\end}%
\def\aaddname{\addname}

\def\addname{%
    \toksb{}%
    \@ddname
}

\def\@ddname#1{%
    \def\next{#1}%
    \ifx\next\aaddname
        \concat\yytname\toksb
        \expandafter\edef\csname term\parsernamespace\the\toksb \endcsname{\the\tempca}%
        \advance\tempca1\relax
        \appendortoyytname
        \let\next\addname
    \else
        \ifx\next\endcontainer
            \concat\yytname\toksb
            \expandafter\edef\csname term\parsernamespace\the\toksb \endcsname{\the\tempca}%
            \let\next\relax
        \else
            \ifnum#1=`\_\relax
                \uccode`\.=#1\relax
                \uppercase{\toksa{.}}%
                \uccode`\.=`\.%
            \else
                \ifnum#1=`\-\relax
                    \toksa{-}%
                \else
                    \ifnum#1=`\ \relax
                        \toksa{ }%
                    \else
                        \uccode`\@=#1\relax
                        \uppercase{\toksa{@}}%
                        \uccode`\@=`\@\relax
                    \fi
                \fi
            \fi
            \concat\toksb\toksa
            \let\next\@ddname
        \fi
    \fi
    \next
}

\def\appendortoyytname{%
    \toksa{\or}%
    \concat\yytname\toksa
}

\def\print#1{{\endlinechar=`\^^J\immediate\write16{#1}}}

% token and state setup
% these are defined for a specific (though dynamic) namespace only

\def\tokenset#1#2{%
    \expandafter\edef\csname token\parsernamespace \fgetelemof{yytname}\at{#1}\endcsname{#2}%
}

\def\settokens{%
    \tempca=1\relax
    \loop
        \tempcb=\fgetelemof{yytranslate}\at\tempca\relax % important \relax!
        \tokenset\tempcb{\the\tempca}%
        \advance\tempca\@ne
    \ifnum\YYTRANSLATESIZE>\tempca
    \repeat
    \relax
}

\def\stateset#1#2{%
    \expandafter\def\csname flexstate\parsernamespace #1\endcsname{#2}%
}

% token equivalence for bootstrap

\def\tokeneq#1#2{%
    \toksa{}%
    \numberstochars#2\end
    \toksa\expandafter{\csname token\parsernamespace\the\toksa\endcsname}%
    \toksb\expandafter{\csname token\parsernamespace#1\endcsname}%
    \edef\next{\let\the\toksb\the\toksa}\next
}%

\def\numberstochars#1{%
    \def\next{#1}%
    \ifx\next\endcontainer
        \let\next\relax
    \else
        \uccode`.=#1\relax
        \uppercase{\edef\next{\toksa{\the\toksa.}}}\next
        \uccode`.=`\.%
        \let\next\numberstochars
    \fi
    \next
}

% other parser and lexer variables

\newtoks\pinittoks % a token register to hold the code that switches the parser to a different namespace

% tables

\def\newtable@full#1{%
    \toksa{\csname newtoks\endcsname}%
    \expandafter\the\expandafter\toksa\csname #1\parsernamespace\endcsname
    \edef\next{\let\csname #1\endcsname\csname #1\parsernamespace\endcsname}\next
    \toksa\expandafter{\next}\concat\pinittoks\toksa
    \csname #1\parsernamespace\endcsname=%
}

\let\newtable\newtable@full % optimization can change this

% constants

\def\constset#1#2{%
    % a \mathchardef would be nicer but it cannot handle negative numbers
    \expandafter\def\csname #1\parsernamespace\endcsname{#2}%
    \toksa\expandafter{\csname #1\endcsname}%
    \toksb\expandafter{\csname #1\parsernamespace\endcsname}%
    \edef\next{\let\the\toksa\the\toksb}\next
    \toksa\expandafter{\next}\concat\pinittoks\toksa
}

\def\uconstset#1#2{%
    % a \mathchardef for positive constants
    \expandafter\mathchardef\csname #1\parsernamespace\endcsname=#2 %
    \toksa\expandafter{\csname #1\endcsname}%
    \toksb\expandafter{\csname #1\parsernamespace\endcsname}%
    \edef\next{\let\the\toksa\the\toksb}\next
    \toksa\expandafter{\next}\concat\pinittoks\toksa
}

\def\charset#1#2{%
    \expandafter\chardef\csname #1\parsernamespace\endcsname=#2\relax%
    \toksa\expandafter{\csname #1\endcsname}%
    \toksb\expandafter{\csname #1\parsernamespace\endcsname}%
    \edef\next{\let\the\toksa\the\toksb}\next
    \toksa\expandafter{\next}\concat\pinittoks\toksa
}

% switch macro

\def\stashswitch#1{%
    \toksa\expandafter{\csname #1\endcsname}%
    \toksb\expandafter{\csname #1\parsernamespace\endcsname}%
    \edef\next{\let\the\toksb\the\toksa}\next
    \edef\next{\let\the\toksa\the\toksb}%
    \toksa\expandafter{\next}\concat\pinittoks\toksa
}

% parser and lexer state control

\def\settokreg#1{%
    \toksa{\csname newtoks\endcsname}%
    \expandafter\the\expandafter\toksa\csname #1\parsernamespace\endcsname
    \edef\next{\let\csname #1\endcsname\csname #1\parsernamespace\endcsname}\next
    \toksa\expandafter{\next}\concat\pinittoks\toksa
}

\def\setcntreg#1{%
    \toksa{\csname newcount\endcsname}%
    \expandafter\the\expandafter\toksa\csname #1\parsernamespace\endcsname
    \edef\next{\let\csname #1\endcsname\csname #1\parsernamespace\endcsname}\next
    \toksa\expandafter{\next}\concat\pinittoks\toksa
}

\def\setnulstack#1{%
    \toksa\expandafter{\csname #1\endcsname}%
    \toksb\expandafter{\csname #1\parsernamespace\endcsname}%
    \edef\next{\let\the\toksb\noexpand\empty}\next
    \edef\next{\let\the\toksa\the\toksb}\next
    \toksa\expandafter{\next}\concat\pinittoks\toksa
    \setcntreg{#1<count>}% this is only needed for the accelerated stack
}

\def\setcurrentcs#1{%
    \toksa\expandafter{\csname #1\endcsname}%
    \toksb\expandafter{\csname #1\parsernamespace\endcsname}%
    \edef\next{\let\the\toksb\the\toksa}\next
    \edef\next{\toksa{\let\the\toksa\the\toksb}}\next
    \concat\pinittoks\toksa
}

\def\newparserstate{%
  \setcntreg{yytoken}%
  \setcntreg{yystate}%
  \setcntreg{yyn}%
  \setcntreg{yylen}%
  %
  \setcntreg{yyilen}% depth of stack before an inline action
  % \yyval and \yylval will be token registers
  \settokreg{yyval}%
  \settokreg{yylval}%
  \setnulstack{yyssa}%
  \setnulstack{yyvsa}%
}

\def\newlexerstate{%
  \settokreg{yytext}%
  \settokreg{yytextseen}%
  \settokreg{yybyte}%
  \settokreg{yyfutureyytext}%    token register used to save read text in eob macros
  %
  \settokreg{yytextpure}%      % the registers that serve the same role
  \settokreg{yytextseenpure}%  % as \yytext et al except they collect characters
  \settokreg{yybytepure}%      % with category code 11 (letter) and the same character code
  %
  \settokreg{yyformat}%        % the registers that serve the same role
  \settokreg{yyformatseen}%    % as \yytext et al except they collect format
  \settokreg{yyfbyte}%         % commands
  %
  \settokreg{yystash}%         % the registers that serve the same role
  \settokreg{yystashseen}%     % as \yytext et al except they collect stashed
  \settokreg{yysbyte}%         % tokens
  %
  \setcntreg{yycurrentstate}%
  \setcntreg{yycurrentstatelocal}% a state variable used in eob macros
  \setcntreg{yyc}%
  \setcntreg{yyclocal}% another eob macro variable
  \setcntreg{yyact}%
  \setcntreg{yyg@yyinit}%
  \setcntreg{yyg@yystart}%
  \setcntreg{yyg@yylastacceptingstate}%
  \setcntreg{yycp@}%
  \setcntreg{YYATBOL}%
  \setcntreg{yychar}%
  %
  \setcntreg{yytextlastchar}%
  \setcntreg{yytextseenlastchar}%
  %
  \setcntreg{yyfmark}%        % the last marker in the current token
  \setcntreg{yyfmarklast}%
  \setcntreg{yysmark}%
  \setcntreg{yysmarklast}%
  \setnulstack{yystatestack}%
  \setcurrentcs{setflexstates}%
  \setcurrentcs{ifyytextbackup}% this is purely to record the recovery command in \pinittoks
}

\def\savestatelist#1{% this elaborate definition is needed to ensure that `throwaway token registers' like \toksa
                     % survive the namespace switch
    \edef\next{\expandafter\expandafter\expandafter
                   \let\expandafter\expandafter\expandafter\noexpand\expandafter\expandafter\csname #1\parsernamespace\endcsname
                   \expandafter\noexpand\csname #1\endcsname}\next
}

\def\savefullstate{%
  \savestatelist{yyssa}%
  \savestatelist{yyvsa}%
  \savestatelist{yystatestack}%
  \savestatelist{ifyytextbackup}% buffer
}

% use \pinittoks to compose a `parser restore' macro along with 
% \let\yyvsa\yyvsa[parser namespace] and \let\yyssa\yyssa[parser namespace];
% `parser save' macro only has to set up \yy?sa[parser namespace]'s;
% if a fully reentrant parser is required, use the macros above to save the contents of
% all the variables by redefining \settokreg and \setcntreg and saving \yy?sa stacks 

% switch and dfa macros

%% character class checking: #1 is the character, #2 is the character class control sequence

\newif\ifinclass

\def\ifclassof#1\is#2{%
  \tempca#1\relax
  \expandafter\checkclass#2\end
}

\def\checkclass#1{%
  \ifx#1\end
      \let\next\relax
      \inclassfalse
  \else
      \ifnum`#1=\tempca
          \let\next\classend
          \inclasstrue
      \else
          \let\next\checkclass
      \fi
  \fi
  \next
}

\tempca=\catcode`\^^A
\catcode`\^^A=12 % so that the characters inside sequences get the appropriate catcodes

\def\classend#1\end{}

\def\setspecialchars#1{%
  \toksa{}%
  \tempca=`\ %
  \loop
      \advance\tempca\@ne
      \tempcb=\fgetelemof{yytranslate}\at\tempca\relax
      \ifnum\tempcb>\tw@
          \tempcb=\uccode`^^A\uccode`^^A=\tempca
          \uppercase{\edef\next{\toksa{\the\toksa^^A}}}\next
          \uccode`^^A=\tempcb
      \fi
  \ifnum\tempca<\@cclv
  \repeat
  \edef#1{\the\toksa}%
}

\newif\iftraceswitchlabels
\traceswitchlabelstrue

\def\setspecialcharsfrom#1{%
  \toksb\expandafter{#1}%
  \iftraceswitchlabels
      \edef\next{\toksa{\expandafter\noexpand\csname <state: \expandafter\eatone\string#1>\endcsname}}\next
      \expandafter\let\the\toksa\empty
  \else
      \toksa{}%
  \fi
  \tempcb=\uccode`\^^A\relax
  \expandafter\s@tspecialcharsfrom\the\toksb\end
  \edef#1{\the\toksa}%
  \uccode`\^^A=\tempcb
}

\def\s@tspecialcharsfrom{%
  \futurelet\next\s@tsp@cialcharsfrom
}%

\def\s@tsp@cialcharsfrom{%
  \ifcat\space\noexpand\next
      \expandafter
      \s@tsp@cialch@rsfr@m
  \else
      \expandafter
      \s@tsp@cialch@rsfrom
  \fi
}

\def\s@tsp@cialch@rsfr@m{%
  \afterassignment\s@tspecialcharsfrom\let\next= % this is the optional space
}

\def\s@tsp@cialch@rsfrom#1{%
  \toksb{#1}%
  \ifx\next\bgroup % this is an action group
      \edef\next{\toksa{\the\toksa{\the\toksb}}}\next
      \let\next\s@tspecialcharsfrom
  \else
      \let\default\specchardefault
      \iftracedfa % to avoid conflicts while tracing actions
          \tracedfafalse
          \switchon{\the\toksb}\in\speccharswitch
          \tracedfatrue
      \else
          \switchon{\the\toksb}\in\speccharswitch
      \fi           
  \fi
  \next
}%

\def\speccharswitch{%
    \end {%
        \let\next\relax
    }
    \raw \rawcode \classexpand \meanit \statecomment {%
        %
    }
}

\def\specchardefault{%
    \uccode`\^^A\expandafter`\the\toksb
    \uppercase{\edef\next{\toksa{\the\toksa^^A}}}\next
    \let\next\s@tspecialcharsfrom
}

\def\raw#1\raw{%
  \toksb{#1}%
  \concat\toksa\toksb
  \s@tspecialcharsfrom
}

\def\rawcode#1{%
  \ifx#1\rawcode
      \let\next\s@tspecialcharsfrom
  \else
      \uccode`\^^A=#1\relax
      \uppercase{\edef\next{\toksa{\the\toksa^^A}}}\next
      \let\next\rawcode
  \fi
  \next
}

\def\classexpand#1{%
  \toksb\expandafter{#1}%
  \concat\toksa\toksb
  \s@tspecialcharsfrom
}

\def\meanit#1{%
  \toksb\expandafter{\meaning#1}%
  \concat\toksa\toksb
  \s@tspecialcharsfrom
}

\def\statecomment#1\statecomment{\edef\next{\toksa{\the\toksa
    \expandafter\noexpand\csname (#1)\endcsname}}\next\s@tspecialcharsfrom}

\def\putother#1\in#2{%
  \tempca=\catcode`\^^A\relax
  \tempcb=\uccode`\^^A\relax
  \uccode`\^^A=#1\relax
  \uppercase{\edef\next{#2{^^A}}}\next
  \uccode`\^^A=\tempcb         
}

% listing all the rules in a macro

\def\listrules{%
  \tempcb=\tw@ % start with a user-set rule
  \loop
      \listrule
  \ifnum\tempcb<\YYNRULES\relax
      \advance\tempcb\@ne
  \repeat
}

\catcode`\$=11

\def\listrule{%
  \tempcc=\fgetelemof{yyrone}\at\tempcb\relax % get the symbol this rule derives
  \edef\next{\toksa{\fgetelemof{yytname}\at\tempcc}}\next
  \expandafter\checkforimplicit\the\toksa $@\end%$
  \ifimplicit
      \edef\next{\toksc{\noexpand\ih{\the\toksa}}}\next
      \edef\next{\toksa{\noexpand\implicitrule[\the\toksa]}}\next
  \else
      \edef\next{\toksa{\the\tempcc}}\next
      \toksc{}%
  \fi
  \appendr\toksa{:}%
  \toksb{}%
  \tempcc=\fgetelemof{yyprhs}\at\tempcb\relax
  \tempcd=\fgetelemof{yyrhs}\at\tempcc\relax
  \listr@le
  \edef\next{\newsymswitch{\the\newsymswitch\the\toksa\def{\the\toksc:\the\tempcb}\noexpand\ruleor}}\next
}

\def\implicitsymbol#1{}%

\newif\ifimplicit

\def\listr@le{%
  \ifnum\tempcd>\m@ne
      \ifnum\tempcd<\YYNTOKENS\relax % it is a terminal
          \edef\next{\toksb{\the\tempcd}}\next
      \else
          \edef\next{\toksb{\fgetelemof{yytname}\at\tempcd}}\next
          \expandafter\checkforimplicit\the\toksb $@\end%$
          \ifimplicit
              \appendl\toksc{\noexpand\imn{\the\toksb}}
              \toksb{@implicit@}%
          \else
              \edef\next{\toksb{\the\tempcd}}\next
          \fi
      \fi
      \appendr\toksa{ }\concat\toksa\toksb
      \advance\tempcc\@ne
      \tempcd=\fgetelemof{yyrhs}\at\tempcc\relax
      \let\next\listr@le
  \else % this is the end of the rhs
      \let\next\relax
  \fi
  \next
}

\def\checkforimplicit#1$@#2\end{% $
  \def\next{#1}%
  \ifx\next\empty
      \implicittrue
  \else
      \implicitfalse
  \fi
}
\catcode`\$=3

% symbol switch macros

\newtoks\oneimplicitrule

% inline explicit symbolic names

\def\itermstack#1#2{%
  \toksa=\oneimplicitrule
  \appendl\toksa{\noexpand\lhs{#1}{#2}}%
  \expandafter
    \yypush
  \expandafter
    {\the\toksa}\on\yyirulestack
  \appendr\oneimplicitrule{\noexpand\implicitterm{#1}{#2}}%
}

\def\makeisymnames#1:#2\end{%
  \tempcc=#2\relax
  \def\next{#1}%
  \ifx\next\empty
      \relax
  \else
      \let\imn\assigninames
  \fi
  #1%
}

\def\assigninames#1{%
  \yypop\yyirulestack\into\toksc
  \def\sts##1{##1}%
  \savehcs{symn}{\term\lhs\implicitterm\onerule}%
  \edef\next{\noexpand\onerule{\the\toksc}}%
  \edef\next{\toksc{\next}}\next
  \toksb{\implicitrule[#1]:\def}%
  \yyreplacestring\toksb\in\newsymswitch\with\toksc
  \restorecs{symn}{\term\lhs\implicitterm\onerule}%
}

\def\oneruleid#1#2{%
  \oneimplicitrule{}%
  \yyinitstack\yyirulestack
  \toksb{#1}%
  #1%
  \makeisymnames#2\end
}

\def\termexsymname#1#2{%
  \appendr\oneimplicitrule{\noexpand\term{#1}{#2}}%
}

\def\iruleplaceholder#1:\def#2{}

\def\setexplicitinlinerules#1{%
  \let\onerule\oneruleid
  \let\term\termexsymname
  \let\lhs\eattwo
  \let\implicitterm\itermstack
  \let\ruleor\relax
  \let\ih\eatone
  \let\imn\eatone
  \let\implicitrule\iruleplaceholder
  \the#1% set explicit names  
}

\catcode`\^^A=\tempca

\def\makesymrefs#1{% #1 is the symbolic rule switch
    \let\ruleor\or
    \nameflagtoks{}%
    \edef\next{\setsncommands{\noexpand\or\space\harmlesscomment\space (rule 0)^^J%
                              \noexpand\or\space\harmlesscomment\space (rule 1)^^J}}\next
    \edef\next{\unsetsncommands{\noexpand\or\space\harmlesscomment\space (rule 0)^^J%
                                \noexpand\or\space\harmlesscomment\space (rule 1)^^J}}\next
    \yyn=\tw@
    \loop
        \yylen=\fgetelemof{yyrtwo}\at\yyn\relax
        \ifnum\yylen>\z@
        \else 
            \yylen=\fgetelemof{yyrthree}\at\yyn\relax
        \fi
        \xm@kesymrefs#1%
        \xm@k@symrefs#1%
        \addseparator
    \ifnum\yyn<\YYNRULES
        \advance\yyn\@ne
    \repeat
}

% these macros are here to hide \else and \fi tokens

\def\xm@kesymrefs#1{\expandafter\m@kesymrefs\expandafter\yyn\the#1\else\fi} % explicit names
\def\xm@k@symrefs#1{\expandafter\m@k@symrefs\expandafter\yyn\the#1\else\fi} % implicit names

\def\m@kesymrefs#1{%
  \let\onerule\rulesymsetup
  \ifcase#1\or\or % skip the first two rules
}

\def\m@k@symrefs#1{%
  \let\onerule\r@lesymsetup
  \ifcase#1\or\or % skip the first two rules
}

\def\addseparator{%
    \sprintrule\yyn\to\toksa
    \edef\next{\setsncommands{\the\setsncommands\noexpand\or\space\harmlesscomment\the\toksa^^J}}\next
    \edef\next{\unsetsncommands{\the\unsetsncommands\noexpand\or\space\harmlesscomment\the\toksa^^J}}\next
    \the\nameflagtoks
    \nameflagtoks{}%
}

\def\rulesymsetup#1#2{%
  \let\term\termsymname
  \let\lhs\lhssymname
  \let\implicitterm\term
  \tempca\@ne
  #1% set explicit names
}

\newtoks\setsncommands
\newtoks\unsetsncommands

\def\termsymname#1#2{% set the symbolic name, if it is nonempty
  \def\next{#2}%
  \ifx\next\empty
  \else % there is a symolic name
      \expandafter\ifx\csname$[#2]\endcsname\relax% i$ this name unassigned? 
          \setnameflag{#2}\end % ... this name has been explicitly set
          \setuprefs{#2}{\the\tempca}%
      \else
          \errmessage{ambiguous symbolic name: #2}%
      \fi
  \fi
  \advance\tempca\@ne
}

\newtoks\nameflagtoks

\def\setnameflag#1#2{%
    \toksa\expandafter{\csname$[#1]sym\endcsname}% $
    \edef\next{\let\the\toksa\noexpand#2}\next
    \appendr\nameflagtoks{\noexpand\unsetnameflag{#1}}%
}

\def\unsetnameflag#1{%
    \toksa\expandafter{\csname$[#1]sym\endcsname}% $
    \edef\next{\let\the\toksa\relax}\next
}

\def\unsetsymname#1{%
    \toksa\expandafter{\csname$[#1]\endcsname}% $
    \edef\next{\let\the\toksa\relax}\next
}

\def\unsetsym#1{%
    \toksa\expandafter{\csname$[#1]\endcsname}% $
    \toksb\expandafter{\csname$$[#1]\endcsname}% $
    \edef\next{\let\the\toksa\relax\let\the\toksb\relax}\next
}

\def\lhssymname#1#2{% set an explicit symbolic name for the left hand side
  \setuplhsref{#2}%
  \setnameflag{#2}\end % signal that this name has ben explicitly set
}

% the code below is just an example. One has to figure out how to record percent symbols and other 
% dangerous \TeX\ symbols; it is rather slow, a faster two stage scheme can be envisioned 
% where the first stage builds the command while iterating over the rule numbers; this is left as an exercise. 

\let\hc\harmlesscomment
\let\uu\space
\def\setuprefs#1#2{%
    \expandafter\let\csname$[#1]\endcsname.% $o the \ifx ... \relax test makes sense ...
    \appendr\nameflagtoks{\noexpand\unsetsymname{#1}}% clean it up later
    \toksa{}\expandafter\charstonumbers#1\end    
    \edef\next{\toksb{\space\space\noexpand\setsym{\the\toksa}{#2}\hc^^J\uu\uu\hc\hc\hc\uu#1 --> #2^^J}}\next
    \edef\next{\toksc{\space\space\noexpand\unsetsym{\the\toksa}\hc^^J\uu\uu\hc\hc\hc\uu#1 --> \relax^^J}}\next
    \concat\setsncommands\toksb
    \concat\unsetsncommands\toksc
}%

\def\setuplhsref#1{%
    \def\next{#1}%
    \ifx\next\empty
    \else
        \expandafter\let\csname$[#1]\endcsname.% $o the \ifx ... \relax test makes sense ...
        \appendr\nameflagtoks{\noexpand\unsetsymname{#1}}% clean it up later
        \toksa{}\expandafter\charstonumbers#1\end
        \edef\next{\toksb{\uu\uu\noexpand\setlhs{\the\toksa}\hc^^J\uu\uu\hc\hc\hc\uu#1 --> $$^^J}}\next
        \edef\next{\toksc{\uu\uu\noexpand\unsetsym{\the\toksa}\hc^^J\uu\uu\hc\hc\hc\uu#1 --> \relax^^J}}\next
        \concat\setsncommands\toksb
        \concat\unsetsncommands\toksc
    \fi
}%

\def\sprintrule#1\to#2{%
    \tempcc=#1\relax  
    \tempcd=\fgetelemof{yyrone}\at\tempcc\relax
    \edef\ruleline{\space(rule \number#1)\space\fgetelemof{yytname}\at\tempcd:}%
    \tempcd=\fgetelemof{yyprhs}\at\tempcc\relax
    \tempcc=\fgetelemof{yyrhs}\at\tempcd\relax
    \ifnum\tempcc>\m@ne
        \fillruleline
    \else
        \edef\ruleline{\ruleline\space<empty>}%
    \fi
    #2\expandafter{\ruleline}%
}

\def\fillruleline{%
    \edef\ruleline{\ruleline\space\fgetelemof{yytname}\at\tempcc}%
    \advance\tempcd\@ne
    \tempcc=\fgetelemof{yyrhs}\at\tempcd\relax
    \ifnum\tempcc>\m@ne
        \let\next\fillruleline
    \else
        \let\next\relax
    \fi
    \next
}

\def\r@lesymsetup#1#2{%
  \let\term\termsymextra
  \let\lhs\lhssymextra
  \let\implicitterm\term
  \tempca=\@ne
  #1% set implicit names
}

\def\termsymextra#1#2{% set an implicit symbolic name: 
                      %   1) if it has not been defined yet, define it
                      %   2) if it is defined implicitly, make it \def
  \expandafter\let\expandafter\next\csname$[#1]\endcsname% $ this is the name we are trying to define
  \ifx\next\relax % it is not defined yet
      \setuprefs{#1}{\the\tempca}%
  \else % already defined
      \expandafter\ifx\csname$[#1]sym\endcsname\end % it i$ an existing explicit symbolic name, done
      \else
          \expandafter\ifx\csname$[#1]sym\endcsname\noindent % it i$ an existing implicit symbolic name of lhs, overwrite it
              \setuprefs{#1}{\the\tempca}%
          \else
              \edef\next{\toksa{\space\space\noexpand\locksymname{#1}\harmlesscomment^^J}}\next
              \edef\next{\toksb{\space\space\noexpand\unlocksymname{#1}\harmlesscomment^^J}}\next
              \concat\setsncommands\toksa % symbolic name defined implicitly, disallow it
              \concat\unsetsncommands\toksb
          \fi
      \fi
  \fi
  \advance\tempca\@ne
}

\def\locksymname#1{\expandafter\let\csname$[#1]\endcsname\def} %$
\def\unlocksymname#1{\expandafter\let\csname$[#1]\endcsname\relax} %$

\def\lhssymextra#1#2{% set the name of the left hand side implicitly
  \expandafter\let\expandafter\next\csname$[#1]\endcsname%$
  \ifx\next\relax % not defined yet
      \setuplhsref{#1}%
      \setnameflag{#1}\noindent % this lhs name has been implicitly set
  \fi % it is already defined
}

\catcode`\^^A=\tempca

% stack variable access

\def\ym#1){%
  \ifnum#1<\@ne
      \putyyvalt
  \else
      \ifnum#1>\yylen
          \yylen\errmessage{parameter out of range (#1\space out of\space \the\yylen)}%
          \putvarerror
      \else
          \expandafter\putyytoksx\csname$$'#1\endcsname
      \fi
  \fi
}

\def\yn#1#{%
  \ifnum#1<\@ne
      \putyyval
  \else
      \ifnum#1>\yylen
          \errmessage{parameter out of range}%
          \putvarerror
      \else
          \expandafter\putyyassignment\csname$'#1\endcsname%$
      \fi
  \fi
}

\def\yx#1[{%
  \expandafter\ifx\csname$[#1]\endcsname\yyval%$
      \putyyvalt
  \else
      \expandafter\ifx\csname$[#1]\endcsname\def%$
          \errmessage{reference to ambiguous symbolic name: #1}%
          \putvarerror
      \else
          \expandafter\putyytoksx\csname$$[#1]\endcsname
      \fi
  \fi
}

\def\yz#1]{%
  \expandafter\ifx\csname$[#1]\endcsname\yyval%$
      \putyyval
  \else
      \expandafter\ifx\csname$[#1]\endcsname\def%$
          \errmessage{reference to ambiguous symbolic name: #1}%
          \putvarerror
      \else
          \expandafter\putyyassignment\csname$[#1]\endcsname%$
      \fi
  \fi
}

\def\putyyval\else#1\fi\fi{%
  \fi\yyvalx
}

\def\yyvalx#1{%
  \edef\next{\yyval{#1}}\next
}

\def\putyyvalt\else#1\fi\fi{%
  \fi\yyval
}

\def\putvarerror\else#1\fi\fi{%
  \fi\fi\p@tvarerror
}

\def\p@tvarerror#1{%
  \ifx#1\empty\empty
  \else
      #1{ERROR}%
  \fi
}

\def\putyyassignment#1\fi\fi{%
  \fi\fi\p@tyyassignment#1%
}

\def\putyytoksx#1\fi\fi{%
  \fi\fi#1%
}

\def\p@tyyassignment#1#2{%
  \ifx#2\empty\empty
      #1%
  \else
      #2\expandafter{#1}%
  \fi
}

\def\yy#1{%
  \ifx#1[%
      \yysym
  \else
      \ifx#1]%
          \yysymt
      \else
          \ifx#1(%
              \yynumt
          \else
              \yynum#1%
          \fi
      \fi
  \fi
}

\def\yysym\else#1\fi\fi\fi{%
  \fi\yz
}

\def\yysymt\else#1\fi\fi\fi{%
  \fi\fi\yx
}

\def\yynumt#1\fi\fi\fi{%
  \fi\fi\fi\ym%
}

\def\yynum#1\fi\fi\fi{%
  \fi\fi\fi\yn#1%
}

% symbolic name macros that prepare the environment for the use of the stack access macros above

\def\setsymcs#1#2{%
    \toksa{}\numberstochars#1\end
    \toksb\expandafter{\csname$[\the\toksa]\endcsname}% $
    \toksc\expandafter{\csname$'#2\endcsname}% $
    \edef\next{\let\the\toksb\the\toksc}\next
}

\def\setsymtr#1#2{%
    \toksa{}\numberstochars#1\end
    \toksb\expandafter{\csname$$[\the\toksa]\endcsname}%
    \toksc\expandafter{\csname$$'#2\endcsname}%
    \edef\next{\let\the\toksb\the\toksc}\next
}

\def\setlhs#1{%
    \toksa{}\numberstochars#1\end
    \toksb\expandafter{\csname$[\the\toksa]\endcsname}% $
    \edef\next{\let\the\toksb\yyval}\next
}

\def\setsym#1#2{%
    \setsymcs{#1}{#2}%
    \setsymtr{#1}{#2}%
}
