/// unit obsahujuci triedy a procedury pouzite pri syntaktickej analyze
{$I main.inc}
unit main;

interface

uses constants, structures, log, filereader, tables, hashmap, Windows, graphics, classes, forms;

type
  PTAState = ^TAState;
  PTALink = ^TALink;

  /// record popisujuci link (prechod) v konecnom automate
  TALink = record
    src: PTAState;
    dest: PTAState;
    t: Integer;
    linktype: Integer;  //LT_EPS = 0, LT_CHAR = 1, LT_SET = 2
  end;

  /// record popisujuci stav v konecnom automate
  TAState = record
    links: TQueueStack;
    accepts: Integer;
  end;

  PToken = ^TToken;
  /// record popisujuci vrchol v syntaktickom strome
  TToken = record
    tokentype: Integer;
    value: xString;
    startp,endp: Integer;
    child: PToken;
    sibling: PToken;
    _parent: PToken;
    _inorder: Integer;
    _id: Integer;
    _hash: DWORD;
  end;

  PTokenHighlight = ^TTokenHighlight;
  /// record popisujuci farbu, farbu pozadia a styl textu pre dany token
  TTokenHighlight = record
    tc: TColor;
    bc: TColor;
    style: Integer;
  end;

  PTokenData = ^TTokenData;
  /// record popisujuci formatovanie textu pre dany token
  TTokenData = record
    bspace: Integer;
    bindent: Integer;
    beoln: Integer;
    bexcept: TIntHashSet;
    aspace: Integer;
    aindent: Integer;
    aeoln: Integer;
    aexcept: TIntHashSet;
  end;

  /// trieda - nedeterministicky konecny automat
  TNFA = class
    states: TQueueStack;
    currentStates: TQueueStack;
    FCaseSensitive: Boolean;
    FSetTable: TSetTable;
  private
  public
    constructor create(bCaseSensitive: Boolean = true); overload;
    constructor create(SetTable: TSetTable; bCaseSensitive: Boolean = true); overload;
    procedure freeState(st00: PTAState);
    destructor Destroy; override;
    function epsilonClosure(var Queue: TQueueStack): Integer;
    procedure startSimulation;
    function simulate(c: char): Integer;
    procedure addState(state: PTAState);
    procedure addLink(state: PTAState; link: PTALink);
    procedure addLinks(link: PTALink);    //add to link.src and link.dest
    function addLinkEx(state: PTAState; src, dest: PTAState; t: Integer; linktype: Integer): PTALink;
    function addLinksEx(src, dest: PTAState; t: Integer; linktype: Integer): PTALink;
    function buildLinkEx(src, dest: PTAState; t: Integer; linktype: Integer): PTALink;
    function addStateEx(acc: Integer): PTAState;
    function toString(onlyOutgoing: boolean = false): xString;
    procedure setCaseSensitivity(bCaseSensitive: Boolean = True);
    function getCaseSensitivity: Boolean;
    procedure addAcceptingBranch(src: PTAState; S: xString; acc: Integer);
    function getSize: Integer;
    function getNumOfLinks: Integer;
//    function epsilonClosure2(NStates_in: TIntHashSet; var NStates_out, PossibleMoves, PossibleSetMoves: TIntHashSet): Integer;
    function epsilonClosure3(NStates_in: TIntHashSet; var NStates_out: TIntHashSet; var PossibleMoves, PossibleSetMoves: TIntIntHashSet): Integer;
//    function move2(NStates_in: TIntHashSet; symbol: Integer; var NStates_out: TIntHashSet; setindices: TIntHashSet = nil): Integer;
  end;

  PDState = ^TDState;
  /// record popisujuci stav v DFA
  TDState = record
    nstates: TIntHashSet;
    moves: TIntIntHashSet;
    accepts: Integer;
  end;

  /// trieda - deterministicky konecny automat
  TDFA = class
  private
    FAllocated: Integer;
    FCount: Integer;
    FCurrentState: Integer;
    LNFA: TNFA;
    FCaseSensitive: Boolean;
  public
    a: array of PDState;  //dstates
    items: TIntIntHashSet;  //nstates
    constructor create(NFA: TNFA; size: Integer = 256); overload;
    destructor Destroy; override;
    function add(Nstates: TIntHashSet; var index_out: Integer): Boolean;
    function simulate(c: char): Integer;
    procedure reset;
//    function build: Boolean;
//    function build2: Boolean;
    function build3: Boolean;
    function toString: xString;
    function getSize: Integer;
    procedure setCaseSensitivity(bCaseSensitive: Boolean = True);
    function getCaseSensitivity: Boolean;
  end;

  /// trieda implementujuca lexikalny analyzator postaveny na NFA/DFA
  TLexer = class
    LFileReader: TSimpleReader;
    LNFA: TNFA;
    LDFA: TDFA;
    FCaseSense: Boolean;
  private
  public
    constructor create(FileReader: TSimpleReader; NFA: TNFA = nil; DFA: TDFA = nil);
    function getTokenNFA(var tokentype: Integer; var position: Integer): xString;
    function getTokenDFA(var tokentype: Integer; var position: Integer): xString;
    function getTokenDFA_faster(var tokentype, position: Integer): xString;
    function getTToken: TToken; overload;
    function getToken: PToken; overload;
    procedure setCaseSensitivity(bCaseSensitive: Boolean = True);
    function getCaseSensitivity: Boolean;
  end;

  /// trieda, ktora analyzuje vstupnu gramatiku a vytvara NFA
  TGrammarFileLexer = class
  private
    Fline, Fcol: Integer;
    FFileReader: TSimpleReader;
    FPropertyTable: TPropertyTable;
    FSetTable: TSetTable;
    FWhiteSpaceSet,FOpSet,FAlphanumericSet,FRegExpOpSet,FDigitSet: TCharSet;
    FHexDigitSet,FSLSSet: TCharSet;
    FLexerNFA: TNFA;
//    state: Integer;
    FTokenNameTypeTable: TStrIntHashMap;
    FStateNameTypeTable: TStrIntHashMap;
    function isNumber(S: xString): Boolean;
  public
    FIndTable: TIntIntHashSet;
    FHLTable: TIntIntHashSet;
    FRuleTable: TRuleTable;
    FLog: TLog; //!TODO! general Log class
    constructor create(FileReader: TSimpleReader);
    destructor Destroy; override;
    function getNextToken_old(var TokenType: Integer): xString;
    function getNextToken(var TokenType: Integer): xString;
    function parse: boolean;
    function getNFA: TNFA;
    function getTokenNameTypeTable: TStrIntHashMap;
    function getStateNameTypeTable: TStrIntHashMap;
    function getRuleTable: TRuleTable;
    function resolveEscSeq(S: xString): xString;
//    function getNextNonWSToken: xString;
//    function getNextToken(var name,value: xString):boolean;
  end;

  /// record popisujuci LR0 polozku
  TLR0Item = record
    rulenum: Integer;
    dot: Integer;
  end;

  PLALRItem = ^TLALRItem;
  /// record popisujuci LALR polozku
  TLALRItem = record
    lr0: TLR0Item;
    lookahead: TIntHashSet;
    lapropagate: TIntHashSet;
    bPropagated: Boolean; //if not bPropagated then add lookahead to all items in lapropagate
  end;

  PItemSetItem = ^TItemSetItem;
  /// record popisujuci prvok mnoziny (zretazeneho zoznamu) LALR poloziek
  TItemSetItem = record
    lalr: PLALRItem;
    next: PItemSetItem;
  end;

  /// trieda implementujuca mnozinu LALR poloziek (stav LALR parsera)
  TItemSet = class
  private
    FCapacity: Integer;
    FCount: Integer;
    FBuckets: array of PItemSetItem;
//    FUnique: Boolean;
    FCurrentItem: PItemSetItem;
    FCurrentBucket: Integer;
    function getFirst(var plalritem: PLALRItem): Boolean;
    function getNext(var plalritem: PLALRItem): Boolean;
    procedure getNextReset;
    function isEmpty: Boolean;
  public
    constructor create; overload;
    constructor create(cap: Integer; icap: Integer); overload;
    procedure clear;
    destructor Destroy; override;
    function hash(lr0: TLR0Item): DWORD;
    function add(lr0: TLR0Item): Boolean; overload;
    function remove(lr0: TLR0Item): Boolean;
    function add(lalr: TLALRItem): Boolean; overload;
    function add(var lalr: PLALRItem): Boolean; overload;
//    function add2(lalr: TLALRItem): Boolean; overload;
    function add2(var lalr: PLALRItem): Boolean; overload;
    function contains(lr0: TLR0Item): Boolean;
    function get(lr0: TLR0Item; var lalritem: TLALRItem): Boolean; overload;
    function get(lr0: TLR0Item; var plalritem: PLALRItem): Boolean; overload;
    function getSize: Integer;  //??? error
    function toString(bComplex: Boolean = false; tnametable: TStrIntHashMap = nil; snametable: TStrIntHashMap = nil; ruletable: TRuleTable = nil): xString;
    function toStringList(bComplex: Boolean = false; tnametable: TStrIntHashMap = nil; snametable: TStrIntHashMap = nil; ruletable: TRuleTable = nil): TStringList;
  end;

  /// trieda implementujuca mnozinu mnozin LALR stavov (vsetky stavy LALR parsera)
  TSetOfItemSets = class
  private
    FCount: Integer;
    FAllocated: Integer;
  public
    a: array of TItemSet;
    items: TIntIntHashSet;  //TItemSet,index in array
//    function hash(key: Integer; capacity: Integer): DWORD;
//    function equal(key1: Integer; key2: Integer): Boolean;
    constructor create(size: Integer = 64);
    destructor Destroy; override;
    function add(itemset: TItemSet; var outindex: Integer): Boolean;  //-1 => ERROR, otherwise index of a (added or changed)
    function getByIndex(index: Integer): TItemSet;
    function getIndexByItemSet(itemset: TItemSet): Integer;
    function toString(bComplex: Boolean = false; tnametable: TStrIntHashMap = nil; snametable: TStrIntHashMap = nil; ruletable: TRuleTable = nil): xString;
    function getSize: Integer;
  end;

  /// trieda implementujuca LALR parsovanie na zaklade vstupnych parsovacich tabuliek
  TParser = class
  private
    LLexer: TLexer;
  public
    LTokenNameTypeTable: TStrIntHashMap;
    LStateNameTypeTable: TStrIntHashMap;
    LRuleTable: TRuleTable;
    FGotoTable,FReduceTable: TIntIntHashSet;
    FGlobalSet: TSetOfItemSets;
    FConflictStates: TIntIntHashSet;
    FLog: TLog;
    constructor create(lexer: TLexer; ttable,stable: TStrIntHashMap; ruletable: TRuleTable; gototable,reducetable: TIntIntHashSet);
    destructor Destroy; override;
    function parse(var root: PToken; var size: Integer): Boolean;
    function toString: xString;
    procedure linkwithLexer(lexer: TLexer);
  end;

  /// trieda vytvarajuca LALR parser
  TParserBuilder = class
  private
    LLexer: TLexer;
    LTokenNameTypeTable: TStrIntHashMap;
    LStateNameTypeTable: TStrIntHashMap;
    LRuleTable: TRuleTable;
    FFirst: TIntIntHashSet;
    FFollow: TIntIntHashSet;
    FParser: TParser;
//    function lr0hash(item: TLR0Item): Integer;
    procedure copyLALR(_from, _to: PLALRItem);
    function cloneLALR(_from: PLALRItem): PLALRItem;
  public
    FLog: TLog;
    constructor create(lexer: TLexer; ttable,stable: TStrIntHashMap; ruletable: TRuleTable);
    destructor Destroy; override;
    function First(symbol: Integer): TIntHashSet; overload;
    function First(symbols: TQueueStack): TIntHashSet; overload;
    function Follow(symbol: Integer): TIntHashSet;
    function calculateFirst: Boolean;
    function calculateFollow: Boolean;
    function buildLALRParser: boolean;
    function getParser: TParser;
    function closure(itemset: TItemSet): TItemSet;
    function closure3(itemset: TItemSet): TItemSet;
    function closure2(itemset: TItemSet): TItemSet;
  end;

implementation

uses
  SysUtils,  simple, treedist, logframe;

/// funkcia vracajuca hasovaciu hodnotu pre mnozinu poloziek (pointer na tuto mnozinu je vo vstupnom integeri - key)
/// vystupne hodnoty su v rozmedzi <0-capacity)
function ItemSet_hash(key, capacity: Integer): DWORD;
var I: Integer;
    k: Integer;
    itemset: TItemSet;
    plalr: PLALRItem;
begin
  result := 5381;
  itemset := TItemSet(key);
  itemset.getNextReset;
  while itemset.getNext(plalr) do begin
    result := result shl 5 + result + plalr.lr0.rulenum;
    result := result shl 5 + result + plalr.lr0.dot;
  end;
  result := result mod DWORD(capacity);
end;

/// funkcia vracajuca hasovaciu hodnotu pre danu mnozinu poloziek (pointer na tuto mnozinu je vo vstupnom integeri - key)
/// s tym, ze sa nepocita s epsilon polozkami,
/// vystupne hodnoty su v rozmedzi <0-capacity)
function ItemSet_kernelhash(key, capacity: Integer): DWORD;
var I: Integer;
    k: Integer;
    itemset: TItemSet;
    plalr: PLALRItem;
begin
  result := 5381;
  itemset := TItemSet(key);
  itemset.getNextReset;
  while itemset.getNext(plalr) do begin
    if plalr.lr0.dot > 1 then begin
      result := result shl 5 + result + plalr.lr0.rulenum;
      result := result shl 5 + result + plalr.lr0.dot;
    end;
  end;
  result := result mod DWORD(capacity);
end;

/// funkcia, ktora vracia ci sa dane dve vstupne mnoziny poloziek zhoduju;
/// vrati true ak maju rovnake polozky, inak false
function ItemSet_equal(key1, key2: Integer): Boolean;  //itemset compare - equal if all items are equal
var itemset,itemset2: TItemSet;
    plalr,plalr2: PLALRItem;
begin
  result := false;
  itemset := TItemSet(key1);
  itemset2 := TItemSet(key2);
  itemset.getNextReset;
  while itemset.getNext(plalr) do begin
    if not itemset2.contains(plalr.lr0) then
      Exit;
  end;
  itemset2.getNextReset;
  while itemset2.getNext(plalr) do begin
    if not itemset.contains(plalr.lr0) then
      Exit;
  end;
  result := true;
end;

/// funkcia, ktora vracia ci sa dane dve vstupne mnoziny poloziek zhoduju;
/// vrati true ak maju rovnake polozky (okrem epsilon poloziek), inak false
function ItemSet_kernelequal(key1, key2: Integer): Boolean;  //itemset kernel compare - equal if all (itemsets - .X LALRItems) are equal
var itemset,itemset2: TItemSet;
    plalr,plalr2: PLALRItem;
begin
  result := false;
  itemset := TItemSet(key1);
  itemset2 := TItemSet(key2);
  itemset.getNextReset;
  while itemset.getNext(plalr) do begin
    if plalr.lr0.dot > 1 then
      if not itemset2.contains(plalr.lr0) then
        Exit;
  end;
  itemset2.getNextReset;
  while itemset2.getNext(plalr) do begin
    if plalr.lr0.dot > 1 then
      if not itemset.contains(plalr.lr0) then
        Exit;
  end;
  result := true;
end;

/// funkcia vracajuca hasovaciu hodnotu pre mnozinu stavov NFA (pointer na tuto mnozinu je vo vstupnom integeri - key)
/// vystupne hodnoty su v rozmedzi <0-capacity)
function NStates_hash(key, capacity: Integer): DWORD;
var I: Integer;
    k: Integer;
    hashset: TIntHashSet;
begin
  result := 5381;
  hashset := TIntHashSet(key);
  hashset.getNextReset;
  while hashset.getNext(I) do begin
    result := result shl 5 + result + I;
  end;
  result := result mod DWORD(capacity);
end;

/// funkcia, ktora vracia ci sa dane dve vstupne mnoziny stavov NFA zhoduju;
/// vrati true ak maju rovnake prvky, inak false
function NStates_equal(key1, key2: Integer): Boolean;
var hashset,hashset2: TIntHashSet;
    I: Integer;
begin
  result := false;
  hashset := TIntHashSet(key1);
  hashset2 := TIntHashSet(key2);
  hashset.getNextReset;
  while hashset.getNext(I) do begin
    if not hashset2.contains(I) then
      Exit;
  end;
  hashset2.getNextReset;
  while hashset2.getNext(I) do begin
    if not hashset.contains(I) then
      Exit;
  end;
  result := true;
end;

{TNFA}

/// konstuktor vytvarajuci NFA objekt
constructor TNFA.create(bCaseSensitive: Boolean = true);
begin
  create(nil,bCaseSensitive);
end;

/// konstuktor vytvarajuci NFA objekt na zaklade vstupnych paramaterov;
/// SetTable je tabulka mnozin znakov definovana v subore popisujucom gramatiku;
/// bCaseSensitive urcuje ci NFA bude odlisovat male a velke pismena pri odvodzovani
constructor TNFA.create(SetTable: TSetTable; bCaseSensitive: Boolean = true);
begin
  inherited create;
  states := TQueueStack.create;
  currentStates:= TQueueStack.create;
  FSetTable := SetTable;
  FCaseSensitive := bCaseSensitive;
end;

/// funkcia uvolnujuca stav NFA z pamate (aj so vsetkymi k nemu prisluchajucimi linkami)
procedure TNFA.freeState(st00: PTAState);
var lk00: PTALink;
begin
  if st00 = nil then Exit;
  while not st00.links.isEmpty do begin
    lk00 := PTALink(st00.links.dequeue);
    if st00 = lk00.src then begin
      PTAState(lk00.dest).links.list.remandfree(PTAState(lk00.dest).links.list.getByData(Integer(lk00)));
    end
    else begin
      PTAState(lk00.src).links.list.remandfree(PTAState(lk00.src).links.list.getByData(Integer(lk00)));
    end;
    dispose(lk00);
  end;
  st00.links.free;
  dispose(st00);
end;

/// destruktor uvolnujuci tento objekt (NFA) z pamate
destructor TNFA.Destroy;
var st00: PTAState;
    lk00: PTALink;
begin
  while not states.isEmpty do begin
    freeState(PTAState(states.dequeue));
  end;
  states.Free;

  currentStates.Free;
  inherited;
end;

/// procedura pridavajuca novy stav - state do automatu
procedure TNFA.addState(state: PTAState);
begin
  if state = nil then exit;
  states.push(Integer(state));
end;

/// procedura pridavajuca novy prechod - link do stavu - state v automate
procedure TNFA.addLink(state: PTAState; link: PTALink);
begin
  if state = nil then exit;
  state.links.push(Integer(link));
end;

/// procedura pridavajuca novy prechod - link do relevantnych stavov (aj do zdrojoveho aj cieloveho stavu)
procedure TNFA.addLinks(link: PTALink);
begin
  addLink(link.src,link);
  addLink(link.dest,link);
end;

/// procedura pridavajuca novy prechod, ktory je vytvoreny zo vstupnych argumentov - src,dest,t,linktype do stavu - state v automate;
/// src je zdrojovy stav, dest je cielovy stav, t je znak, resp. id mnoziny v tabulke znakov po precitani ktoreho, resp. ktorej sa
/// uskutocni krok odvodenia v automate, linktype je type prechodu (0 - epsilon, 1 - znak, 2 - mnozina znakov)
function TNFA.addLinkEx(state: PTAState; src, dest: PTAState; t: Integer; linktype: Integer): PTALink;
var L: PTALink;
begin
//  if state = nil then exit;
  new(L);
  L.src := src;
  L.dest := dest;
  L.t := t;
  L.linktype := linktype;
  addLink(state,L);
  result := L;
end;

/// procedura vytvarajuca prechod (link) zo vstupnych parametrov;
/// src je zdrojovy stav, dest je cielovy stav, t je znak, resp. id mnoziny v tabulke znakov po precitani ktoreho, resp. ktorej sa
/// uskutocni krok odvodenia v automate, linktype je type prechodu (0 - epsilon, 1 - znak, 2 - mnozina znakov)
function TNFA.buildLinkEx(src, dest: PTAState; t: Integer; linktype: Integer): PTALink;
var L: PTALink;
begin
//  if state = nil then exit;
  new(L);
  L.src := src;
  L.dest := dest;
  L.t := t;
  L.linktype := linktype;
//  addLink(state,L);
  result := L;
end;

/// procedura pridavajuca nove prechody (src->dest aj spatny dest->src), ktore su vytvorene zo vstupnych argumentov - src,dest,t,linktype do stavu - state v automate;
/// src je zdrojovy stav, dest je cielovy stav, t je znak, resp. id mnoziny v tabulke znakov po precitani ktoreho, resp. ktorej sa
/// uskutocni krok odvodenia v automate, linktype je type prechodu (0 - epsilon, 1 - znak, 2 - mnozina znakov)
function TNFA.addLinksEx(src, dest: PTAState; t: Integer; linktype: Integer): PTALink;
var L: PTALink;
begin
//  if state = nil then exit;
  new(L);
  L.src := src;
  L.dest := dest;
  L.t := t;
  L.linktype := linktype;
  addLink(src,L);
  if src <> dest then addLink(dest,L);
  result := L;
end;

/// procedura vytvarajuca novy stav v automate s akceptacnym symbolom - acc (ak je 0 tak neakceptacny stav,
/// inak akceptuje a vracia hodnotu acc (identifikator tokenu)
/// funkcia vracia pointer na tento novy stav
function TNFA.addStateEx(acc: Integer): PTAState;
var P: PTAState;
begin
  new(P);
  P.links := TQueueStack.create;
  P.accepts := acc;
  addState(P);
  result := P;
end;

/// procedura na pridanie akceptujucej vetvy do NFA
/// vetva zacne v stave src (ak je nil, tak v pociatocnom stave),
/// a po precitani stringu S akceptuje a vrati hodnotu acc (identifikator tokenu)
procedure TNFA.addAcceptingBranch(src: PTAState; S: xString; acc: Integer);
var i: Integer;
    state: PTAState;
begin
//  S := resolveEscSeq(S);
  if src = nil then begin
    src := PTAState(states.peek);
    if src = nil then
      Exit;
  end;

//  state0 := addStateEx(0);
//  addLinksEx(src,state0,S[1],LT_CHAR);
  for i := 1 to Length(S) do begin
    state := addStateEx(0);
    addLinksEx(src,state,Integer(S[i]),LT_CHAR);
    src := state;
  end;
  src.accepts := acc;
end;

/// funkcia na vypis NFA do stringu, onlyOutgoing udava, ci sa budu vypisovat
/// aj prichadzajuce linky k danemu stavu, alebo len vychadzajuce
function TNFA.toString(onlyOutgoing: boolean = false): xString;
var state: PTAState;
    link: PTALink;
    i: Integer;
    S: xString;
begin
    S := '[[NFA]]'+ EOL;
    states.peekNextReset;  //currentNode = nil
    while not (states.getCurrentNode = states.getLastNode) do begin
      state := PTAState(states.peekNext);
      S := S + '  [' + IntToStr(Integer(state)) + ':' + IntToStr(state.accepts) + ']' + EOL;
      state.links.peekNextReset;  //currentNode = nil
      while not (state.links.getCurrentNode = state.links.getLastNode) do begin
        link := PTALink(state.links.peekNext);
        if (link.src = state) or (not onlyOutgoing) then begin
          case link.linktype of
            LT_EPS: S := S + '    -[' + IntToStr(Integer(link.src)) + ':' + IntToStr(link.src.accepts) + ']----(' + IntToStr(Integer(link)) + ':' + '{Epsilon}' + '_' + IntToStr(link.linktype) + ')---->[' + IntToStr(Integer(link.dest)) + ':' + IntToStr(link.dest.accepts) + ']' + EOL;
            LT_CHAR: S := S + '    -[' + IntToStr(Integer(link.src)) + ':' + IntToStr(link.src.accepts) + ']----(' + IntToStr(Integer(link)) + ':' + char(link.t) + '_' + IntToStr(link.linktype) + ')---->[' + IntToStr(Integer(link.dest)) + ':' + IntToStr(link.dest.accepts) + ']' + EOL;
            LT_SET: begin
                      if not (FSetTable = nil) then begin
                        if not (FSetTable.getByIndex(link.t) = nil) then begin
                          S := S + '    -[' + IntToStr(Integer(link.src)) + ':' + IntToStr(link.src.accepts) + ']----(' + IntToStr(Integer(link)) + ':' + '{' + FSetTable.getByIndex(link.t).name + '}' + '_' + IntToStr(link.linktype) + ')---->[' + IntToStr(Integer(link.dest)) + ':' + IntToStr(link.dest.accepts) + ']' + EOL;
                        end
                        else begin
                          S := S + '    -[' + IntToStr(Integer(link.src)) + ':' + IntToStr(link.src.accepts) + ']----(' + IntToStr(Integer(link)) + ':' + '{' + '#UNDEFINED SET!!!' + '}' + '_' + IntToStr(link.linktype) + ')---->[' + IntToStr(Integer(link.dest)) + ':' + IntToStr(link.dest.accepts) + ']' + EOL;
                        end
                      end
                      else begin
                        S := S + '    -[' + IntToStr(Integer(link.src)) + ':' + IntToStr(link.src.accepts) + ']----(' + IntToStr(Integer(link)) + ':' + IntToStr(link.t) + '_' + IntToStr(link.linktype) + ')---->[' + IntToStr(Integer(link.dest)) + ':' + IntToStr(link.dest.accepts) + ']' + EOL;
                      end;
                    end;
            else begin
              S := S + '    -[' + IntToStr(Integer(link.src)) + ':' + IntToStr(link.src.accepts) + ']----(' + IntToStr(Integer(link)) + ':' + IntToStr(link.t) + '_' + IntToStr(link.linktype) + ')---->[' + IntToStr(Integer(link.dest)) + ':' + IntToStr(link.dest.accepts) + ']' + EOL;
            end;
          end;
        end;
      end;
    end;
    result := S;
end;

/// funkcia vracajuca pocet stavov NFA
function TNFA.getSize: Integer;
begin
  result := states.getSize;
end;

/// funkcia vracajuca pocet linkov (prechodov) v NFA
function TNFA.getNumOfLinks: Integer;
var state: PTAState;
begin
  result := 0;
  states.peekNextReset;
  while states.peekNext(Integer(state)) do begin
    result := result + state.links.getSize;
  end;
end;

/// procedura na nastavanie citlivosti na velke a male pismena
procedure TNFA.setCaseSensitivity(bCaseSensitive: Boolean = True);
begin
  FCaseSensitive := bCaseSensitive;
end;

/// procedura vracajuca momentalne nastavanie citlivosti na velke a male pismena
function TNFA.getCaseSensitivity: Boolean;
begin
  result := FCaseSensitive;
end;

(*
function TNFA.epsilonClosure2(NStates_in: TIntHashSet; var NStates_out: TIntHashSet; var PossibleMoves: TIntHashSet; var PossibleSetMoves: TIntHashSet): Integer;
var state: PTAState;
    link: PTALink;
    Q: TQueueStack;
    I: Integer;
begin
  result := 0;
  Q := TQueueStack.create;
  NStates_out := TIntHashSet.create;
  NStates_in.getNextReset;
  while NStates_in.getNext(I) do begin
    Q.enqueue(I);
    NStates_out.add(I);
  end;
//  NStates_out := NStates_in.clone;
  PossibleMoves := nil;
  PossibleSetMoves := nil;
  while not Q.isEmpty do begin
    state := PTAState(Q.dequeue);
//    if NStates_out.add(Integer(state)) then begin
      if state.accepts > result then result := state.accepts;
      state.links.peekNextReset;  //currentNode = nil
      while not (state.links.getCurrentNode = state.links.getLastNode) do begin
        link := PTALink(state.links.peekNext);
        if (link.src = state) then begin  //following outgoing links only
          if link.linktype = LT_EPS then begin
            if NStates_out.add(Integer(link.dest)) then begin
              Q.push(Integer(link.dest));
            end;
          end
          else if link.linktype = LT_CHAR then begin
            if PossibleMoves = nil then
              PossibleMoves := TIntHashSet.create;
            PossibleMoves.add(link.t);
          end
          else begin
            if PossibleSetMoves = nil then
              PossibleSetMoves := TIntHashSet.create;
            PossibleSetMoves.add(link.t);
          end;
        end;
      end;
//    end;
  end;
  Q.Free;
end;

function TNFA.move2(NStates_in: TIntHashSet; symbol: Integer; var NStates_out: TIntHashSet; setindices: TIntHashSet = nil): Integer;
var state: PTAState;
    link: PTALink;
begin
    result := 0;
    NStates_out := TIntHashSet.create;
    NStates_in.getNextReset;
    while NStates_in.getNext(Integer(state)) do begin
      if state.accepts > result then result := state.accepts;
      state.links.peekNextReset;  //currentNode = nil
      while not (state.links.getCurrentNode = state.links.getLastNode) do begin
        link := PTALink(state.links.peekNext);
        if (link.src = state) then begin  //following outgoing links only
          if link.t = symbol then begin
            if link.linktype = LT_CHAR then
              NStates_out.add(Integer(link.dest));
          end
          else if (setindices <> nil) and (link.linktype = LT_SET) then begin
            if setindices.contains(link.t) then begin
              NStates_out.add(Integer(link.dest));
            end;
          end;
        end;
      end;
    end;
end;
*)

/// funkcia vytvarajuca uzaver mnoziny vstupnych NFA stavov - NStates_in, ktory vracia v mnozine NStates_out, dalej
/// vytvori mnozinu PossibleMoves, kde su vsetky znaky pre ktore je definovany prechod z nejakeho stavu von z tejto mnoziny stavov,
/// vysledkom funkcie je aj to, ci sa v tomto uzavere nachadza aj akceptacny stav
function TNFA.epsilonClosure3(NStates_in: TIntHashSet; var NStates_out: TIntHashSet; var PossibleMoves: TIntIntHashSet; var PossibleSetMoves: TIntIntHashSet): Integer;
var state: PTAState;
    link: PTALink;
    Q: TQueueStack;
    I: Integer;
    hashSet: TIntHashSet;
begin
  result := 0;
  Q := TQueueStack.create;
  NStates_out := TIntHashSet.create;
  NStates_in.getNextReset;
  while NStates_in.getNext(I) do begin
    Q.enqueue(I);
    NStates_out.add(I);
  end;
//  NStates_out := NStates_in.clone;
  PossibleMoves := nil;
  PossibleSetMoves := nil;
  while not Q.isEmpty do begin
    state := PTAState(Q.dequeue);
//    if NStates_out.add(Integer(state)) then begin
      if state.accepts > result then result := state.accepts;
      state.links.peekNextReset;  //currentNode = nil
      while not (state.links.getCurrentNode = state.links.getLastNode) do begin
        link := PTALink(state.links.peekNext);
        if (link.src = state) then begin  //following outgoing links only
          if link.linktype = LT_EPS then begin
            if NStates_out.add(Integer(link.dest)) then begin
              Q.push(Integer(link.dest));
            end;
          end
          else if link.linktype = LT_CHAR then begin
            if PossibleMoves = nil then
              PossibleMoves := TIntIntHashSet.create;
            if not PossibleMoves.get(link.t,Integer(hashset)) then begin
              hashset := TIntHashSet.create(Integer(link.dest));
              PossibleMoves.add(link.t,Integer(hashset));
            end
            else begin
              hashset.add(Integer(link.dest));
            end;
          end
          else if link.linktype = LT_SET then begin
            if PossibleSetMoves = nil then
              PossibleSetMoves := TIntIntHashSet.create;
            if not PossibleSetMoves.get(link.t,Integer(hashset)) then begin
              hashset := TIntHashSet.create(Integer(link.dest));
              PossibleSetMoves.add(link.t,Integer(hashset));
            end
            else begin
              hashset.add(Integer(link.dest));
            end;
          end;
        end;
      end;
//    end;
  end;
  Q.Free;
end;

/// funkcia vytvarajuca uzaver fronty vstupnych NFA stavov - Queue, ktory vracia v tiez v tejto premennej,
/// vysledkom funkcie je aj to, ci sa v tomto uzavere nachadza aj akceptacny stav
function TNFA.epsilonClosure(var Queue: TQueueStack): Integer;
var state: PTAState;
    link: PTALink;
    acc: Integer;
begin
    acc := 0;
    Queue.peekNextReset;
    while not (Queue.getCurrentNode = Queue.getLastNode) do begin
      state := PTAState(Queue.peekNext);
      if state.accepts > acc then acc := state.accepts;
      state.links.peekNextReset;  //currentNode = nil
      while not (state.links.getCurrentNode = state.links.getLastNode) do begin
        link := PTALink(state.links.peekNext);
        if (link.src = state) then begin  //following outgoing links only
          if link.linktype = LT_EPS then begin
            Queue.pushEx(Integer(link.dest));
          end;
        end;
      end;
    end;
    result := acc;
end;

/// procedura inicializujuca spracovavanie vstupneho slova
procedure TNFA.startSimulation;
begin
//  currentStates.Free;
//  currentStates := TQueueStack.create;
  currentStates.clear;
  currentStates.push(Integer(states.peek));
  epsilonClosure(currentStates);
//  currentStates.push(Integer(startState));
end;

/// funkcia, ktora simuluje krok automatu pri vstupnom znaku c,
/// vrati, ci sa NFA nachadza po spracovani tohto znaku v akceptacnom stave
function TNFA.simulate(c: char): Integer;
var state: PTAState;
    link: PTALink;
    tempQueue: TQueueStack;
    acc: Integer;
    tempCharSet: TCharSet;
begin
  acc := 0;
  if currentStates.isEmpty then begin
    result := -1;
  end
  else begin
    tempQueue := TQueueStack.create;
    while not currentStates.isEmpty do begin
      state := PTAState(currentStates.dequeue);
      state.links.peekNextReset;  //currentNode = nil
      while not (state.links.getCurrentNode = state.links.getLastNode) do begin
        link := PTALink(state.links.peekNext);
        if (link.src = state) then begin  //following outgoing links only
          case link.linktype of
            LT_EPS: begin
                      //tempQueue.push(Integer(link.dest)); //!TODO! epsilon closure later
                      //if PTAState(link.dest).accepts > 0 then acc := PTAState(link.dest).accepts;
                    end;
            LT_CHAR: if FCaseSensitive then begin
                       if c = char(link.t) then begin
                         tempQueue.pushEx(Integer(link.dest));
//                         if PTAState(link.dest).accepts > acc then acc := PTAState(link.dest).accepts;
                       end;
                     end
                     else begin
                       if AnsiUpperCase(c) = AnsiUpperCase(char(link.t)) then begin
                         tempQueue.pushEx(Integer(link.dest));
//                         if PTAState(link.dest).accepts > acc then acc := PTAState(link.dest).accepts;
                       end;
                     end;
            LT_SET: if not (FSetTable = nil) then begin
                      tempCharSet := FSetTable.getByIndex(link.t);
                      if tempCharSet = nil then begin
                        //!TODO! set index not found - warning (not fatal)
                      end
                      else begin
                        if FCaseSensitive then begin
                          if tempCharSet.hasChar(c) then begin
                            tempQueue.pushEx(Integer(link.dest));
//                            if PTAState(link.dest).accepts > acc then acc := PTAState(link.dest).accepts;
                          end;
                        end
                        else begin
                          if (tempCharSet.hasChar(char(AnsiLowerCase(c)[1]))) or (tempCharSet.hasChar(char(AnsiUpperCase(c)[1]))) then begin
                            tempQueue.pushEx(Integer(link.dest));
//                            if PTAState(link.dest).accepts > acc then acc := PTAState(link.dest).accepts;
                          end;
                        end;
                      end;
                    end
                    else begin
                      //!TODO! set table not defined - warning (not fatal)
                    end;
          end;
        end;
      end;
    end;
    //tempQueue contains the next set of states (after reading character c)
    //epsilon closure of tempQueue nodes
    result := epsilonClosure(tempQueue);      //!TODO! convert to hashset ?

(*    tempQueue.peekNextReset;
    while not (tempQueue.getCurrentNode = tempQueue.getLastNode) do begin
      state := PTAState(tempQueue.peekNext);
      state.links.peekNextReset;  //currentNode = nil
      while not (state.links.getCurrentNode = state.links.getLastNode) do begin
        link := PTALink(state.links.peekNext);
        if (link.src = state) then begin  //following outgoing links only
          if link.linktype = LT_EPS then begin
            tempQueue.pushEx(Integer(link.dest));
          end;
        end;
      end;
    end;*)

{
    //copy tempQueue to currentStates
    while not tempQueue.isEmpty do begin
      state := PTAState(tempQueue.dequeue);
      currentStates.enqueue(Integer(state));
    end;

    //free tempQueue
    tempQueue.Free;
}
    currentStates.Free;
    currentStates := tempQueue;

//    result := acc;
  end;
end;

{TLexer}

/// konstruktor lexikalneho analyzatora postaveneho na zaklade NFA alebo DFA, ktory
/// dostava vstup z FileReadera
constructor TLexer.create(FileReader: TSimpleReader; NFA: TNFA = nil; DFA: TDFA = nil);
begin
  inherited create;
  LFileReader := FileReader;
  LNFA := NFA;
  LDFA := DFA;
  FCaseSense := false;  // nedominantny case sensitive - urcuje ho tym padom konkretny automat
//  FNFA.startSimulation;
end;

/// funkcia vracajuca nasledujuci token (ak pouzivame NFA);
/// tokentype je identifikator tokenu (tak ako nam ho vrati konecny automat);
/// position je pozicia vo vstupe, na ktorej tento token zacina;
/// vysledkom tejto funkcie je string obsahujuci znaky celeho tokenu
function TLexer.getTokenNFA(var tokentype: Integer; var position: Integer): xString;
var c: char;
    accepts: Integer;
    laststartpos, lastendpos: Integer;
    lasttoken: Integer;
    lasttext, tokentext: xString;
    pos: Integer;
begin
  LNFA.startSimulation;
  laststartpos := LFileReader.getPosition;
//  writeln(FilePos(FFileReader.F));
//  writeln(laststartpos);
  lastendpos := 0;
  lasttoken := L_ERROR;
  lasttext := '';
  tokentext := '';
  accepts := 0;
//  tokentype := L_ERROR;
  if not LFileReader.atEOF then
    repeat
      LFileReader.readChar(c);//if FFileReader.readChar(c) then
      begin
        accepts := LNFA.simulate(c);
//        writeln(c + ' - ' + IntToStr(accepts) + ' | ' + LNFA.currentStates.toString);
        if accepts > 0 then begin   //some accepting state
          tokentext := tokentext + c;
          lastendpos := LFileReader.getPosition;
          lasttoken := accepts;
          lasttext := tokentext;
        end
        else if accepts = 0 then begin  //not accepting state, but still working
          tokentext := tokentext + c;
        end
        else begin  //halt
          //if was at EOF then - should work - reset to start state and then exit via above
          LFileReader.unreadChar2;  //halt symbol
          tokentype := lasttoken;
          if (lasttoken <> L_ERROR) then begin
            result := lasttext;
            if LFileReader.getPosition - lastendpos < FILE_READER_BUFFER_SIZE / CHAR_SIZE / 2 then begin
              while LFileReader.getPosition > lastendpos do begin
                LFileReader.unreadChar2;
              end;
            end
            else begin  //character unread limit reached
              LFileReader.unreadChar2;  //error symbol - just before halt symbol
              //!TODO! nothing to do - hmm perhaps warning to Log
            end;
          end
          else begin  //L_ERROR
            result := tokentext;
            pos := LFileReader.getPosition;
            if LFileReader.getPosition - (laststartpos + 1) < FILE_READER_BUFFER_SIZE / CHAR_SIZE / 2 then begin
              while LFileReader.getPosition > (laststartpos + 1) do begin
                LFileReader.unreadChar2;
              end;
            end
            else begin  //character unread limit reached
              LFileReader.unreadChar2;  //error symbol - just before halt symbol
              //!TODO! nothing to do - hmm perhaps warning to Log
            end;
            pos := Length(result) - (pos - LFileReader.getPosition); //pos := length(result) - (number of trailing chars unread) => pos = correct length of result
            SetLength(result,pos);
          end;
          position := LFileReader.getPosition - Length(result);
          exit;  //EXIT FROM THE FUNCTION - automaton halted
        end;
      end
  //    else begin  //EOF
        //assert failed !!! should not reach
  //    end;
    until LFileReader.atEOF;
  if lasttoken <> L_ERROR then begin
    tokentype := lasttoken;
    result := lasttext;
    if accepts = 0 then begin
          LFileReader.unreadChar2;  //halt symbol
          if LFileReader.getPosition - lastendpos < FILE_READER_BUFFER_SIZE / CHAR_SIZE / 2 then begin
            while LFileReader.getPosition > lastendpos do begin
              LFileReader.unreadChar2;
            end;
          end
          else begin  //character unread limit reached
            LFileReader.unreadChar2;  //error symbol - just before halt symbol
            //!TODO! nothing to do - hmm perhaps warning to Log
          end;
    end;
  end
  else begin
    tokentype := lasttoken;
    result := tokentext;

    if LFileReader.getPosition > (laststartpos + 1) then
      LFileReader.unreadChar2;  //halt symbol
    pos := LFileReader.getPosition;
    if LFileReader.getPosition - (laststartpos + 1) < FILE_READER_BUFFER_SIZE / CHAR_SIZE / 2 then begin
      while LFileReader.getPosition > (laststartpos + 1) do begin
        LFileReader.unreadChar2;
      end;
    end
    else begin  //character unread limit reached
      if LFileReader.getPosition > (laststartpos + 1) then
        LFileReader.unreadChar2;  //error symbol - just before halt symbol
      //!TODO! nothing to do - hmm perhaps warning to Log
    end;
    pos := Length(result) - (pos - LFileReader.getPosition); //pos := length(result) - (number of trailing chars unread) => pos = correct length of result
    SetLength(result,pos);


    if result = '' then
      tokentype := L_EOF;
  end;
  position := LFileReader.getPosition - Length(result);
end;

/// funkcia vracajuca nasledujuci token (ak pouzivame DFA);
/// tokentype je identifikator tokenu (tak ako nam ho vrati konecny automat);
/// position je pozicia vo vstupe, na ktorej tento token zacina;
/// vysledkom tejto funkcie je string obsahujuci znaky celeho tokenu
function TLexer.getTokenDFA(var tokentype: Integer; var position: Integer): xString;
var c: char;
    accepts: Integer;
    laststartpos, lastendpos: Integer;
    lasttoken: Integer;
    lasttext, tokentext: xString;
    pos: Integer;
begin
  LDFA.reset;
  laststartpos := LFileReader.getPosition;
  lastendpos := 0;
  lasttoken := L_ERROR;
  lasttext := '';
  tokentext := '';
  accepts := 0;
  if not LFileReader.atEOF then
    repeat
      LFileReader.readChar(c);
      begin
        accepts := LDFA.simulate(c);
//        writeln(c + ' - ' + IntToStr(accepts) + ' | ' + LNFA.currentStates.toString);
        if accepts > 0 then begin   //some accepting state
          tokentext := tokentext + c;
          lastendpos := LFileReader.getPosition;
          lasttoken := accepts;
          lasttext := tokentext;
        end
        else if accepts = 0 then begin  //not accepting state, but still working
          tokentext := tokentext + c;
        end
        else begin  //halt
          //if was at EOF then - should work - reset to start state and then exit via above
          LFileReader.unreadChar2;  //halt symbol
          tokentype := lasttoken;
          if (lasttoken <> L_ERROR) then begin
            result := lasttext;
            if LFileReader.getPosition - lastendpos < FILE_READER_MAX_UNREAD then begin
              while LFileReader.getPosition > lastendpos do begin
                LFileReader.unreadChar2;
              end;
            end
            else begin  //character unread limit reached
//              LFileReader.unreadChar2;  //error symbol - just before halt symbol
              //!TODO! nothing to do - hmm perhaps warning to Log
            end;
          end
          else begin  //L_ERROR
            result := tokentext;
            pos := LFileReader.getPosition;
            if LFileReader.getPosition - (laststartpos + 1) < FILE_READER_MAX_UNREAD then begin
              while LFileReader.getPosition > (laststartpos + 1) do begin
                LFileReader.unreadChar2;
              end;
            end
            else begin  //character unread limit reached
//              LFileReader.unreadChar2;  //error symbol - just before halt symbol
              //!TODO! nothing to do - hmm perhaps warning to Log
            end;
            pos := Length(result) - (pos - LFileReader.getPosition); //pos := length(result) - (number of trailing chars unread) => pos = correct length of result
            SetLength(result,pos);
          end;
          position := LFileReader.getPosition - Length(result);
          exit;  //EXIT FROM THE FUNCTION - automaton halted
        end;
      end
  //    else begin  //EOF
        //assert failed !!! should not reach
  //    end;
    until LFileReader.atEOF;
  if lasttoken <> L_ERROR then begin
    tokentype := lasttoken;
    result := lasttext;
    if accepts = 0 then begin
          LFileReader.unreadChar2;  //halt symbol
          if LFileReader.getPosition - lastendpos < FILE_READER_MAX_UNREAD then begin
            while LFileReader.getPosition > lastendpos do begin
              LFileReader.unreadChar2;
            end;
          end
          else begin  //character unread limit reached
//            LFileReader.unreadChar2;  //error symbol - just before halt symbol
            //!TODO! nothing to do - hmm perhaps warning to Log
          end;
    end;
  end
  else begin
    tokentype := lasttoken;
    result := tokentext;

    if LFileReader.getPosition > (laststartpos + 1) then
      LFileReader.unreadChar2;  //halt symbol
    pos := LFileReader.getPosition;
    if LFileReader.getPosition - (laststartpos + 1) < FILE_READER_MAX_UNREAD then begin
      while LFileReader.getPosition > (laststartpos + 1) do begin
        LFileReader.unreadChar2;
      end;
    end
    else begin  //character unread limit reached
      if LFileReader.getPosition > (laststartpos + 1) then ;
//        LFileReader.unreadChar2;  //error symbol - just before halt symbol
      //!TODO! nothing to do - hmm perhaps warning to Log
    end;
    pos := Length(result) - (pos - LFileReader.getPosition); //pos := length(result) - (number of trailing chars unread) => pos = correct length of result
    SetLength(result,pos);


    if result = '' then
      tokentype := L_EOF;
  end;
  position := LFileReader.getPosition - Length(result);
end;

/// funkcia rychlejsie vracajuca (vyuziva pole znakov namiesto stringu) nasledujuci token (ak pouzivame DFA);
/// tokentype je identifikator tokenu (tak ako nam ho vrati konecny automat);
/// position je pozicia vo vstupe, na ktorej tento token zacina;
/// vysledkom tejto funkcie je string obsahujuci znaky celeho tokenu
function TLexer.getTokenDFA_faster(var tokentype: Integer; var position: Integer): xString;
var c: char;
    accepts: Integer;
    laststartpos, lastendpos: Integer;
    lasttoken: Integer;
    lasttext: xString;
    pos: Integer;
begin
  LDFA.reset;
  laststartpos := LFileReader.getPosition;
  lastendpos := 0;
  lasttoken := L_ERROR;
  clearGCA;
  lasttext := '';
  accepts := 0;
  if not LFileReader.atEOF then
    repeat
      LFileReader.readChar(c);
      begin
        accepts := LDFA.simulate(c);
//        writeln(c + ' - ' + IntToStr(accepts) + ' | ' + LNFA.currentStates.toString);
        if accepts > 0 then begin   //some accepting state
          addtoGCA(c);
          lastendpos := LFileReader.getPosition;
          lasttoken := accepts;
          toStringGCA(lasttext);
        end
        else if accepts = 0 then begin  //not accepting state, but still working
          addtoGCA(c);
        end
        else begin  //halt
          //if was at EOF then - should work - reset to start state and then exit via above
//          LFileReader.unreadChar2;  //halt symbol
          tokentype := lasttoken;
          if (lasttoken <> L_ERROR) then begin
            result := lasttext;
            if LFileReader.getPosition - lastendpos < FILE_READER_MAX_UNREAD then begin
              while LFileReader.getPosition > lastendpos do begin
                LFileReader.unreadChar2;
              end;
            end
            else begin  //character unread limit reached
//              LFileReader.unreadChar2;  //error symbol - just before halt symbol
              //!TODO! nothing to do - hmm perhaps warning to Log
            end;
          end
          else begin  //L_ERROR
            if LFileReader.getPosition > (laststartpos + 1) then
              LFileReader.unreadChar2;
            pos := LFileReader.getPosition;
            if LFileReader.getPosition - (laststartpos + 1) < FILE_READER_MAX_UNREAD then begin
              while LFileReader.getPosition > (laststartpos + 1) do begin
                LFileReader.unreadChar2;
              end;
            end
            else begin  //character unread limit reached
//              LFileReader.unreadChar2;  //error symbol - just before halt symbol
              //!TODO! nothing to do - hmm perhaps warning to Log
            end;
//            result := tokentext;
            pos := lengthGCA - (pos - LFileReader.getPosition); //pos := length(result) - (number of trailing chars unread) => pos = correct length of result
            cuttoGCA(pos);
//            SetLength(result,pos);
            toStringGCA(result);
          end;
          position := LFileReader.getPosition - Length(result);
          exit;  //EXIT FROM THE FUNCTION - automaton halted
        end;
      end
  //    else begin  //EOF
        //assert failed !!! should not reach
  //    end;
    until LFileReader.atEOF;
  if lasttoken <> L_ERROR then begin
    tokentype := lasttoken;
    result := lasttext;
    if accepts = 0 then begin
          LFileReader.unreadChar2;  //halt symbol
          if LFileReader.getPosition - lastendpos < FILE_READER_MAX_UNREAD then begin
            while LFileReader.getPosition > lastendpos do begin
              LFileReader.unreadChar2;
            end;
          end
          else begin  //character unread limit reached
//            LFileReader.unreadChar2;  //error symbol - just before halt symbol
            //!TODO! nothing to do - hmm perhaps warning to Log
          end;
    end;
  end
  else begin
    tokentype := lasttoken;

    if LFileReader.getPosition > (laststartpos + 1) then
      LFileReader.unreadChar2;  //halt symbol
    pos := LFileReader.getPosition;
    if LFileReader.getPosition - (laststartpos + 1) < FILE_READER_MAX_UNREAD then begin
      while LFileReader.getPosition > (laststartpos + 1) do begin
        LFileReader.unreadChar2;
      end;
    end
    else begin  //character unread limit reached
      if LFileReader.getPosition > (laststartpos + 1) then ;
//        LFileReader.unreadChar2;  //error symbol - just before halt symbol
      //!TODO! nothing to do - hmm perhaps warning to Log
    end;
//    result := tokentext;
    pos := lengthGCA - (pos - LFileReader.getPosition); //pos := length(result) - (number of trailing chars unread) => pos = correct length of result
//    SetLength(result,pos);
    cuttoGCA(pos);
    toStringGCA(result);


    if result = '' then
      tokentype := L_EOF;
  end;
  position := LFileReader.getPosition - Length(result);
end;

/// funkcia vracajuca nasledujuci token v naplnenom recorde TToken
function TLexer.getTToken: TToken;
var pt: PToken;
begin
  pt := getToken;
  result := pt^;
end;

/// funkcia vracajuca pointer na naplneny record TToken, ktory obsahuje nasledujuci token
function TLexer.getToken: PToken;
var len: Integer;
    bCaseSensitive: Boolean;
begin
  new(result);
  if LDFA <> nil then begin
    result.value := getTokenDFA_faster(result.tokentype,result.startp);
    bCaseSensitive := LDFA.getCaseSensitivity;
  end
  else if LNFA <> nil then begin
    result.value := getTokenNFA(result.tokentype,result.startp);
    bCaseSensitive := LNFA.getCaseSensitivity;
  end
  else begin
    result.value := '';
    result.tokentype := L_EOF;
    result.startp := 0;
    bCaseSensitive := true;  //pri '' stringu je to jedno
  end;

  if CASE_CONVERSION then begin
    if (not FCaseSense) and (not bCaseSensitive) then begin  // !UNICODE! toto je len pre ASCII
      result.value := AnsiLowerCase(result.value);
    end;
  end;
  
  len := Length(result.value);
//  if len > 0 then
    result.endp := result.startp + len - 1;
//  else
//    result.endp := result.startp;
  result.child := nil;
  result.sibling := nil;
end;

{TGrammarFileLexer}

/// konstruktor vytvarajuci analyzator vstupnej gramatiky
constructor TGrammarFileLexer.create(FileReader: TSimpleReader);
var chset,chset2: TCharSet;
    i: Integer;
begin
  inherited create;
  FFileReader := FileReader;

  FPropertyTable := TPropertyTable.create;

  FSetTable := TSetTable.create;
  chset := TCharSet.create('Whitespace');
  chset.addString(SET_WS);
  FSetTable.add(chset);
  FWhitespaceSet := TCharSet.create('Whitespace');
//  FWhitespaceSet.addCharSet(chset);
  FWhitespaceSet.addString(SET_INLINEWS);
  chset := TCharSet.create('Op');
  chset.addString(SET_OPS);
  FSetTable.add(chset);
  FOpSet := TCharSet.create('Op');
  FOpSet.addCharSet(chset);
  chset2 := TCharSet.create('Alphanumeric');
  chset := TCharSet.create('SmallLetter');
  chset.addString(SET_Alpha);
  chset2.addCharSet(chset);
  FSetTable.add(chset);
  chset := TCharSet.create('CapitalLetter');
  chset.addString(SET_AlphaCaps);
  chset2.addCharSet(chset);
  FSetTable.add(chset);
  chset := TCharSet.create('Letter');
  chset.addCharSet(chset2);
  FSetTable.add(chset);
  chset := TCharSet.create('Digit');
  chset.addString(SET_Number);
  FDigitSet := TCharSet.create('Op');
  FDigitSet.addCharSet(chset);
  chset2.addCharSet(chset);
  FSetTable.add(chset);
  FSetTable.add(chset2);
  FAlphanumericSet := TCharSet.create('Alphanumeric');
  FAlphanumericSet.addCharSet(chset2);
  chset := TCharSet.create('RegExpOp');
  chset.addString(SET_RegExpOps);
  FSetTable.add(chset);
  FRegExpOpSet := TCharSet.create('RegExpOp');
  FRegExpOpSet.addCharSet(chset);

  chset := TCharSet.create('Printable');
  for i := 0 to 31 do begin
    chset.add(char(i));
  end;
  chset.add(char(9));   //TAB
  chset.invert;
  FSetTable.add(chset);

  chset := TCharSet.create('EOLN');
  chset.add(char(#10));
  FSetTable.add(chset);

  chset := TCharSet.create('HexDigit');
  chset.addCharSet(FDigitSet);
  chset.addString('ABCDEFabcdef');
  FHexDigitSet := chset;

  chset := TCharSet.create('SLS');
  chset.addString(SET_SPECIALLINESSYMBOLS);
  FSLSSet := chset;

  FLog := TLog.create;

  FTokenNameTypeTable := TStrIntHashMap.create;
  FStateNameTypeTable := TStrIntHashMap.create;

  FRuleTable := TRuleTable.create;

  FIndTable := nil;
  FHLTable := nil;
end;

/// destruktor uvolnujuci tento objekt z pamate
destructor TGrammarFileLexer.Destroy;
var pdata: PTokenData;
    phighlight: PTokenHighlight;
    k: Integer;
begin
  FRuleTable.Free;
  FLexerNFA.free;

  FPropertyTable.free;
  FLog.free;
  FSetTable.free;
  FTokenNameTypeTable.free;
  FStateNameTypeTable.free;

  FWhiteSpaceSet.Free;
  FOpSet.Free;
  FDigitSet.Free;
  FAlphanumericSet.Free;
  FRegExpOpSet.Free;
  FHexDigitSet.Free;
  FSLSSet.Free;

  if FIndTable <> nil then begin
    FIndTable.getNextReset;
    while FIndTable.getNext(k,Integer(pdata)) do begin
      if pdata.bexcept <> nil then
        pdata.bexcept.Free;
      if pdata.aexcept <> nil then
        pdata.aexcept.Free;
      dispose(pdata);
    end;
    FIndTable.Free;
  end;

  if FHLTable <> nil then begin
    FHLTable.getNextReset;
    while FHLTable.getNext(k,Integer(phighlight)) do begin
      dispose(phighlight);
    end;
    FHLTable.Free;
  end;

  inherited Destroy;
end;

/// funkcia vracajuca nasleduci token (type TokenType) zo suboru so vstupnou gramatikou
function TGrammarFileLexer.getNextToken_old(var TokenType: Integer): xString;
var c,c2: char;
    S: xString;
    state: Integer;
begin
  result := '';
  S := '';
  state := 0;
  TokenType := GT_ERROR;
  while not FFileReader.atEOF do begin

    FFileReader.readChar(c);
    Fcol := Fcol + 1;
    if c = SET_EOLWS then begin
      if (state > 0) and (state < 99) then begin
        FFileReader.unreadChar2;
        Fcol := Fcol - 1;
        case state of
          1: S:= '"';
          2: S:= '}';
          3: S:= 'AlphaNumeric';
          4: S:= '>';
          5: S:= ':';
          6: S:= '=';
          7: S:= ']';
          20: S:= '''';
        end;
//        FLog.add(Fline,Fcol,FFileReader.getPosition,S,-2);
        FLog.addSFR2(FFileReader,0,E_GF_EXPECTED_1_INSTEAD_OF_EOL,S);
        exit;
      end
      else if state = 99 then begin
        FFileReader.unreadChar2;
        Fcol := Fcol - 1;
        TokenType := GT_COMMENT;
        exit;
      end
      else if state = 100 then begin
        FFileReader.unreadChar2;
        Fcol := Fcol - 1;
        TokenType := GT_TERMINAL;
        exit;
      end
      else begin
        Fcol := 0;
        Fline := Fline + 1;
        TokenType := GT_EOL;
        result := SET_EOLWS;
        exit;
      end;
    end
    else case state of
      0:if not FWhitespaceSet.hasChar(c) then begin
          case c of
            '"': state := 1;
            '{': state := 2;
            '<': state := 4;
            ':': begin state := 5; result := result + c; end;
            '[': state := 7;
            '''':state := 20;
            '!': state := 99;
            else begin
              if FAlphanumericSet.hasChar(c) then begin
                state := 100;
                result := result + c;
              end
              else if FOpSet.hasChar(c) then begin
                TokenType := GT_OP;
                result := c;
                exit;
              end
              else begin
//                FLog.add(Fline,min(Fcol-1,0),FFileReader.getPosition,c,-1);
                FLog.addSFR2(FFileReader,-1,E_GF_UNEXPECTED_CHAR_FOUND_1,c);
                exit;
              end;
            end;
          end;
        end;
      1:if c <> '"' then begin
          if (c <> '\') or FFileReader.atEOF then begin            //ESCAPE CHARACTER
            result := result + c;
          end
          else begin
            FFileReader.readChar(c2);
            if c2 <> '"' then begin
              result := result + c;
              FFileReader.unreadChar2;
            end
            else begin
              result := result + c2;
              Fcol := Fcol + 1;
            end;
          end;
        end
        else begin
          TokenType := GT_PROPERTY;
          exit;
        end;
      2:if c <> '}' then begin
          if (c <> '\') or FFileReader.atEOF then begin            //ESCAPE CHARACTER
            result := result + c;
          end
          else begin
            FFileReader.readChar(c2);
            if c2 <> '}' then begin
              result := result + c;
              FFileReader.unreadChar2;
            end
            else begin
              result := result + c2;
              Fcol := Fcol + 1;
            end;
          end;
        end
        else begin
          TokenType := GT_SET;
          exit;
        end;
      4:if c <> '>' then begin
          if (c <> '\') or FFileReader.atEOF then begin            //ESCAPE CHARACTER
            result := result + c;
          end
          else begin
            FFileReader.readChar(c2);
            if c2 <> '>' then begin
              result := result + c;
              FFileReader.unreadChar2;
            end
            else begin
              result := result + c2;
              Fcol := Fcol + 1;
            end;
          end;
        end
        else begin
          TokenType := GT_STATE;
          exit;
        end;
      5:if c = ':' then begin
          result := result + c;
          state := 6;
        end
        else begin
//          FLog.add(Fline,max(Fcol-1,0),FFileReader.getPosition,':',-3);
          FLog.addSFR2(FFileReader,-1,E_GF_EXPECTED_1,':');
          exit;
        end;
      6:if c = '=' then begin
          result := result + c;
          TokenType := GT_ASSIGN;
          exit;
        end
        else begin
//          FLog.add(Fline,max(Fcol-2,0),FFileReader.getPosition,'=',-3);
          FLog.addSFR2(FFileReader,-2,E_GF_EXPECTED_1,':');
          exit;
        end;
        7:if c <> ']' then begin
          if (c <> '\') or FFileReader.atEOF then begin            //ESCAPE CHARACTER
            result := result + c;
          end
          else begin
            FFileReader.readChar(c2);
            if c2 <> ']' then begin
              result := result + c;
              FFileReader.unreadChar2;
            end
            else begin
              result := result + c2;
              Fcol := Fcol + 1;
            end;
          end;
        end
        else begin
          TokenType := GT_SETSTRING;
          exit;
        end;
      20:if c <> '''' then begin
          if (c <> '\') or FFileReader.atEOF then begin            //ESCAPE CHARACTER
            result := result + c;
          end
          else begin
            FFileReader.readChar(c2);
            if c2 <> '''' then begin
              result := result + c;
              FFileReader.unreadChar2;
            end
            else begin
              result := result + c2;
              Fcol := Fcol + 1;
            end;
          end;
        end
        else begin
          TokenType := GT_STRING;
          exit;
        end;
      99:result := result + c;
      100:if not FAlphanumericSet.hasChar(c) then begin
          FFileReader.unreadChar2;
          Fcol := Fcol - 1;
          TokenType := GT_TERMINAL;
          exit;
        end
        else begin
          result := result + c;
        end;
    end;

  end;

      if (state > 0) and (state < 99) then begin
        case state of
          1: S:= '"';
          2: S:= '}';
          3: S:= 'AlphaNumeric';
          4: S:= '>';
          5: S:= ':';
          6: S:= '=';
          7: S:= ']';
          20: S:= '''';
        end;
//        FLog.add(Fline,Fcol,FFileReader.getPosition,S,-4);
        FLog.addSFR2(FFileReader,0,E_GF_EXPECTED_1_INSTEAD_OF_EOF,S);
//        exit;
      end
      else if state = 99 then begin
        TokenType := GT_COMMENT;
//        exit;
      end
      else if state = 100 then begin
        TokenType := GT_TERMINAL;
//        exit;
      end
      else begin
        TokenType := GT_EOF;
      end;

end;

/// funkcia vracajuca nasleduci token (type TokenType) zo suboru so vstupnou gramatikou;
/// kvoli zrychleniu pouziva na zretazovanie retazcov pole znakov definovane v constants.pas
function TGrammarFileLexer.getNextToken(var TokenType: Integer): xString;
var c,c2: char;
    S: xString;
    state: Integer;
begin
  result := '';
  S := '';
  state := 0;
  TokenType := GT_ERROR;
  while not FFileReader.atEOF do begin

    FFileReader.readChar(c);
    if c = SET_EOLWS then begin
      if (state > 0) and (state < 99) then begin
        FFileReader.unreadChar2;
        case state of
          1: S:= '"';
          2: S:= '}';
          3: S:= 'AlphaNumeric';
          4: S:= '>';
          5: S:= ':';
          6: S:= '=';
          7: S:= ']';
          20: S:= '''';
        end;
        FLog.addSFR2(FFileReader,0,E_GF_EXPECTED_1_INSTEAD_OF_EOL,S);
        exit;
      end
      else if state = 99 then begin
        FFileReader.unreadChar2;
        TokenType := GT_COMMENT;
        exit;
      end
      else if state = 100 then begin
        FFileReader.unreadChar2;
        TokenType := GT_TERMINAL;
        exit;
      end
      else if state = 101 then begin
        FFileReader.unreadChar2;
        TokenType := GT_OP;
        exit;
      end
      else if state = 102 then begin
        FFileReader.unreadChar2;
        FLog.addSFR2(FFileReader,0,E_GF_EXPECTED_1_INSTEAD_OF_EOL,'HexDecimals');
        TokenType := GT_ERROR;
      end
      else begin
        TokenType := GT_EOL;
        result := SET_EOLWS;
        exit;
      end;
    end
    else case state of
      0:if not FWhitespaceSet.hasChar(c) then begin
          case c of
            '"': state := 1;
            '{': state := 2;
            '<': state := 4;
            ':': begin state := 5; result := result + c; end;
            '[': state := 7;
            '''':state := 20;
            '!': state := 99;
            '^': begin TokenType := GT_EQ; result := c; Exit; end;
            '~': begin TokenType := GT_SIM; result := c; Exit; end;
            '&': begin TokenType := GT_IND; result := c; Exit; end;
            '%': begin TokenType := GT_HL; result := c; Exit; end;
            else begin
              if FAlphanumericSet.hasChar(c) then begin
                state := 100;
                result := result + c;
              end
              else if FOpSet.hasChar(c) then begin
                TokenType := GT_OP;
                result := c;
                if c <> '-' then
                  Exit
                else begin
                  state := 101;
                end;
              end
              else if c = '$' then begin
                result := c;
                state := 102;
              end
              else begin
                FLog.addSFR2(FFileReader,-1,E_GF_UNEXPECTED_CHAR_FOUND_1,c);
                exit;
              end;
            end;
          end;
        end;
      1:if c <> '"' then begin
          result := result + c;
        end
        else begin
          TokenType := GT_PROPERTY;
          exit;
        end;
      2:if c <> '}' then begin
          result := result + c;
        end
        else begin
          TokenType := GT_SET;
          exit;
        end;
      4:if c <> '>' then begin
          result := result + c;
        end
        else begin
          TokenType := GT_STATE;
          exit;
        end;
      5:if c = ':' then begin
          result := result + c;
          state := 6;
        end
        else begin
          FLog.addSFR2(FFileReader,-1,E_GF_EXPECTED_1,':');
          exit;
        end;
      6:if c = '=' then begin
          result := result + c;
          TokenType := GT_ASSIGN;
          exit;
        end
        else begin
          FLog.addSFR2(FFileReader,-2,E_GF_EXPECTED_1,'=');
          exit;
        end;
       7:if c <> ']' then begin
            result := result + c;
          end
          else begin
            TokenType := GT_SETSTRING;
            exit;
          end;
      20:if c <> '''' then begin
           result := result + c;
         end
         else begin
           TokenType := GT_STRING;
           exit;
         end;
      99:result := result + c;
     100:if not FAlphanumericSet.hasChar(c) then begin
           FFileReader.unreadChar2;
           TokenType := GT_TERMINAL;
           exit;
         end
         else begin
           result := result + c;
         end;
     101:if c = '$' then begin
           result := result + c;
           state := 102;
         end
         else if not FAlphanumericSet.hasChar(c) then begin
           FFileReader.unreadChar2;
           TokenType := GT_OP;
           exit;
         end
         else begin
           result := result + c;
           state := 100;
         end;
     102:if not FAlphanumericSet.hasChar(c) then begin
           FLog.addSFR2(FFileReader,-1,E_GF_EXPECTED_1,'HexDecimals');
           FFileReader.unreadChar2;
           TokenType := GT_ERROR;
           exit;
         end
         else begin
           result := result + c;
           state := 100;
         end;
    end;

  end;

  if (state > 0) and (state < 99) then begin
    case state of
      1: S:= '"';
      2: S:= '}';
      3: S:= 'AlphaNumeric';
      4: S:= '>';
      5: S:= ':';
      6: S:= '=';
      7: S:= ']';
      20: S:= '''';
    end;
    FLog.addSFR2(FFileReader,0,E_GF_EXPECTED_1_INSTEAD_OF_EOF,S);
  end
  else if state = 99 then begin
    TokenType := GT_COMMENT;
  end
  else if state = 100 then begin
    TokenType := GT_TERMINAL;
  end
  else if state = 101 then begin
    TokenType := GT_OP;
  end
  else if state = 102 then begin
    FLog.addSFR2(FFileReader,0,E_GF_EXPECTED_1_INSTEAD_OF_EOF,'HexDecimals');
    TokenType := GT_ERROR;
  end
  else begin
    TokenType := GT_EOF;
  end;
end;

/// funkcia, ktora transformuje vstupny string s escape sekvenciami na vystupny string kde su nahradene pozadovanymi znakmi
function TGrammarFileLexer.resolveEscSeq(S: xString): xString;   //\{Digit}+ -> {Digit}+, \' -> ', \" -> ", \n -> EOL, \t -> TAB, \\ -> \, \{any}* -> \{any}*
var i: Integer;
    j: Integer;
    b: Boolean;
    num: xString;
begin
  b := false;
  result := '';
  for i := 1 to Length(S) do begin
    if b then begin
      if FDigitSet.hasChar(S[i]) then begin
        num := num + S[i];
      end
      else begin
        if num = '' then begin
          if S[i] = '''' then
            result := result + ''''
          else if S[i] = '"' then
            result := result + '"'
          else if S[i] = '\' then
            result := result + '\'
          else if S[i] = '\n' then
            result := result + EOL
          else if S[i] = '\t' then
            result := result + TAB
          else if S[i] = '\' then
            result := result + '\'
          else
            result := result + '\' + S[i];
          b := false;
        end
        else begin
          j := StrToInt(num);
          if (j >= 0) and (j <= CHAR_HIGH) then
            result := result + char(j)
          else
            result := result + '\' + num;
          num := '';
          if S[i] = '\' then
            b := true
          else begin
            b := false;
            result := result + S[i];
          end;
        end;
      end;
    end
    else begin
      if S[i] = '\' then
        b := true
      else
        result := result + S[i];
    end;
  end;
  if b then begin
    if num <> '' then begin
      j := StrToInt(num);
      if (j >= 0) and (j <= CHAR_HIGH) then
        result := result + char(j)
      else
        result := result + '\' + num;
    end
    else begin
      result := result + '\';
    end;
  end;
end;

/// funkcia vracajuca ci dany vstupny string S je cislo v dekadickom alebo hexadecimalnom zapise
function TGrammarFileLexer.isNumber(S: xString): Boolean;
var i: Integer;
    start: Integer;
    bHex: Boolean;
    charset: TCharSet;
begin
  result := false;
  if Length(S) < 1 then
    Exit;
  start := 1;
  bHex := false;
  if S[start] = '-' then begin
    start := start + 1;
    if (Length(S) > 1) and (S[start] = '$') then begin
      bHex := true;
      start := start + 1;
    end;
  end
  else if S[start] = '$' then begin
    start := start + 1;
    bHex := true;
    if (Length(S) > 1) and (S[start] = '-') then begin
      start := start + 1;
    end;
  end;
  if bHex then
    charset := FHexDigitSet
  else
    charset := FDigitSet;
  for i := start to Length(S) do begin
    if not charset.hasChar(S[i]) then begin
      Exit;
    end
  end;
  result := true;
end;

/// funkcia ktora zanalyzuje vstupny subor s gramatikou a naplni vsetky relevantne tabulky:
/// tabulku vlastnosti, tabulku znakovych mnozin, tabulku terminalov, neterminalov, tabulku pravidiel,
/// mnoziny formatovacich a highlightovacich pravidiel, mnoziny tried ekvivalencie a podobnosti pre tokeny
/// a vytvori NFA podla definicie tokenov (z regularnych vyrazov)
function TGrammarFileLexer.parse: boolean;
var token: Integer;
    value: xString;
    l,c,p: Integer;
    t1,t2: Integer;
    v1,v2: xString;
    NTidmax: Integer;
    Tidmax: Integer;
    tpos: Integer;
    id: Integer;

  function parseProperty: boolean;
  begin
    result := false;
    v1 := getNextToken(t1);
    if (t1 <> GT_OP) and (v1 <> '=') then begin
//      p := FFileReader.getFullPosition(l,c);
//      FLog.add(l,c,p,'Property declaration parsing failed: ' + v1 + ' found - ' + '= expected',-11);
      FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_PROPERTY_EXPECTED_EQ_INSTEAD_OF_1,v1);
      token := t1; value := v1;
      exit;
    end;
    v1 := getNextToken(t1);
    if (t1 <> GT_STRING) and (t1 <> GT_STATE) and (t1 <> GT_TERMINAL) then begin
//      p := FFileReader.getFullPosition(l,c);
//      FLog.add(l,c,p,'Property declaration parsing failed: ' + v1 + ' found - ' + '''...'' expected',-12);
      FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_PROPERTY_EXPECTED_STRING_OR_STATE_INSTEAD_OF_1,v1);
      token := t1; value := v1;
      exit;
    end;
    FPropertyTable.add(value,resolveEscSeq(v1));
    result := true;
  end;

  function parseSet: boolean;
  var chset: TCharSet;
      tempCharSet: TCharSet;
      pos: Integer;
  begin
    result := false;
    pos := FFileReader.getPosition - Length(value) - GT_LENGTH(token);
    v1 := getNextToken(t1);
    if (t1 <> GT_OP) or (v1 <> '=') then begin
//      p := FFileReader.getChangedFullPosition(l,c,-Length(v1) - GT_LENGTH(t1));
//      FLog.add(l,c,p,'Set declaration parsing failed: ' + v1 + ' found - ' + '= expected',-21);
      FLog.addSFR2(FFileReader,-Length(v1) - GT_LENGTH(t1),E_GF_SET_EXPECTED_EQ_INSTEAD_OF_1,v1);
      token := t1; value := v1;
      exit;
    end;

    if FSetTable.getByName(value) <> nil then begin
      FLog.addSFR2(FFileReader,pos - FFileReader.getPosition,W_GF_SET_REDEFINED_1,value);
    end;

    v1 := getNextToken(t1);
    if (t1 <> GT_STRING) and (t1 <> GT_SET) and (t1 <> GT_SETSTRING) then begin
//      p := FFileReader.getChangedFullPosition(l,c,-Length(v1) - GT_LENGTH(t1));
//      FLog.add(l,c,p,'Set declaration parsing failed: ' + v1 + ' found - ' + '''...'' or {...} or [...] expected',-22);
      FLog.addSFR2(FFileReader,-Length(v1) - GT_LENGTH(t1),E_GF_SET_EXPECTED_STRING_OR_STATE_OR_SETSTRING_INSTEAD_OF_1,v1);
      token := t1; value := v1;
      exit;
    end;
    v2 := '+';
    chset := TCharSet.create(value);
    repeat
      if t1 = GT_STRING then begin
        if v2 = '+' then
          chset.addString(resolveEscSeq(v1))
        else
          chset.subString(resolveEscSeq(v1));
      end
      else if t1 = GT_SET then begin
        tempCharSet := FSetTable.getByName(v1);
        if tempCharSet = nil then begin
//          p := FFileReader.getChangedFullPosition(l,c,-Length(v1) - GT_LENGTH(t1));
//          FLog.add(l,c,p,'Set declaration parsing: set {' + v1 + '} not found (declare before use)',-1023);
          FLog.addSFR2(FFileReader,-Length(v1) - GT_LENGTH(t1),W_GF_SET_SET_1_NOT_DEFINED,v1);
        end;
        if v2 = '+' then
          chset.addCharSet(tempCharSet)
        else
          chset.subCharSet(tempCharSet);
      end
      else if t1 = GT_SETSTRING then begin
        if v2 = '+' then
          chset.addString(v1)
        else
          chset.subString(v1);
      end
      else if t1 = GT_OP then begin
        v2 := v1;
      end;
      v1 := getNextToken(t1);
    until not ((t1 = GT_STRING) or (t1 = GT_SET) or (t1 = GT_SETSTRING) or ((t1 = GT_OP) and ((v1 = '+') or (v1 = '-'))));
    FSetTable.add(chset);
    token := t1; value := v1;
    result := false;
  end;

  function parseComment: boolean;
  begin
    result := true; //not much to do
  end;

  function parseRegExp: boolean;
  var chset: TCharSet;
      NFA: TNFA;
      ParentStack: TQueueStack;
      LeafParents: TQueueStack;
      Leafs: TQueueStack;
      TerminalLink: PTALink;
      RightBracketState: PTAState;
      state,state0,state01,state1,state2,state3,state4,stateTemp1,stateTemp2: PTAState;
      i: Integer;
      prevToken: Integer;
      prevValue: xString;
      b: Boolean;
  begin
    result := false;
    NFA := TNFA.create;
    state := NFA.addStateEx(0);
    ParentStack := TQueueStack.create;
    ParentStack.addFirst(Integer(state));
    LeafParents := TQueueStack.create;
    LeafParents.addFirst(Integer(state));
    Leafs := TQueueStack.create;
    Leafs.addFirst(Integer(state));
    TerminalLink := nil;
    RightBracketState := nil;

    prevToken := t1;
    prevValue := v1;
    v1 := getNextToken(t1);
    while not ((t1 <> GT_OP) and (t1 <> GT_SET) and (t1 <> GT_SETSTRING) and (t1 <> GT_STRING) and (t1 <> GT_TERMINAL)) do begin
  (*    if (t1 <> GT_OP) and (t1 <> GT_SET) and (t1 <> GT_SETSTRING) and (t1 <> GT_STRING) and (t1 <> GT_TERMINAL) then begin
        p := FFileReader.getChangedFullPosition(l,c,-Length(v1) - GT_LENGTH(t1));
        FLog.add(l,c,p,'RegExp parsing failed: ' + v1 + ' found - [...] or {...} or ''...'' or (...) or +*? expected',-62);
        token := t1; value := v1;
        exit;
      end
      else*)
//      if t1 = GT_TERMINAL then begin
//        p := FFileReader.getChangedFullPosition(l,c,-Length(v1) - GT_LENGTH(t1));
//        FLog.add(l,c,p,'RegExp parsing failed: terminal ' + v1 + ' found, not supported',-63);
//        break;
//  //      token := t1; value := v1;
//  //      exit;
//      end
      if t1 = GT_SET then begin
        //S->SET->Accept
        i := FSetTable.findIndexByName(v1);
        if i = -1 then begin
//          p := FFileReader.getChangedFullPosition(l,c,-Length(v1) - GT_LENGTH(t1));
//          FLog.add(l,c,p,'RegExp parsing: set {' + v1 + '} not defined',-1067);
          FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),W_GF_REGEXP_SET_1_NOT_DEFINED_CREATING_DUMMY_2,v1,DUMMY_SET_NAME_PREFIX + v2);

          i := FSetTable.add(TCharSet.create(DUMMY_SET_NAME_PREFIX + v1)); //creating dummy charset with the missing name
        end;

        state0 := NFA.addStateEx(0);
        state01 := NFA.addStateEx(0);  //after set link state
        NFA.addLinksEx(state0,state01,i,LT_SET);


        if not (TerminalLink = nil) then begin
          dispose(TerminalLink);
          TerminalLink := NFA.buildLinkEx(state01,state01,i,LT_SET);
        end
        else begin
          TerminalLink := NFA.buildLinkEx(state01,state01,i,LT_SET);
        end;

        stateTemp1 := PTAState(ParentStack.peek);
        Leafs.peekNextReset;
        LeafParents.peekNextReset;
        while not (LeafParents.getCurrentNode = LeafParents.getLastNode) do begin
          state1 := PTAState(Leafs.peekNext);
          state2 := PTAState(LeafParents.peekNext);
          if state2 = stateTemp1 then begin
            NFA.addLinksEx(state1,state0,0,LT_EPS);
            Leafs.removeCurrentNode;
            LeafParents.removeCurrentNode;
          end
          else
            break;
        end;
        Leafs.addFirst(Integer(state01));
        LeafParents.addFirst(Integer(stateTemp1));

        RightBracketState := state0;

        if (prevToken = GT_OP) and (prevValue = '|') then begin
          NFA.addLinksEx(stateTemp1,state0,0,LT_EPS);
        end;
//        writeln('SET- ' + NFA.toString);
      end
      else if t1 = GT_SETSTRING then begin
        //S->SET breaked into multiple |->Accept
//        p := FFileReader.getChangedFullPosition(l,c,-Length(v1) - GT_LENGTH(t1));
//        FLog.add(l,c,p,'RegExp parsing failed: set definition string [' + v1 + '] found, not supported (yet)',-66);
//        break;

        chset := TCharSet.create(DUMMY_SET_STRING_NAME_PREFIX + v1);
        chset.addString(v1);
        i := FSetTable.add(chset); //creating dummy charset and filling it with set definition string

        //the SAME as above in GT_SET
        state0 := NFA.addStateEx(0);
        state01 := NFA.addStateEx(0);  //after set link state
        NFA.addLinksEx(state0,state01,i,LT_SET);


        if not (TerminalLink = nil) then begin
          dispose(TerminalLink);
          TerminalLink := NFA.buildLinkEx(state01,state01,i,LT_SET);
        end
        else begin
          TerminalLink := NFA.buildLinkEx(state01,state01,i,LT_SET);
        end;

        stateTemp1 := PTAState(ParentStack.peek);
        Leafs.peekNextReset;
        LeafParents.peekNextReset;
        while not (LeafParents.getCurrentNode = LeafParents.getLastNode) do begin
          state1 := PTAState(Leafs.peekNext);
          state2 := PTAState(LeafParents.peekNext);
          if state2 = stateTemp1 then begin
            NFA.addLinksEx(state1,state0,0,LT_EPS);
            Leafs.removeCurrentNode;
            LeafParents.removeCurrentNode;
          end
          else
            break;
        end;
        Leafs.addFirst(Integer(state01));
        LeafParents.addFirst(Integer(stateTemp1));

        RightBracketState := state0;

        if (prevToken = GT_OP) and (prevValue = '|') then begin
          NFA.addLinksEx(stateTemp1,state0,0,LT_EPS);
        end;
      end
      else if (t1 = GT_STRING) or (t1 = GT_TERMINAL) then begin
        //S->string states->Accept
        state0 := NFA.addStateEx(0);  //epsilon transitions target state
        if v1 = '' then begin
//          p := FFileReader.getChangedFullPosition(l,c,-Length(v1[1]) - GT_LENGTH(t1));
//          FLog.add(l,c,p,'RegExp parsing: empty string in token definition',-1065);
          FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),W_GF_REGEXP_EMPTY_STRING_IN_TOKEN_1_DEFINITION,value);
          state3 := state0;
          if not (TerminalLink = nil) then begin
            dispose(TerminalLink);
            TerminalLink := NFA.buildLinkEx(state3,state0,0,LT_EPS);
          end
          else begin
            TerminalLink := NFA.buildLinkEx(state3,state0,0,LT_EPS);
          end;
        end
        else begin
          if t1 = GT_STRING then
            v2 := resolveEscSeq(v1)
          else
            v2 := v1;
          state01 := NFA.addStateEx(0);  //after first character state
          NFA.addLinksEx(state0,state01,Integer(v2[1]),LT_CHAR);
          state3 := state01;         //after last character state
          for i := 2 to Length(v2) do begin
            state4 := NFA.addStateEx(0);
            NFA.addLinksEx(state3,state4,Integer(v2[i]),LT_CHAR);
            state3 := state4;
          end;

          if not (TerminalLink = nil) then begin
            dispose(TerminalLink);
            TerminalLink := NFA.buildLinkEx(state3,state01,Integer(v2[1]),LT_CHAR);
          end
          else begin
            TerminalLink := NFA.buildLinkEx(state3,state01,Integer(v2[1]),LT_CHAR);
          end;
        end;

        stateTemp1 := PTAState(ParentStack.peek);
        Leafs.peekNextReset;
        LeafParents.peekNextReset;
        while not (LeafParents.getCurrentNode = LeafParents.getLastNode) do begin
          state1 := PTAState(Leafs.peekNext);
          state2 := PTAState(LeafParents.peekNext);
          if state2 = stateTemp1 then begin
            NFA.addLinksEx(state1,state0,0,LT_EPS);
            Leafs.removeCurrentNode;
            LeafParents.removeCurrentNode;
          end
          else
            break;
        end;

        Leafs.addFirst(Integer(state3));
        LeafParents.addFirst(Integer(stateTemp1));

        RightBracketState := state0;

        if (prevToken = GT_OP) and (prevValue = '|') then begin
          NFA.addLinksEx(stateTemp1,state0,0,LT_EPS);
        end;
      end
      else if not (FRegExpOpSet.hasChar(v1[1])) then begin
//        p := FFileReader.getChangedFullPosition(l,c,-Length(v1[1]) - GT_LENGTH(t1));
//        FLog.add(l,c,p,'RegExp parsing failed: operator ' + v1[1] + ' found - ()+*?| expected',-64);
        FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_REGEXP_INVALID_OPERATOR_1_FOUND_REGEXPOP_EXPECTED,v1);
        break;
      end
      else begin  //()+*?|
        if v1[1] = '(' then begin
          //pop all leaves (and leaves parent's), whose parent is on top of the parentStack
          //and put epsilon links from them to a new state
          //put the new state (and its parent) into respective stacks and itself into the parentStack
          stateTemp1 := PTAState(ParentStack.peek);
          state0 := NFA.addStateEx(0);
          Leafs.peekNextReset;
          LeafParents.peekNextReset;
          while not (LeafParents.getCurrentNode = LeafParents.getLastNode) do begin
            state1 := PTAState(Leafs.peekNext);
            state2 := PTAState(LeafParents.peekNext);
            if state2 = stateTemp1 then begin
              NFA.addLinksEx(state1,state0,0,LT_EPS);
              Leafs.removeCurrentNode;
              LeafParents.removeCurrentNode;
            end
            else
              break;
          end;
          Leafs.addFirst(Integer(state0));
          LeafParents.addFirst(Integer(state0));

          ParentStack.addFirst(Integer(state0));

          if not (TerminalLink = nil) then begin
            dispose(TerminalLink);
            TerminalLink := nil;
          end;
          RightBracketState := nil;

          if (prevToken = GT_OP) and (prevValue = '|') then begin
            NFA.addLinksEx(stateTemp1,state0,0,LT_EPS);
          end;
        end
        else if v1[1] = ')' then begin
          //change leafparents for leaves which have parentStack.peek as parent to parentStack.peek(2) - if nil then bracketing ERROR
          //pop state from parentStack put it into RightBracketState
          stateTemp1 := PTAState(ParentStack.peek);
          stateTemp2 := PTAState(ParentStack.peek(2));
          if Integer(stateTemp2) = -1 then
            stateTemp2 := nil;
          if stateTemp2 = nil then begin
//            FLog.add(l,c,p,'RegExp parsing failed: ) found but corresponding ( is missing',-65);
            FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_REGEXP_TOO_MANY_CLOSING_BRACKETS_1_CORRESPONDING_OB_MISSING,v1);
            break;
          end;
          LeafParents.peekNextReset;
          while not (LeafParents.getCurrentNode = LeafParents.getLastNode) do begin
            //state1 := PTAState(Leafs.peekNext);
            state2 := PTAState(LeafParents.peekNext);
            if state2 = stateTemp1 then begin
              LeafParents.getCurrentNode.data := Integer(stateTemp2);
            end
            else
              break;
          end;

          if not (TerminalLink = nil) then begin
            dispose(TerminalLink);
            TerminalLink := nil;
          end;
          RightBracketState := PTAState(ParentStack.dequeue);
        end
        else if v1[1] = '|' then begin
          //change leafparents for leaves which have parentStack.peek as parent to parentStack.peek(2) - if nil then remove respective leaves entirely
          stateTemp1 := PTAState(ParentStack.peek);
          stateTemp2 := PTAState(ParentStack.peek(2));
          if Integer(stateTemp2) = -1 then
            stateTemp2 := nil;
          Leafs.peekNextReset;
          LeafParents.peekNextReset;
          while not (LeafParents.getCurrentNode = LeafParents.getLastNode) do begin
            state1 := PTAState(Leafs.peekNext);
            state2 := PTAState(LeafParents.peekNext);
            if state2 = stateTemp1 then begin
              if stateTemp2 = nil then begin
                state1.accepts := id;  //removing this leave from Leafs for good => it is an accepting state
                Leafs.removeCurrentNode;
                LeafParents.removeCurrentNode;
              end
              else begin
                LeafParents.getCurrentNode.data := Integer(stateTemp2);
              end;
            end
            else
              break;
          end;

          if not (TerminalLink = nil) then begin
            dispose(TerminalLink);
            TerminalLink := nil;
          end;
          RightBracketState := nil;
        end
        else begin
          //add "some" links
          if (v1[1] = '*') or (v1[1] = '?') then begin
            stateTemp1 := PTAState(ParentStack.peek);
            if not (RightBracketState = nil) then begin
              state0 := RightBracketState;
            end
            else begin
              state0 := stateTemp1;
            end;
            Leafs.peekNextReset;
            LeafParents.peekNextReset;
            while not (LeafParents.getCurrentNode = LeafParents.getLastNode) do begin
              state1 := PTAState(Leafs.peekNext);
              state2 := PTAState(LeafParents.peekNext);
              if state2 = stateTemp1 then begin
                NFA.addLinksEx(state0,state1,0,LT_EPS);
              end
              else
                break;
            end;
          end;
          if (v1[1] = '*') or (v1[1] = '+') then begin
            if (TerminalLink = nil) then begin
              stateTemp1 := PTAState(ParentStack.peek);
              if not (RightBracketState = nil) then begin
                state0 := RightBracketState;
              end
              else begin
                state0 := stateTemp1;
              end;
              Leafs.peekNextReset;
              LeafParents.peekNextReset;
              while not (LeafParents.getCurrentNode = LeafParents.getLastNode) do begin
                state1 := PTAState(Leafs.peekNext);
                state2 := PTAState(LeafParents.peekNext);
                if state2 = stateTemp1 then begin
                  NFA.addLinksEx(state1,state0,0,LT_EPS);
                end
                else
                  break;
              end;
            end
            else begin
              NFA.addLinksEx(TerminalLink.src,TerminalLink.dest,TerminalLink.t,TerminalLink.linktype);
//              dispose(TerminalLink);
//              TerminalLink := nil;
            end;
          end;

          if not (TerminalLink = nil) then begin
            dispose(TerminalLink);
            TerminalLink := nil;
          end;
          RightBracketState := nil;
        end;
      end;
//      writeln(NFA.toString);
//      writeln('LLL: ' + Leafs.toString);

      prevToken := t1;
      prevValue := v1;
      v1 := getNextToken(t1);
{$IFDEF DEBUG2}
      writeln('@@@NFA@@@ '+ Leafs.toString + ' @@@' + EOL + NFA.toString(true));
{$ENDIF}
    end;

(*    if b then begin
      FTokenNameTypeTable.remove(value);
    end
    else begin
      i := FTokenNameTypeTable.getSize+1;
    end;
    FTokenNameTypeTable.add(value,i);*)

    if not (TerminalLink = nil) then begin
      dispose(TerminalLink);
      TerminalLink := nil;
    end;

// !TODO! errors - root is nil, root.parent not nil etc.
              stateTemp1 := PTAState(ParentStack.peek);
              Leafs.peekNextReset;
              LeafParents.peekNextReset;
              while not (LeafParents.getCurrentNode = LeafParents.getLastNode) do begin
                state1 := PTAState(Leafs.peekNext);
                state2 := PTAState(LeafParents.peekNext);
                if state2 = stateTemp1 then begin
                  state1.accepts := id;
                  //state1.accepts := ord(value[1]);
//                 if FLexerNFA.states.list.head <> FLexerNFA.states.list.tail then
//                  state1.accepts := 2;
(*        i := FSetTable.findIndexByName(value);
        if i = -1 then begin
          p := FFileReader.getChangedFullPosition(l,c,-Length(v1) - GT_LENGTH(t1));
          FLog.add(l,c,p,'RegExp parsing: set {' + v1 + '} not defined',-1067);

          i := FSetTable.add(TCharSet.create('DUMMY_')); //creating dummy charset with the missing name
        end;*)
                end
                else
                  break;
              end;
// !!!!!!

    ParentStack.Free;
    LeafParents.Free;
    Leafs.Free;

//    writeln(NFA.toString);
{$IFDEF DEBUG2}
    writeln('###NFA###'+ EOL+ EOL + NFA.toString);
{$ENDIF}

//    FLexerNFA.addlinksEx(FLexerNFA.startState,NFA.startState,0,LT_EPS);
//    writeln('###TTT###'+ EOL+ EOL + FLexerNFA.toString);
    FLexerNFA.addlinksEx(PTAState(FLexerNFA.states.peek),PTAState(NFA.states.peek),0,LT_EPS);
    //copy tempQueue to currentStates
    while not NFA.states.isEmpty do begin
      state := PTAState(NFA.states.dequeue);
      FLexerNFA.states.enqueue(Integer(state));
    end;
    NFA.Free;

{$IFDEF DEBUG1}
    writeln('###TOTAL###'+ EOL+ EOL + FLexerNFA.toString(true));
{$ENDIF}

    token := t1; value := v1;
    result := false;
  end;

  function parseRule(rulenum: Integer): boolean;
  var b: boolean;
      id: Integer;
      i,olen: Integer;
      tempRule: TRule;
  begin
    SetLength(tempRule.symbols,8);
    tempRule.symbols[0] := rulenum;
    tempRule.length := 1;
    tempRule.pos := -1;
    id := L_ERROR;
    v1 := getNextToken(t1);
    while true do begin
      if tempRule.pos = -1 then
        tempRule.pos := FFileReader.getPosition - Length(v1) - GT_LENGTH(t1);
      if t1 = GT_STATE then begin
        id := FStateNameTypeTable.findByKey(v1,b);
        if not b then begin
          id := NTidmax;
          NTidmax := NTidmax + 1;
          FStateNameTypeTable.add(v1,id,-1);//FFileReader.getPosition - Length(v1) - GT_LENGTH(t1));
        end;
      end
      else if (t1 = GT_TERMINAL) then begin
        i := FTokenNameTypeTable.findByKey(v1,b);
        if not b then begin
          id := Tidmax;
          Tidmax := Tidmax + 1;
          FLexerNFA.addAcceptingBranch(nil,v1,id);
          FTokenNameTypeTable.add(v1,id,FFileReader.getPosition - Length(v1) - GT_LENGTH(t1));
        end
        else begin  //b = true
          id := i;
        end;
      end
      else if (t1 = GT_STRING) then begin
        if (v1 = '') then begin
          FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),W_GF_RULE_EMPTY_STRING_1_IN_RULE_2_DEFINITION_EPSILON,v1,value);
          id := L_EPSILON;  //epsilon
        end
        else begin
//          i := FTokenNameTypeTable.findByKey(v1,b);
//          if b then
//            FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),W_GF_RULE_TERMINAL_WITH_NAME_1_ALREADY_EXISTS_POSSIBLE_MISTAKE,v1);
          begin
            v2 := DUMMY_TERMINAL_STRING_NAME_PREFIX + v1;
            i := FTokenNameTypeTable.findByKey(v2,b);
            if not b then begin
              id := Tidmax;
              Tidmax := Tidmax + 1;
              FLexerNFA.addAcceptingBranch(nil,resolveEscSeq(v1),id);
              FTokenNameTypeTable.add(v2,id,FFileReader.getPosition - Length(v1) - GT_LENGTH(t1));
            end
            else begin
              id := i;
            end;
          end;
        end;
      end
(*
          i := FTokenNameTypeTable.findByKey(v1,b);
          if (t1 = GT_TERMINAL) and (b) then begin
            id := i;  //FTokenNameTypeTable.findByKey(v1,b);
          end
          else begin
//            id := FTokenNameTypeTable.getSize + L_T_BEGIN;
            id := Tidmax;
            Tidmax := Tidmax + 1;
            if (t1 = GT_STRING) and (b) then begin
              v2 := DUMMY_TERMINAL_STRING_NAME_PREFIX + v1;
              i := FTokenNameTypeTable.findByKey(v2,b);
              if b then begin
                FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),W_GF_RULE_TERMINAL_WITH_NAME_1_ALREADY_EXISTS_POSSIBLE_MISTAKE,v1);
                id := i;
              end
              else begin
                FLexerNFA.addAcceptingBranch(nil,resolveEscSeq(v1),id);
                FTokenNameTypeTable.add(v2,id);
              end;
            end
            else begin  //not b
              if t1 = GT_TERMINAL then
                FLexerNFA.addAcceptingBranch(nil,v1,id)
              else
                FLexerNFA.addAcceptingBranch(nil,resolveEscSeq(v1),id);
              FTokenNameTypeTable.add(v1,id);   //!CAUTION nemalo by sa tu davat resolveEscSeq(v1) ak je to string ?
            end;
          end;
        end;
      end
*)
      else if t1 = GT_OP then begin
        if (v1 <> '|') then begin
          FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_RULE_INVALID_OPERATOR_1_IN_RULE_2_DEFINITION_ONLY_OR_ALLOWED,v1,value);
          break;
        end;
        //add current to ruletable and continue as if after <value> ::=
        if tempRule.length >= 1 then begin
          olen := FRuleTable.count;
          i := FRuleTable.addRule(tempRule);
          if i < olen then begin
              FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),W_GF_RULE_EXACT_SAME_RULE_FOR_HEAD_1_ALREADY_DEFINED_AT_POS_2,value,IntToStr(FRuleTable.getRuleByIndex(I).pos));
          end;
        end
        else begin
          FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_RULE_EMPTY_OR_NOT_VALID_RIGHT_SIDE_1,v1);
        end;

//        SetLength(tempRule.symbols,1);
//        tempRule.symbols[0] := rulenum;
//        tempRule.length := 1;

//        if Length(tempRule.symbols) > 8 then begin
//          SetLength(tempRule.symbols,8);
//        end;
        tempRule.symbols := nil;
        SetLength(tempRule.symbols,8);
        tempRule.length := 0;
        tempRule.pos := -1;
        id := rulenum;
      end
      else if (t1 = GT_COMMENT) or (t1 = GT_EOL) then begin
//        if (t1 = GT_COMMENT) then begin
//          v1 := getNextToken(t1);
//        end;
//        if (t1 = GT_EOL) then begin
//          v1 := getNextToken(t1);
          while (t1 = GT_COMMENT) or (t1 = GT_EOL) do
            v1 := getNextToken(t1);
          if (t1 <> GT_OP) or (v1 <> '|') then begin
            if (t1 <> GT_EOF) and (t1 <> GT_STATE) and (t1 <> GT_EQ) and (t1 <> GT_SIM) and (t1 <> GT_IND) and (t1 <> GT_HL) then begin
              FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),W_GF_RULE_EXPECTED_RULE_HEADER_OR_SEPARATOR_AS_FIRST_SYMBOL_ON_NEW_LINE_INSTEAD_OF_1,v1);
            end;
            break;
          end;
          continue;
//        end;
      end
      else begin
//        if (t1 <> GT_EOF) and (t1 <> GT_COMMENT) and (t1 <> GT_EOL) then begin
        if (t1 <> GT_EOF) then begin
          FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_RULE_EXPECTED_STATE_OR_STRING_OR_OP_INSTEAD_OF_1,v1);
        end;
        break;
      end;
      if id <> L_EPSILON then begin
        tempRule.symbols[tempRule.length] := id;
        tempRule.length := tempRule.length + 1;
        if tempRule.length = Length(tempRule.symbols) then begin
          SetLength(tempRule.symbols, Length(tempRule.symbols) * 2);
        end;
      end;

      v1 := getNextToken(t1);
    end;

    //add current to ruletable and continue
    if tempRule.length >= 1 then begin
      olen := FRuleTable.count;
      i := FRuleTable.addRule(tempRule);
      if i < olen then begin
          FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),W_GF_RULE_EXACT_SAME_RULE_FOR_HEAD_1_ALREADY_DEFINED_AT_POS_2,value,IntToStr(FRuleTable.getRuleByIndex(I).pos));
      end;
    end
    else begin
      FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_RULE_EMPTY_OR_NOT_VALID_RIGHT_SIDE_1,v1);
    end;

    tempRule.symbols := nil;
    tempRule.length := 0;

    token := t1; value := v1;
    result := false;
  end;

  function parseTerminal: boolean;
  var i: Integer;
//      pos: Integer;
      b: Boolean;
      rside,rend: Integer;
      S: xString;
      posit: Integer;
      key,regexp: xString;
  begin
    result := false;
    tpos := FFileReader.getPosition - Length(value) - GT_LENGTH(token);
    v1 := getNextToken(t1);
    if (t1 <> GT_OP) or (v1 <> '=') then begin
//      p := FFileReader.getChangedFullPosition(l,c,-Length(v1) - GT_LENGTH(t1));
//      FLog.add(l,c,p,'Terminal declaration parsing failed: ' + v1 + ' found ' + '= expected',-61);
      FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_EXPECTED_EQ_INSTEAD_OF_1,v1);

      token := t1; value := v1;
      exit;
    end;
    rside := FFileReader.getPosition;
(*
    i := FTokenNameTypeTable.findByKey(value,b);
    if b then begin
      FLog.addSFR2(FFileReader,tpos - FFileReader.getPosition,W_GF_TERMINAL_REDEFINED_1,value);
    end;
*)
    id := FTokenNameTypeTable.findByKey(value,b);
    if not b then begin
      id := Tidmax;
      Tidmax := Tidmax + 1;
      FTokenNameTypeTable.add(value,id,tpos);
    end
    else begin
      FLog.addSFR2(FFileReader,tpos - FFileReader.getPosition,W_GF_TERMINAL_REDEFINED_1,value);
    end;

    v2 := '';
    result := parseRegExp;

    rend := FFileReader.getPosition - Length(value) - GT_LENGTH(token);
    if FTokenNameTypeTable.getByValue(id,key,posit,regexp) then begin
      S := FFileReader.getBlock(- Length(value) - GT_LENGTH(token) - (rend - rside),- Length(value) - GT_LENGTH(token) - 1);
//      S := regexp + S;
      if posit = -1 then
        posit := tpos;
      if not FTokenNameTypeTable.replaceByValue(id,key,posit,S) then begin
        FLog.addSFR2(FFileReader,posit - FFileReader.getPosition,E_GF_ERROR_IN_TERMINAL_PARSING_1,key);
      end;
    end;
  end;

  function parseState: boolean;
  var i: Integer;
      b: Boolean;
      rside,rend: Integer;
      S: xString;
      posit: Integer;
      key,regexp: xString;
  begin
    result := false;
    tpos := FFileReader.getPosition - Length(value) - GT_LENGTH(token);

    v1 := getNextToken(t1);
    if (t1 <> GT_ASSIGN) or (v1 <> '::=') then begin
      FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_STATE_EXPECTED_ASSIGN_INSTEAD_OF_1,v1);
      token := t1; value := v1;
      exit;
    end;
    rside := FFileReader.getPosition;
(*
    i := FStateNameTypeTable.findByKey(value,b);
    if b then begin
      FLog.addSFR2(FFileReader,pos - FFileReader.getPosition,W_GF_TERMINAL_REDEFINED_1,value);
    end;
 *)
    i := FStateNameTypeTable.findByKey(value,b);
    if not b then begin
      i := NTidmax;
      NTidmax := NTidmax + 1;
      FStateNameTypeTable.add(value,i,tpos);
    end;

    v2 := '';
    result := parseRule(i);

    rend := FFileReader.getPosition - Length(value) - GT_LENGTH(token);
    if FStateNameTypeTable.getByValue(i,key,posit,regexp) then begin
      S := FFileReader.getBlock(- Length(value) - GT_LENGTH(token) - (rend - rside),- Length(value) - GT_LENGTH(token));
      S := regexp + S;
      if posit = -1 then
        posit := tpos;
      if not FStateNameTypeTable.replaceByValue(i,key,posit,S) then begin
        FLog.addSFR2(FFileReader,posit - FFileReader.getPosition,E_GF_ERROR_IN_STATE_PARSING_1,key);
      end;
    end;
  end;

  function parseEq: Boolean;
  var hashset: TIntHashSet;
      index: Integer;
      k: Integer;
      b: Boolean;
      v3: xString;
      t3: Integer;
  begin
    result := false;
    tpos := FFileReader.getPosition;
    v2 := getNextToken(t2);
    if ((t2 <> GT_TERMINAL) and (t2 <> GT_STATE) and (t2 <> GT_STRING)) or (not IsNumber(v2)) then begin
      FLog.addSFR2(FFileReader,-Length(v2)-GT_LENGTH(t2),E_GF_INTEGER_NUMBER_EXPECTED_INSTEAD_OF_1,v2);
      token := t2; value := v2;
      exit;
    end;
    v3 := getNextToken(t3);
    if ((t3 <> GT_TERMINAL) and (t3 <> GT_STATE) and (t3 <> GT_STRING)) or (not IsNumber(v3)) then begin
      t1 := t3;
      v1 := v3;
      v3 := v2;
    end
    else begin
      v1 := getNextToken(t1);
    end;

    if (t1 <> GT_TERMINAL) and (t1 <> GT_STATE) and (t1 <> GT_STRING) then begin
      if (t1 <> GT_EOL) and (t1 <> GT_EOF) then
        FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_STATE_OR_STRING_EXPECTED_INSTEAD_OF_1,v1);
      FRuleTable.FDefEqScore := StrToIntDef(v2,1);
      FRuleTable.FDefEqScore2 := StrToIntDef(v3,1);
      token := t1; value := v1;
      exit;
    end;
    hashset := TIntHashSet.create;
    hashset.tag := StrToIntDef(v2,1);
    hashset.tag2 := StrToIntDef(v3,1);

    if FRuleTable.FEquivalenceSets <> nil then begin
      index := FRuleTable.FEquivalenceSets.getSize + 1;
    end
    else begin
      FRuleTable.FEquivalenceSets := TIntIntHashSet.create;
      index := 1;
    end;
    FRuleTable.FEquivalenceSets.add(index,Integer(hashset));

    if FRuleTable.FEquivalentIndices = nil then begin
      FRuleTable.FEquivalentIndices := TIntIntHashSet.create;
    end;

    repeat
      k := -1;
      b := false;
      if (t1 = GT_STRING) or (t1 = GT_TERMINAL) then begin
        k := FTokenNameTypeTable.findByKey(v1,b);
      end
      else if (t1 = GT_STATE) then begin
        k := FStateNameTypeTable.findByKey(v1,b);
      end;
      if not b then begin
        FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_OR_NONTERMINAL_1_NOT_DEFINED,v1);
        token := t1; value := v1;
//        hashset.Free;
        exit;
      end;
      hashset.add(k);
      if not FRuleTable.FEquivalentIndices.add(k,index) then begin
        FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_OR_NONTERMINAL_1_ALREADY_IN_OTHER_EQUIVALENCE_SET,v1);
        token := t1; value := v1;
      end;
      v1 := getNextToken(t1);
    until (t1 = GT_EOL) or (t1 = GT_EOF) or ((t1 <> GT_TERMINAL) and (t1 <> GT_STATE) and (t1 <> GT_STRING));

    token := t1; value := v1;
    result := false;
  end;

  function parseSim: Boolean;
  var hashset: TIntHashSet;
      index: Integer;
      k: Integer;
      b: Boolean;
  begin
    result := false;
    tpos := FFileReader.getPosition;
    v2 := getNextToken(t2);
    if ((t2 <> GT_TERMINAL) and (t2 <> GT_STATE) and (t2 <> GT_STRING)) or (not IsNumber(v2)) then begin
      FLog.addSFR2(FFileReader,-Length(v2)-GT_LENGTH(t2),E_GF_INTEGER_NUMBER_EXPECTED_INSTEAD_OF_1,v2);
      token := t2; value := v2;
      exit;
    end;
    v1 := getNextToken(t1);
    if (t1 <> GT_TERMINAL) and (t1 <> GT_STATE) and (t1 <> GT_STRING) then begin
      if (t1 <> GT_EOL) and (t1 <> GT_EOF) then
        FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_STATE_OR_STRING_EXPECTED_INSTEAD_OF_1,v1);
      FRuleTable.FDefSimScore := StrToIntDef(v2,1);
      token := t1; value := v1;
      exit;
    end;
    hashset := TIntHashSet.create;
    hashset.tag := StrToIntDef(v2,1);

    if FRuleTable.FSimilaritySets <> nil then begin
      index := FRuleTable.FSimilaritySets.getSize + 1;
    end
    else begin
      FRuleTable.FSimilaritySets := TIntIntHashSet.create;
      index := 1;
    end;
    FRuleTable.FSimilaritySets.add(index,Integer(hashset));

    if FRuleTable.FSimilarIndices = nil then begin
      FRuleTable.FSimilarIndices := TIntIntHashSet.create;
    end;

    repeat
      k := -1;
      b := false;
      if (t1 = GT_STRING) or (t1 = GT_TERMINAL) then begin
        k := FTokenNameTypeTable.findByKey(v1,b);
      end
      else if (t1 = GT_STATE) then begin
        k := FStateNameTypeTable.findByKey(v1,b);
      end;
      if not b then begin
        FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_OR_NONTERMINAL_1_NOT_DEFINED,v1);
        token := t1; value := v1;
//        hashset.Free;
        exit;
      end;
      hashset.add(k);
      if not FRuleTable.FSimilarIndices.add(k,index) then begin
        FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_OR_NONTERMINAL_1_ALREADY_IN_OTHER_SIMILARITY_SET,v1);
        token := t1; value := v1;
      end;
      v1 := getNextToken(t1);
    until (t1 = GT_EOL) or (t1 = GT_EOF) or ((t1 <> GT_TERMINAL) and (t1 <> GT_STATE) and (t1 <> GT_STRING));

    token := t1; value := v1;
    result := false;
  end;

  function getnextinteger(var res: Integer): Boolean;
  begin
    result := false;
    res := 0;
    v1 := getNextToken(t1);
    if (t1 <> GT_TERMINAL) and (t1 <> GT_STATE) and (t1 <> GT_STRING) or (not IsNumber(v1)) then begin
      FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_INTEGER_NUMBER_EXPECTED_INSTEAD_OF_1,v1);
      token := t1; value := v1;
      exit;
    end;
    res := StrToIntDef(v1,0);
    result := true;
  end;

  function parseInd: Boolean;
  var hashset: TIntHashSet;
      origindex,index: Integer;
      k: Integer;
      b: Boolean;
      bspace,bindent,beol,aspace,aindent,aeol: Integer;
      bexcept,aexcept: TIntHashSet;
      pdata: PTokenData;
  begin
    result := false;
    aexcept := nil;
    bexcept := nil;
    tpos := FFileReader.getPosition;
    v2 := getNextToken(t2);
    if (t2 = GT_TERMINAL) or (t2 = GT_STRING) then begin
      if IsNumber(v2) then begin
        index := 0;  //defining default
      end
      else begin
        index := FTokenNameTypeTable.findByKey(v2,b);
        if not b then begin
          FLog.addSFR2(FFileReader,-Length(v2)-GT_LENGTH(t2),E_GF_TERMINAL_OR_NONTERMINAL_1_NOT_DEFINED,v2);
          token := t2; value := v2;
          exit;
        end;
      end;
    end
    else if (t2 = GT_STATE) then begin
      index := FStateNameTypeTable.findByKey(v2,b);
      if not b then begin
        FLog.addSFR2(FFileReader,-Length(v2)-GT_LENGTH(t2),E_GF_TERMINAL_OR_NONTERMINAL_1_NOT_DEFINED,v2);
        token := t2; value := v2;
        exit;
      end;
    end
    else begin
      FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_STATE_OR_STRING_EXPECTED_INSTEAD_OF_1,v1);
      token := t1; value := v1;
      exit;
    end;
    origindex := index;

    if index = 0 then begin
      t1 := t2; v1 := v2;
    end
    else begin
      v1 := getNextToken(t1);
    end;
    if (t1 <> GT_TERMINAL) and (t1 <> GT_STATE) and (t1 <> GT_STRING) or (not IsNumber(v1)) then begin
      FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_INTEGER_NUMBER_EXPECTED_INSTEAD_OF_1,v1);
      token := t1; value := v1;
      exit;
    end;
    bspace := StrToIntDef(v1,0);
    if not getnextinteger(bindent) then
      Exit;
    if not getnextinteger(beol) then
      Exit;

    v1 := getNextToken(t1);
    while ((t1 = GT_TERMINAL) or (t1 = GT_STATE) or (t1 = GT_STRING)) and (not IsNumber(v1)) do begin
      if bexcept = nil then
        bexcept := TIntHashSet.create;

      if (t1 = GT_TERMINAL) or (t1 = GT_STRING) then begin
          index := FTokenNameTypeTable.findByKey(v1,b);
          if not b then begin
            FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_OR_NONTERMINAL_1_NOT_DEFINED,v1);
            token := t1; value := v1;
            bexcept.Free;
            exit;
          end;
      end
      else if (t1 = GT_STATE) then begin
        index := FStateNameTypeTable.findByKey(v1,b);
        if not b then begin
          FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_OR_NONTERMINAL_1_NOT_DEFINED,v1);
          token := t1; value := v1;
          bexcept.Free;
          exit;
        end;
      end;
      bexcept.add(index);
      v1 := getNextToken(t1);
    end;

    if (t1 <> GT_TERMINAL) and (t1 <> GT_STATE) and (t1 <> GT_STRING) or (not IsNumber(v1)) then begin
      FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_INTEGER_NUMBER_EXPECTED_INSTEAD_OF_1,v1);
      token := t1; value := v1;
      exit;
    end;
    aspace := StrToIntDef(v1,0);
    if not getnextinteger(aindent) then
      Exit;
    if not getnextinteger(aeol) then
      Exit;

    v1 := getNextToken(t1);
    while ((t1 = GT_TERMINAL) or (t1 = GT_STATE) or (t1 = GT_STRING)) and (not IsNumber(v1)) do begin
      if aexcept = nil then
        aexcept := TIntHashSet.create;

      if (t1 = GT_TERMINAL) or (t1 = GT_STRING) then begin
          index := FTokenNameTypeTable.findByKey(v1,b);
          if not b then begin
            FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_OR_NONTERMINAL_1_NOT_DEFINED,v1);
            token := t1; value := v1;
            aexcept.Free;
            exit;
          end;
      end
      else if (t1 = GT_STATE) then begin
        index := FStateNameTypeTable.findByKey(v1,b);
        if not b then begin
          FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_OR_NONTERMINAL_1_NOT_DEFINED,v1);
          token := t1; value := v1;
          aexcept.Free;
          exit;
        end;
      end;
      aexcept.add(index);
      v1 := getNextToken(t1);
    end;

    if FIndTable = nil then
      FIndTable := TIntIntHashSet.create;

    new(pdata);
    pdata.bspace := bspace;
    pdata.bindent := bindent;
    pdata.beoln := beol;
    pdata.bexcept := bexcept;
    pdata.aspace := aspace;
    pdata.aindent := aindent;
    pdata.aeoln := aeol;
    pdata.aexcept := aexcept;
    FIndTable.add(origindex,Integer(pdata));

    token := t1; value := v1;
    result := false;
  end;

  function parseHL: Boolean;
  var hashset: TIntHashSet;
      origindex,index: Integer;
      k: Integer;
      b: Boolean;
      tc,bc,style: Integer;
      phighlight: PTokenHighlight;
  begin
    result := false;
    tpos := FFileReader.getPosition;
    v2 := getNextToken(t2);
    if (t2 = GT_TERMINAL) or (t2 = GT_STRING) then begin
      if IsNumber(v2) then begin
        index := 0;  //defining default
      end
      else begin
        index := FTokenNameTypeTable.findByKey(v2,b);
        if not b then begin
          FLog.addSFR2(FFileReader,-Length(v2)-GT_LENGTH(t2),E_GF_TERMINAL_OR_NONTERMINAL_1_NOT_DEFINED,v2);
          token := t2; value := v2;
          exit;
        end;
      end;
    end
    else if (t2 = GT_STATE) then begin
      index := FStateNameTypeTable.findByKey(v2,b);
      if not b then begin
        FLog.addSFR2(FFileReader,-Length(v2)-GT_LENGTH(t2),E_GF_TERMINAL_OR_NONTERMINAL_1_NOT_DEFINED,v2);
        token := t2; value := v2;
        exit;
      end;
    end
    else begin
      FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_TERMINAL_STATE_OR_STRING_EXPECTED_INSTEAD_OF_1,v1);
      token := t1; value := v1;
      exit;
    end;
    origindex := index;

    if index = 0 then begin
      t1 := t2; v1 := v2;
    end
    else begin
      v1 := getNextToken(t1);
    end;
    if (t1 <> GT_TERMINAL) and (t1 <> GT_STATE) and (t1 <> GT_STRING) or (not IsNumber(v1)) then begin
      FLog.addSFR2(FFileReader,-Length(v1)-GT_LENGTH(t1),E_GF_INTEGER_NUMBER_EXPECTED_INSTEAD_OF_1,v1);
      token := t1; value := v1;
      exit;
    end;
    tc := StrToIntDef(v1,0);
    if not getnextinteger(bc) then
      Exit;
    if not getnextinteger(style) then
      Exit;

    v1 := getNextToken(t1);

    if FHLTable = nil then
      FHLTable := TIntIntHashSet.create;

    new(phighlight);
    phighlight.tc := tc;
    phighlight.bc := bc;
    phighlight.style := style;
    FHLTable.add(origindex,Integer(phighlight));

    token := t1; value := v1;
    result := false;
  end;

  function main: boolean;
  begin
    result := false;
    while result = false do begin
      case token of
        GT_PROPERTY: result := parseProperty;
        GT_SET: result := parseSet;
        GT_EOL: result := true;
        GT_EOF: result := true;
        GT_ERROR: result := true;           //reported in lexer
        GT_TERMINAL: result := parseTerminal;
        GT_COMMENT: result := parseComment;
        GT_STATE: result := parseState;
        GT_EQ: result := parseEq;
        GT_SIM: result := parseSim;
        GT_IND: result := parseInd;
        GT_HL: result := parseHL;
    (*
        GT_ERROR: ;      //(error)
        GT_EOL: ;         //#10
        GT_PROPERTY: ;    //"..."
        GT_SET: ;         //{...}
        GT_SETSTRING: ;   //[...]
        GT_COMMENT: ;     //!...
        GT_STRING: ;      //'...'
        GT_TERMINAL: ;    //...
        GT_STATE: ;       //<...>
        GT_OP: ;          //+-*?|=()
        GT_ASSIGN: ;      //::=
        GT_EOF: ;        //(eof)
    *)
      else
        result := true;
      end;
    end;
    //result := true;
  end;
var hashset: TIntHashSet;
    S: xString;
    firstRule: TRule;
    i,j: Integer;
    Cname: array[1..9] of xString;
    Ci: array[1..9] of Integer;
    b: Boolean;
    StartState, WSTerminal: Integer;
    tempArray: array of TStrIntEntry;
begin
  NTidmax := L_NT_BEGIN;
  Tidmax := L_T_BEGIN;

  FLexerNFA := TNFA.create(FSetTable);
  FLexerNFA.addStateEx(0);

  FStateNameTypeTable.add('$S''$',L_S,-1);
  SetLength(firstRule.symbols,8);
  firstRule.symbols[0] := L_S;
  firstRule.symbols[1] := -1;
  firstRule.symbols[2] := L_EOF;
  firstRule.length := 2;//3;  //still not sure
  firstRule.pos := -1;
  FRuleTable.addRule(firstRule);
  FStateNameTypeTable.add('$REJECT$',L_SPECIAL,-1);

  //building NFA
  repeat
    Application.ProcessMessages;
    if FGlobalHalt then
      break;
    value := getNextToken(token);
    main;
  until FFileReader.atEOF;


{$IFDEF DEBUG0}
    writeln('NFA States: '+ IntToStr(FLexerNFA.getSize));
    writeln('NFA Links: '+ IntToStr(FLexerNFA.getNumOfLinks));
{$ENDIF}
{$IFDEF LOG0}
    if LogForm <> nil then begin
      LogForm.AddLine('NFA States: '+ IntToStr(FLexerNFA.getSize));
      LogForm.AddLine('NFA Links: '+ IntToStr(FLexerNFA.getNumOfLinks));
    end;
{$ENDIF}

  S := FPropertyTable.getByName('Case Sensitive').value;
  if (S = '1') or (AnsiUpperCase(S) = AnsiUpperCase('true')) then begin
    FLexerNFA.setCaseSensitivity(True);
    FRuleTable.setCaseSensitivity(True);
  end
  else if (S = '0') or (AnsiUpperCase(S) = AnsiUpperCase('false')) then begin
    FLexerNFA.setCaseSensitivity(False);
    FRuleTable.setCaseSensitivity(False);
  end
  else begin
    FLexerNFA.setCaseSensitivity(DEFAULT_CASE_SENSITIVITY);
    FRuleTable.setCaseSensitivity(DEFAULT_CASE_SENSITIVITY);
  end;

  S := FPropertyTable.getByName('Line End Char').value;
  if S <> '' then
    FRuleTable.setLEC(S[1]);

  //every non-terminal(state) in ruletable must be is in some rule header
  hashset := TIntHashSet.create;
  for i := 0 to Pred(FRuleTable.count) do begin
    hashset.add(FRuleTable.rules[i].symbols[0]);
  end;
  SetLength(tempArray,FStateNameTypeTable.getSize);
  b := FStateNameTypeTable.fillArray(tempArray);
  for i := 0 to Pred(Length(tempArray)) do begin
    if tempArray[i].value >= L_NT_BEGIN then  //check only user defined non-terminals
      if not hashset.contains(tempArray[i].value) then begin
        FStateNameTypeTable.getByValue(tempArray[i].value,S,j);
        FLog.addSFR2(FFileReader,-FFileReader.getPosition + j,E_GF_NONTERMINAL_1_IN_NO_RULE_AS_HEADER,tempArray[i].key);
      end;
  end;
  hashset.free;
  tempArray := nil;

  //checking if start symbol is in rule headers
  S := FPropertyTable.getByName('Start Symbol').value;
  if S = '' then begin
    id := L_S + 1;
    S := FStateNameTypeTable.findByValue(id,b);
    if b then begin
      FLog.addSFR2(FFileReader,0,W_GF_START_SYMBOL_NOT_DECLARED_USING_FIRST_FOUND_1,S);
    end
    else begin
      FLog.addSFR2(FFileReader,0,E_GF_START_SYMBOL_NOT_DECLARED_AND_NO_VALID_RULES_DECLARED);
      id := -1;
    end;
  end
  else begin
//    id := FTokenNameTypeTable.findByKey(S,b);

    id := FStateNameTypeTable.findByKey(S,b);
    if b then begin
    end
    else begin
      FLog.addSFR2(FFileReader,0,W_GF_START_SYMBOL_1_RULE_IS_MISSING,S);
      id := L_S + 1;
      S := FStateNameTypeTable.findByValue(id,b);
      if b then begin
        FLog.addSFR2(FFileReader,0,W_GF_START_SYMBOL_MISSING_USING_FIRST_FOUND_1,S);
      end
      else begin
        FLog.addSFR2(FFileReader,0,E_GF_START_SYMBOL_NOT_DECLARED_AND_NO_VALID_RULES_DECLARED);
        id := -1;
      end;
    end;
  end;

  StartState := id;
  firstRule.symbols[1] := StartState; //correcting rule 0 - from -1 id, to correct, read from file id
//  FRuleTable.replaceRuleAtIndex(0,firstRule);

  FTokenNameTypeTable.add('$EPSILON$',L_EPSILON,-1);
  FTokenNameTypeTable.add('$ERROR$',L_ERROR,-1);
  FTokenNameTypeTable.add('$EOF$',L_EOF,-1);
  FTokenNameTypeTable.add('$DUMMY$',L_DUMMY,-1);
//  FTokenNameTypeTable.add('$WS$',L_WS);

  //whitespace terminal
  S := FPropertyTable.getByName('WS Terminal').value;
  if S = '' then begin
    WSTerminal := -1;
  end
  else begin
    WSTerminal := FTokenNameTypeTable.findByKey(S,b);
    if not b then begin
      FLog.addSFR2(FFileReader,0,W_GF_SUPPOSED_WS_TERMINAL_1_IS_NOT_EVEN_DEFINED,S);
      WSTerminal := -1;
    end;
  end;

  if WSTerminal >= 1 then begin   //0 is realy reserved
    FRuleTable.setWSTerminal(WSTerminal);
  end;

  S := FPropertyTable.getByName('EOLNWS Terminal').value;
  if S = '' then begin
    WSTerminal := -1;
  end
  else begin
    WSTerminal := FTokenNameTypeTable.findByKey(S,b);
    if not b then begin
      FLog.addSFR2(FFileReader,0,W_GF_SUPPOSED_EOLNWS_TERMINAL_1_IS_NOT_EVEN_DEFINED,S);
      WSTerminal := -1;
    end;
  end;

  if WSTerminal >= 1 then begin   //0 is realy reserved
    FRuleTable.setEOLNWSTerminal(WSTerminal);
  end;

  S := FPropertyTable.getByName('Parse Comments').value;
  if (S = '1') or (AnsiUpperCase(S) = AnsiUpperCase('true')) then begin
    FRuleTable.setParseComments(True);
  end
  else if (S = '0') or (AnsiUpperCase(S) = AnsiUpperCase('false')) then begin
    FRuleTable.setParseComments(False);
  end
  else begin
    FRuleTable.setParseComments(DEFAULT_PARSE_COMMENTS);
  end;

  //comment terminals
  Cname[1] := 'CLINE Terminal';
  Cname[2] := 'CLINEEND Terminal';
  Cname[3] := 'CSTART Terminal';
  Cname[4] := 'CEND Terminal';
  Cname[5] := 'CSTART2 Terminal';
  Cname[6] := 'CEND2 Terminal';
  Cname[7] := 'COMMENTLINE Terminal';
  Cname[8] := 'COMMENT1 Terminal';
  Cname[9] := 'COMMENT2 Terminal';

  for i := 1 to 9 do begin
    S := FPropertyTable.getByName(Cname[i]).value;
    if S = '' then begin
      Ci[i] := -1;
    end
    else begin
      Ci[i] := FTokenNameTypeTable.findByKey(S,b);
      if not b then begin
        FLog.addSFR2(FFileReader,0,W_GF_SUPPOSED_COMMENT_TERMINAL_1_FROM_PROPERTY_2_IS_NOT_EVEN_DEFINED,S,Cname[i]);
        Ci[i] := -1;
      end;
    end;
  end;
  FRuleTable.setCommentTerminals(Ci[1],Ci[2],Ci[3],Ci[4],Ci[5],Ci[6]);
  FRuleTable.setComments(Ci[7],Ci[8],Ci[9]);


  //not used, but declared terminals - warnings
  hashset := TIntHashSet.create;
  for i := 0 to Pred(FRuleTable.count) do begin
    for j := 1 to Pred(FRuleTable.rules[i].length) do begin
      if FRuleTable.rules[i].symbols[j] <= MAX_TERMINAL then
        hashset.add(FRuleTable.rules[i].symbols[j]);
    end;
  end;
  SetLength(tempArray,FTokenNameTypeTable.getSize);
  b := FTokenNameTypeTable.fillArray(tempArray);
  for i := 0 to Pred(Length(tempArray)) do begin
    if (tempArray[i].value >= L_T_BEGIN) and (tempArray[i].value <> WSTerminal) then  //check only user defined terminals
      if not hashset.contains(tempArray[i].value) then begin
        FTokenNameTypeTable.getByValue(tempArray[i].value,S,j);
        if (S <> FPropertyTable.getByName('WS Terminal').value) and (S <> FPropertyTable.getByName('COMMENTLINE Terminal').value) and (S <> FPropertyTable.getByName('COMMENT1 Terminal').value) and (S <> FPropertyTable.getByName('COMMENT2 Terminal').value) and (S <> FPropertyTable.getByName('EOLNWS Terminal').value) then
          FLog.addSFR2(FFileReader,-FFileReader.getPosition + j,W_GF_TERMINAL_1_DECLARED_BUT_NOT_USED,tempArray[i].key);
      end;
  end;
  hashset.free;
  tempArray := nil;

{$IFDEF DEBUG2}
  writeln(FTokenNameTypeTable.toString(true));
  writeln(FStateNameTypeTable.toString(true));
  writeln(FRuleTable.toString(true,FTokenNameTypeTable,FStateNameTypeTable));
{$ENDIF}

//  S := FSetTable.toString;
//  ShowMessage(S);
//  MessageBoxW(0,PWideChar(S),PWideChar('Sets:'),MB_OK);

  result := true;
end;

/// funkcia vracajuca NFA
function TGrammarFileLexer.getNFA: TNFA;
begin
  result := FLexerNFA;
end;


/// funkcia vracajuca tabulku nazvov tokenov (terminalov) s ich identifikatormi
function TGrammarFileLexer.getTokenNameTypeTable: TStrIntHashMap;
begin
  result := FTokenNameTypeTable;
end;

/// funkcia vracajuca tabulku nazvov stavov (neterminalov) s ich identifikatormi
function TGrammarFileLexer.getStateNameTypeTable: TStrIntHashMap;
begin
  result := FStateNameTypeTable;
end;

/// funkcia vracajuca tabulku pravidiel extrahovanu zo subora popisujuceho vstupnu gramatiku
function TGrammarFileLexer.getRuleTable: TRuleTable;
begin
  result := FRuleTable;
end;

{ TParserBuilder }

/// konstruktor vytvarajuci objekt zodpovedny za vytvorenie LALR parsera
/// zo vstupnej tabulky pravidiel;
/// ttable a stable su tabulky nazvov terminalov a neterminalov posielane najma kvoli vypisu parsovacich tabuliek
/// vstupny lexer sluzi po vytvoreni, aby mal rovno link na lexikalny analyzator
constructor TParserBuilder.create(lexer: TLexer; ttable, stable: TStrIntHashMap;
  ruletable: TRuleTable);
begin
  inherited create;
  LLexer := lexer;
  LTokenNameTypeTable := ttable;
  LStateNameTypeTable := stable;
  LRuleTable := ruletable;
  FFirst := TIntIntHashSet.create;
  FFollow := TIntIntHashSet.create;
  FLog := TLog.create;
end;

/// funkcia budujuca LALR parser, teda vlastne parsovacie tabulky, algoritmus samotneho parsovania
/// na zaklade parsovacih tabuliek je implementovane v TParser triede
/// popis tohto algoritmu a referencie je mozne najst v texte diplomovej prace
function TParserBuilder.buildLALRParser: boolean;
var itemset: TItemSet;
    closureset: TItemSet;
    lr0: TLR0Item;
    lalr: TLALRItem;
    S: xString;
    substTable: TIntIntHashSet;
    globalSet: TSetOfItemSets;
    index: Integer;
    plalr,pnewlalr: PLALRItem;
    I,J: Integer;
    itemsetLR0Jumps: TIntIntHashSet;
    symbol: Integer;
    dot: Integer;
    rule: TRule;
    hashset: TIntHashSet;
    plalrParent: PLALRItem;
    newitemset: TItemSet;
    olditemset: TItemSet;
    lastsetitem: PIntHashSetItem;
    lastbucket: Integer;
    gotoTable: TIntIntHashSet;
    reduceTable: TIntIntHashSet;
    IIhashset: TIntIntHashSet;
//    reducableitemsSet: TIntIntHashSet; //non-unique allowed
    reducableitemsQueue: TQueueStack;
    b,bAdded: Boolean;
    propagateQueue: TQueueStack;
    propagateSet: TIntHashSet;
    IIhashsetTemp: TIntIntHashSet;
    pair: PPair;
    conflictStates: TIntIntHashSet;
begin
  calculateFirst;
//  calculateFollow;  //not needed

  //create first kernel item set containing of only 1 LALRItem = S' -> S. , $
  LRuleTable.rules[0].length := 2;  //we're building LALR parser from S' -> .S, $ is lookahead (NOT S' -> .S$, $)
  itemset := TItemSet.create;
  lr0.rulenum := 0;
  lr0.dot := 1;
  lalr.lr0 := lr0;
  lalr.lookahead := TIntHashSet.create(L_EOF);
  lalr.lapropagate := nil;
  lalr.bPropagated := true;  //false - safety first, true - aggressive
  itemset.add(lalr);
  lalr.lookahead.Free;

  globalSet := TSetOfItemSets.create;
  if not globalSet.add(itemset,I) then begin
    //!TODO! assert - not possible to add to SetOfItemSets ???
    itemset.Free;
    Exit;
  end;

  conflictStates := TIntIntHashset.create;

  propagateSet := TIntHashSet.create;
  propagateQueue := TQueueStack.create;

  gotoTable := TIntIntHashSet.create;
//  reducableitemsSet := TIntIntHashSet.create(false);
  reducableitemsQueue := TQueueStack.create;

  index := 0;
  repeat
    Application.ProcessMessages;
    if FGlobalHalt then
      break;
{$IFDEF DEBUG1}
    writeln('Current itemset: ' + IntToStr(index));
{$ENDIF}
    itemset := globalSet.getByIndex(index);
    //generate substitution table
    substTable := TIntIntHashSet.create;
    I := MAX_TERMINAL + 1;
    itemSet.getNextReset;
    while itemSet.getNext(plalr) do begin
      substTable.add(I,Integer(plalr));
      //add lookaheads to kernel items according to substitution table
      plalr.lookahead.add(I);
      I := I + 1;
//      //add LALRItems which have dot at end to reducableitemsSet, which will be later used in building reducetable (except S->. rules, will be added later)
//      if plalr.lr0.dot = LRuleTable.rules[plalr.lr0.rulenum].length then begin
//        reducableitemsSet.add(index,Integer(plalr));
//      end;
    end;
    //build closure from kernel items
    closureSet := closure(itemset);
//    S := closureSet.toString(true,LTokenNameTypeTable,LStateNameTypeTable,LRuleTable);
//    writeln(S);
    //generate possible jumps from closure set -> generate LR0 jump table
    itemsetLR0Jumps := TIntIntHashSet.create;
    closureSet.getNextReset;
    while closureSet.getNext(plalr) do begin
      dot := plalr.lr0.dot;
      rule := LRuleTable.rules[plalr.lr0.rulenum];
      if dot = rule.length then begin  //S -> E. - reduce => no jump
        if dot = 1 then begin  //S -> . - epsilon reduce => no jump, but...
          plalr := cloneLALR(plalr);
          itemset.add(plalr);  //add epsilon rule to kernel
//          reducableitemsSet.add(index,Integer(plalr));
          //add epsilon rule to lapropagate list of relevant kernel items
          plalr.lookahead.getNextReset;
          while plalr.lookahead.getNext(I) do begin
            if I > MAX_TERMINAL then begin
              if substTable.get(I,Integer(plalrParent)) then begin
                if plalrParent.lapropagate <> nil then begin
                  plalrParent.lapropagate.add(Integer(plalr));
                end
                else begin
                  plalrParent.lapropagate := TIntHashSet.create(Integer(plalr));
                end;
              end
              else begin
                //!TODO! assert error - we have lookahead propagation from item whose pointer doesn't point to valid PLALRItem
              end;
            end;
          end;
          plalr.bPropagated := true;
        end;
      end
      else begin  //S -> A.[symbol]B - shift on [symbol] => add [symbol],plalr to jump table
        symbol := rule.symbols[dot];
        hashset := TIntHashSet.create(Integer(plalr));
        if not itemsetLR0Jumps.add(symbol,Integer(hashset)) then begin
          hashset.Free;
          itemsetLR0Jumps.get(symbol,Integer(hashset));
          hashset.add(Integer(plalr));
        end;
      end;
    end;
    //for each symbol in itemsetLR0Jumps "table" generate new kernel set
    itemsetLR0Jumps.getNextReset;
    while itemsetLR0Jumps.getNext(symbol,Integer(hashset)) do begin
      newitemset := TItemSet.create;
      hashset.getNextReset;
      while hashset.getNext(Integer(plalr)) do begin
        new(pnewlalr);
        copyLALR(plalr,pnewlalr);
        pnewlalr.lr0.dot := pnewlalr.lr0.dot + 1;
        newitemset.add(pnewlalr);
//        plalr.bPropagated := true;  //NO EFFECT!!!
      end;
      if globalSet.add(newitemset,I) then begin
        //kernel set added to global set
      end
      else begin
        //same kernel set already exists
        olditemset := globalSet.getByIndex(I);
        newitemset.getNextReset;
        while newitemset.getNext(plalr) do begin  //newitemset cannot have epsilon rule in kernel
          new(pnewlalr);
          copyLALR(plalr,pnewlalr);
          if olditemset.add2(pnewlalr) then begin //zhadzuje bPropagated z false na true a ked nic neprida tak to tak aj zostane (uz sa nepouziva bPropagated)
            pnewlalr.bPropagated := false;
            //if (pnewlalr.lr0.dot < LRuleTable.rules[pnewlalr.lr0.rulenum].length) and (pnewlalr.lapropagate <> nil) then
            if pnewlalr.lapropagate <> nil then
              if propagateSet.add(Integer(pnewlalr)) then
                propagateQueue.enqueue(Integer(pnewlalr));
          end;
//          else
//            pnewlalr.bPropagated := true;
        end;
        newitemset.Free;
        newitemset := olditemset;
      end;
      //add to goto table - [itemset index],TIntIntHashSet containing (symbol,[itemset I]),...
      if not gotoTable.get(index,Integer(IIhashset)) then begin
        IIhashset := TIntIntHashSet.create;
        IIhashset.add(symbol,I);
        if not gotoTable.add(index,Integer(IIhashset)) then begin
          //!TODO! assert error - index not there (checked with get) and not possible to add, eh ??
        end;
      end
      else begin
        IIhashset.add(symbol,I);
      end;
      //process lookaheads, find those which are > MAX_TERMINAL and resolve lapropagate to them and remove them
      newitemset.getNextReset;
      while newitemset.getNext(plalr) do begin
//        hashset := TIntHashSet.create;
        lastbucket := -1;
        lastsetitem := nil;
        plalr.lookahead.getNextReset;
        while plalr.lookahead.getNext(I) do begin
          if I > MAX_TERMINAL then begin
            if substTable.get(I,Integer(plalrParent)) then begin
              if plalrParent.lapropagate <> nil then begin
                plalrParent.lapropagate.add(Integer(plalr));
              end
              else begin
                plalrParent.lapropagate := TIntHashSet.create(Integer(plalr));
              end;
            end
            else begin
              //!TODO! assert error - we have lookahead propagation from item whose pointer doesn't point to valid PLALRItem
            end;
            plalr.lookahead.remove(I);
            plalr.lookahead.FCurrentBucket := lastbucket;
            plalr.lookahead.FCurrentItem := lastsetitem;
          end
          else begin
//            hashset.add(I);
          end;
          lastbucket := plalr.lookahead.FCurrentBucket;
          lastsetitem := plalr.lookahead.FCurrentItem;
        end;
//        plalr.lookahead.Free;
//        plalr.lookahead := hashset;
      end;
    end;
    //removing extra lookahead information - dummy lookaheads (i.e. >MAX_TERMINAL) used in lookahead propagation
    itemset.getNextReset;
    while itemset.getNext(plalr) do begin
      //add LALRItems which have dot at end to reducableitemsSet, which will be later used in building reducetable (except S->. rules, will be added later)
      if plalr.lr0.dot = LRuleTable.rules[plalr.lr0.rulenum].length then begin
        new(pair);
        pair.x := index;
        pair.y := Integer(plalr);
        reducableitemsQueue.enqueue(Integer(pair));
      end;
      lastbucket := -1;
      lastsetitem := nil;
      plalr.lookahead.getNextReset;
      while plalr.lookahead.getNext(I) do begin
        if I > MAX_TERMINAL then begin
          plalr.lookahead.remove(I);
          plalr.lookahead.FCurrentBucket := lastbucket;
          plalr.lookahead.FCurrentItem := lastsetitem;
        end;
        lastbucket := plalr.lookahead.FCurrentBucket;
        lastsetitem := plalr.lookahead.FCurrentItem;
      end;
    end;

//    writeln(globalSet.toString(true,LTokenNameTypeTable,LStateNameTypeTable,LRuleTable));

    index := index + 1;
    itemsetLR0Jumps.getNextReset;
    while itemsetLR0Jumps.getNext(I,Integer(hashset)) do begin
      hashset.Free;
    end;
    itemsetLR0Jumps.Free;
    closureSet.Free;
    substTable.Free;

  until (index = globalSet.Fcount);

  //lookahead propagation along lapropagate lines
  I := 0;
  while not propagateQueue.isEmpty do begin
    plalr := PLALRItem(propagateQueue.pop);
    if plalr.lapropagate <> nil then begin  //v principe zbytocny check
//      writeln('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%');
//      writeln(plalr.lr0.rulenum,' , ',plalr.lr0.dot,' - ',I);
      I := I + 1;
      plalr.lapropagate.getNextReset;
      while plalr.lapropagate.getNext(Integer(pnewlalr)) do begin
        b := pnewlalr.lookahead.add(plalr.lookahead);
        if b then begin
          pnewlalr.bPropagated := false;
          //if (pnewlalr.lr0.dot < LRuleTable.rules[pnewlalr.lr0.rulenum].length) and (pnewlalr.lapropagate <> nil) then
          if pnewlalr.lapropagate <> nil then
            if propagateSet.add(Integer(pnewlalr)) then
              propagateQueue.enqueue(Integer(pnewlalr));
        end;
      end;
    end;
    propagateSet.remove(Integer(plalr));
  end;
  propagateSet.Free;
  propagateQueue.Free;
(*
  repeat
    bAdded := false;
    index := 0;
    I := 0;
    while index < globalSet.FCount do begin
      itemset := globalSet.getByIndex(index);
      itemset.getNextReset;
      while itemset.getNext(plalr) do begin
        begin //if not plalr.bPropagated then begin   //changed to trivial iteration till no change, due to bug in bPropagated
          if plalr.lapropagate <> nil then begin
            writeln('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%');
            writeln(index,' - ',I);
            I := I + 1;
            plalr.lapropagate.getNextReset;
            while plalr.lapropagate.getNext(Integer(pnewlalr)) do begin
              b := pnewlalr.lookahead.add(plalr.lookahead);
              if b then begin
                pnewlalr.bPropagated := false;
                bAdded := true;
              end;
            end;
          end;
          plalr.bPropagated := true;
        end;
      end;
      index := index + 1;
    end;
  until bAdded = false;
  *)
{$IFDEF DEBUG2}
  S := globalSet.toString(true,LTokenNameTypeTable,LStateNameTypeTable,LRuleTable);
  writeln(S);
//  Form1.Memo1.Text := Form1.Memo1.Text + S;
{$ENDIF}
(*
  writeln('**reducableitemsSet: ');
  reducableitemsQueue.peekNextReset;
  while reducableitemsQueue.peekNext(Integer(pair)) do begin
    writeln(IntToStr(pair.x) + ' - ' + IntToStr(pair.y));
  end;
*)
//  writeln(reducableitemsSet.toString(true));

  //add to reduce table [itemset I].<1>...<n>.symbol.<1>.....<n>.rulenum
  reduceTable := TIntIntHashSet.create;
//  reducableitemsSet.getNextReset;
//  while reducableitemsSet.getNext(I,Integer(plalr)) do begin
  while not reducableitemsQueue.isEmpty do begin
    pair := PPair(reducableitemsQueue.dequeue);
    I := pair.x;
    plalr := PLALRItem(pair.y);
    dispose(pair);
    plalr.lookahead.getNextReset;
    while plalr.lookahead.getNext(symbol) do begin
      if gotoTable.get(I,Integer(IIhashsetTemp)) then
        if IIhashsetTemp.get(symbol,J) then begin
          //shift/reduce conflict
          conflictStates.add(I,conflictStates.getSize);
{$IFDEF DEBUG1}
          writeln('SHIFT/REDUCE conflict in Parser');
          write('![State ' + IntToStr(I) + '] on ' + LTokenNameTypeTable.findByValue(symbol,b) + ' shift to [State ' + IntToStr(J) + ']' + EOL);
          write('![State ' + IntToStr(I) + '] on ' + LTokenNameTypeTable.findByValue(symbol,b) + ' reduce by rule ' + LRuleTable.toString(true,LTokenNameTypeTable,LStateNameTypeTable,plalr.lr0.rulenum,plalr.lr0.rulenum));
{$ENDIF}
          rule := LRuleTable.getRuleByIndex(plalr.lr0.rulenum);
            if (LLexer <> nil) and (LLexer.LFileReader <> nil) then begin
              FLog.addSFR2(LLexer.LFileReader,rule.pos - LLexer.LFileReader.getPosition,W_PB_SHIFT_REDUCE_SHIFT,IntToStr(I),LTokenNameTypeTable.findByValue(symbol,b),IntToStr(J));
              FLog.addSFR2(LLexer.LFileReader,rule.pos - LLexer.LFileReader.getPosition,W_PB_SHIFT_REDUCE_REDUCE,IntToStr(I),LTokenNameTypeTable.findByValue(symbol,b),IntToStr(plalr.lr0.rulenum));
            end
            else begin
              FLog.add('',rule.pos,W_PB_SHIFT_REDUCE_SHIFT,IntToStr(I),LTokenNameTypeTable.findByValue(symbol,b),IntToStr(J));
              FLog.add('',rule.pos,W_PB_SHIFT_REDUCE_REDUCE,IntToStr(I),LTokenNameTypeTable.findByValue(symbol,b),IntToStr(plalr.lr0.rulenum));
            end;
        end;
      if not reduceTable.get(I,Integer(IIhashset)) then begin
        IIhashset := TIntIntHashSet.create;
        if not IIhashset.add(symbol,Integer(TIntHashSet.create(plalr.lr0.rulenum))) then begin
            //!TODO! assert error - no get and still no add ??
        end;
        if not reduceTable.add(I,Integer(IIhashset)) then begin
          //!TODO! assert error - no get and still no add ??
        end;
      end
      else begin
        if not IIhashset.get(symbol,Integer(hashset)) then begin
          if not IIhashset.add(symbol,Integer(TIntHashSet.create(plalr.lr0.rulenum))) then begin
            //!TODO! assert error - no get and still no add ??
          end;
        end
        else begin  //[itemset I],symbol exists
          hashset.add(plalr.lr0.rulenum);
          //reduce/reduce conflict
          if hashset.getSize >= 2 then begin
            conflictStates.add(I,conflictStates.getSize);
{$IFDEF DEBUG1}
            writeln('REDUCE/REDUCE conflict in Parser');
{$ENDIF}
            hashset.getFirst(J);
            if J = plalr.lr0.rulenum then
              hashset.getNext(J);
{$IFDEF DEBUG1}
            write('![State ' + IntToStr(I) + '] on ' + LTokenNameTypeTable.findByValue(symbol,b) + ' reduce by rule ' + LRuleTable.toString(true,LTokenNameTypeTable,LStateNameTypeTable,J,J));
            write('![State ' + IntToStr(I) + '] on ' + LTokenNameTypeTable.findByValue(symbol,b) + ' reduce by rule ' + LRuleTable.toString(true,LTokenNameTypeTable,LStateNameTypeTable,plalr.lr0.rulenum,plalr.lr0.rulenum));
{$ENDIF}
            rule := LRuleTable.getRuleByIndex(plalr.lr0.rulenum);
            if (LLexer <> nil) and (LLexer.LFileReader <> nil) then begin
              FLog.addSFR2(LLexer.LFileReader,rule.pos - LLexer.LFileReader.getPosition,W_PB_REDUCE_REDUCE_ORIG,IntToStr(I),LTokenNameTypeTable.findByValue(symbol,b),IntToStr(J));
              FLog.addSFR2(LLexer.LFileReader,rule.pos - LLexer.LFileReader.getPosition,W_PB_REDUCE_REDUCE_REDUCE,IntToStr(I),LTokenNameTypeTable.findByValue(symbol,b),IntToStr(plalr.lr0.rulenum));
            end
            else begin
              FLog.add('',rule.pos,W_PB_REDUCE_REDUCE_ORIG,IntToStr(I),LTokenNameTypeTable.findByValue(symbol,b),IntToStr(J));
              FLog.add('',rule.pos,W_PB_REDUCE_REDUCE_REDUCE,IntToStr(I),LTokenNameTypeTable.findByValue(symbol,b),IntToStr(plalr.lr0.rulenum));
            end;
          end;
        end;
      end;
    end;
  end;

  reducableitemsQueue.Free;
//  reducableitemsSet.Free;

  //end - let's clear setofitemsets - we don't need this globalset anymore

{$IFDEF DEBUG0}
    writeln('LALR States: '+ IntToStr(globalSet.FCount));
{$ENDIF}
{$IFDEF LOG0}
    if LogForm <> nil then begin
      LogForm.AddLine('LALR States: '+ IntToStr(globalSet.FCount));
    end;
{$ENDIF}
//  globalSet.Free;

  //reduceTable and gotoTable prepared, create TParser
  FParser := TParser.create(LLexer,LTokenNameTypeTable,LStateNameTypeTable,LRuleTable,gotoTable,reducetable);
  FParser.FGlobalSet := globalSet;
  FParser.FConflictStates := conflictStates;

  result := true;
end;

/// destruktor uvolnujuci objekt z pamate
destructor TParserBuilder.Destroy;
var HashSet: TIntHashSet;
    key: Integer;
begin
  FFirst.getNextReset;
  while FFirst.getNext(key,Integer(HashSet)) do begin
    HashSet.Free;
  end;
  FFirst.Free;
  FFollow.getNextReset;
  while FFollow.getNext(key,Integer(HashSet)) do begin
    HashSet.Free;
  end;
  FFollow.Free;
  FLog.Free;
  inherited Destroy;
end;

/// funkcia, ktora vrati mnozinu terminalov ktore budu odvodene ako prve zlava zo vstupnej vetnej formy
/// ako prve zlava, tato funkcia vyuziva uz vybudovanu mnozinu First pre kazdy neterminal;
/// ak vo vstupnej vetnej forme ako prvy symbol terminal, tak tento je aj vystupom algoritmu
/// ak je vo vstupnej vetnej forme ako prvy neterminal, tak jeho First mnozina bude na vystupe,
/// ak obsahuje aj epsilon, tak sa do mnoziny prida aj mnozina First zvysku vetnej formy
function TParserBuilder.First(symbols: TQueueStack): TIntHashSet;
var HashSet: TIntHashSet;
    b: Boolean;
    symbol: Integer;
begin
  result := TIntHashSet.create;
  if symbols.isEmpty then
    Exit;
  symbols.peekNextReset;
  while symbols.peekNext(symbol) do begin
    if symbol = L_EPSILON then begin
    end
    else if symbol < MAX_TERMINAL then begin
      result.add(symbol);
      Exit;
    end
    else begin
      b := FFirst.get(symbol,Integer(HashSet));
      if b then begin
        result.addNonEpsilon(HashSet);
        if not HashSet.contains(L_EPSILON) then
          Exit;
      end
      else begin
        //result := nil; //UNKNOWN SYMBOL - symbol
        Exit;
      end;
    end;
  end; //until (not HashSet.contains(L_EPSILON));
  if HashSet.contains(L_EPSILON) then
    result.add(L_EPSILON);
end;

/// funkcia vracajuca mnozinu First, pre dany nerminal, tato obsahuje pre kazdy neterminal mnozinu
/// terminalnych symbolov, ktore sa odvodia z tohto neterminalu ako prve zlava
function TParserBuilder.First(symbol: Integer): TIntHashSet;
var HashSet: TIntHashSet;
    b: Boolean;
begin
  result := TIntHashSet.create;
  if symbol = L_EPSILON then begin
    result.add(symbol);
//    Exit;
  end
  else if symbol < MAX_TERMINAL then begin
    result.add(symbol);
  end
  else begin
    b := FFirst.get(symbol,Integer(HashSet));
    if b then begin
      result.add(HashSet);
    end
    else begin
      //result := nil;  //UNKNOWN SYMBOL - symbol
    end;
  end;
end;

/// funkcia vracajuca mnozinu Follow pre dany neterminal, tato mnozina obsahuje terminaly,
/// ktore mozu nasledovat vo vetnej forme po danom neterminale
function TParserBuilder.Follow(symbol: Integer): TIntHashSet;
var HashSet: TIntHashSet;
    b: Boolean;
begin
  result := TIntHashSet.create;
  if symbol < MAX_TERMINAL then begin
    Exit;
  end
  else begin
    b := FFollow.get(symbol,Integer(HashSet));
    if b then begin
      result.add(HashSet);
    end
    else begin
      //result := nil;
    end;
  end;
end;

/// funkcia pocitajuca mnoziny First, pre kazdy nerminal, tato mnozina obsahuje pre kazdy neterminal mnozinu
/// terminalnych symbolov, ktore sa odvodia z tohto neterminalu ako prve zlava
function TParserBuilder.calculateFirst: boolean;
var I,J: Integer;
    lastrule,lastsymbol: Integer;
    head,symbol: Integer;
    b, bAdded: Boolean;
    HashSet,HashSet2: TIntHashSet;
    hs: Integer;
begin
  result := false;
  if not FFirst.isEmpty then begin
    FFirst.getNextReset;
    while FFirst.getNext(I,Integer(HashSet)) do begin
      HashSet.free;
    end;
    FFirst.free;
    FFirst := TIntIntHashSet.create;
  end;
  lastrule := Pred(LRuleTable.count);
  repeat
    bAdded := false;
    for I := 0 to lastrule do begin
      head := LRuleTable.rules[I].symbols[0];
      lastsymbol := Pred(LRuleTable.rules[I].length);
      b := FFirst.get(head,Integer(HashSet));
      if not b then begin
        HashSet := TIntHashSet.create;
        FFirst.add(head,Integer(HashSet));
      end;
      if lastsymbol = 0 then begin
        bAdded := HashSet.add(L_EPSILON) or bAdded;
      end
      else begin
        for J := 1 to lastsymbol do begin
          symbol := LRuleTable.rules[I].symbols[J];
          if (symbol < MAX_TERMINAL) then begin
            bAdded := HashSet.add(symbol) or bAdded;
            break;
          end
          else begin
            b := FFirst.get(symbol,Integer(HashSet2));
            if not b then begin
              HashSet2 := TIntHashSet.create;
              FFirst.add(symbol,Integer(HashSet2));
            end;
            bAdded := HashSet.addNonEpsilon(HashSet2) or bAdded;
            if not HashSet2.contains(L_EPSILON) then
              break
            else if J = lastsymbol then  //last symbol of the rule and still contains Epsilon => head symbol should also have it
              bAdded := HashSet.add(L_EPSILON) or bAdded;
          end;
        end;
      end;
    end;
  until not bAdded;

{$IFDEF DEBUG2}
  writeln(FFirst.toString(true));
  FFirst.getNextReset;
  while FFirst.getNext(I,J) do begin
    writeln('{' + IntToStr(I) + '}' + EOL + TIntHashSet(J).toString(true));
  end;
{$ENDIF}
  result := true;
end;

/// funkcia pocitajuca mnoziny Follow, pre kazdy nerminal, tato mnozina obsahuje pre kazdy neterminal mnozinu
/// terminalnych symbolov, ktore mozu nasledovat po nom vo vetnej forme
function TParserBuilder.calculateFollow: Boolean;
var I,J: Integer;
    lastrule,lastsymbol: Integer;
    head,symbol: Integer;
    b, bAdded: Boolean;
    HashSet,HashSet2: TIntHashSet;
    hs: Integer;
    QS: TQueueStack;
begin
  result := false;
  if not FFollow.isEmpty then begin
    FFollow.getNextReset;
    while FFollow.getNext(I,Integer(HashSet)) do begin
      HashSet.free;
    end;
    FFollow.free;
    FFollow := TIntIntHashSet.create;
  end;
  lastrule := Pred(LRuleTable.count);
  repeat
    bAdded := false;
    for I := 0 to lastrule do begin
      head := LRuleTable.rules[I].symbols[0];
      lastsymbol := Pred(LRuleTable.rules[I].length);
      for J := 1 to lastsymbol do begin
        symbol := LRuleTable.rules[I].symbols[J];
        if (symbol < MAX_TERMINAL) then begin
//          break;
        end
        else begin
          b := FFollow.get(symbol,Integer(HashSet));
          if not b then begin
            HashSet := TIntHashSet.create;
            FFollow.add(symbol,Integer(HashSet));
          end;
          QS := TQueueStack.create;
          QS.push(LRuleTable.rules[I].symbols,J+1,lastSymbol);
          HashSet2 := First(QS);
          QS.Free;
          bAdded := HashSet.addNonEpsilon(HashSet2) or bAdded;
          if (j = lastsymbol) or (HashSet2.contains(L_EPSILON)) then begin
            HashSet2.Free;
            b := FFollow.get(head,Integer(HashSet2));
            if not b then begin
              HashSet2 := TIntHashSet.create;
              FFollow.add(head,Integer(HashSet2))
            end;
            bAdded := HashSet.add(HashSet2) or bAdded;
          end
          else begin
            HashSet2.Free;
          end;
        end;
      end;
    end;
  until not bAdded;

{$IFDEF DEBUG2}
  writeln(FFollow.toString(true));
  FFollow.getNextReset;
  while FFollow.getNext(I,J) do begin
    writeln('{' + IntToStr(I) + '}' + EOL + TIntHashSet(J).toString(true));
  end;
{$ENDIF}
  result := true;
end;

(*
function TParserBuilder.lr0hash(item: TLR0Item): Integer;
begin
  result := (item.rulenum - MAX_TERMINAL) * (item.dot + 1) + MAX_TERMINAL;
end;
*)

/// procedura na zkopirovanie recordu popisujuceho LALR polozku (TLALRItem) z _from do _to
procedure TParserBuilder.copyLALR(_from: PLALRItem; _to: PLALRItem);
begin
  if _from = nil then begin
    _to := nil;
  end
  else begin
    _to.lr0 := _from.lr0;
    if _from.lookahead <> nil then
      _to.lookahead := _from.lookahead.clone
    else
      _to.lookahead := nil;
    if _from.lapropagate <> nil then
      _to.lapropagate := _from.lapropagate.clone
    else
      _to.lapropagate := nil;
    _to.bPropagated := _from.bPropagated;
  end;
end;

/// funkcia na vytvorenie presnej kopie (klonu) recordu popisujuceho LALR polozku (TLALRItem) - _from
function TParserBuilder.cloneLALR(_from: PLALRItem): PLALRItem;
begin
  if _from = nil then begin
    result := nil;
  end
  else begin
    new(result);
    result.lr0 := _from.lr0;
    if _from.lookahead <> nil then
      result.lookahead := _from.lookahead.clone
    else
      result.lookahead := nil;
    if _from.lapropagate <> nil then
      result.lapropagate := _from.lapropagate.clone
    else
      result.lapropagate := nil;
    result.bPropagated := _from.bPropagated;
  end;
end;

/// funkcia pocitajuca uzaver vstupnej mnoziny poloziek
/// vrati novu mnozinu poloziek tvoriacu uzaver vstupnej mnoziny (obvykle je vstupom jadro mnoziny)
function TParserBuilder.closure(itemset: TItemSet): TItemSet;
var key,data: Integer;
    h: Integer;
    item,newitem: PLALRItem;
    Q: TQueueStack;
    I: Integer;
    X: Integer;
    rule: TRule;
    restofRHS: TQueueStack;
    b: Boolean;
    lookaheadHashSet: TIntHashSet;
    ruleIndexSet: TIntHashSet;
    counter: Integer;
begin
  // L.XB .... X =>* A.ny - podobne ako closure ale treba si pamatat aj kus pravidla za X (ako stack) a to podavat dalej a pridavat k tomu (mozne zrychlenie)
  if (itemset = nil) or (itemset.isEmpty) then begin
    result := nil;//TIntIntHashSet.create;
  end
  else begin
    result := TItemSet.create;
    Q := TQueueStack.create;
    itemset.getNextReset;
    while itemset.getNext(item) do begin
      new(newitem);
      copyLALR(item,newitem);
//      newitem.bPropagated := false;
      Q.push(Integer(newitem));
      result.add(newitem);
    end;
    //process all LALR items in FIFO order and add new at the back
    counter := 0;
//    Q.peekNextReset;
    while not Q.isEmpty do begin//Q.peekNext(I) do begin
      I := Q.dequeue;
      counter := counter + 1;
//      writeln('Count: ' + IntToStr(counter));
//      writeln('###now###' + EOL + result.toString(true,LTokenNameTypeTable,LStateNameTypeTable,LRuleTable));
      item := PLALRItem(I);
      rule := LRuleTable.rules[item.lr0.rulenum];
      if item.lr0.dot < rule.length then begin
        X := rule.symbols[item.lr0.dot];
        if X > MAX_TERMINAL then begin
          restofRHS := TQueueStack.create;
          restofRHS.push(rule.symbols,Succ(item.lr0.dot),Pred(rule.length));
          if not restofRHS.isEmpty then begin
            lookaheadHashSet := First(restofRHS);  //lookahead spontaneous symbols (all except Epsilon)
            if lookaheadHashSet.contains(L_EPSILON) then begin  //lookahead propagate (& spontaneous all except Epsilon)
              lookaheadHashSet.remove(L_EPSILON);
              if item.lookahead <> nil then
                lookaheadHashSet.add(item.lookahead);
            end
            else begin  //lookahead spontaneous
            end;
          end
          else begin
            lookaheadHashSet := First(restofRHS);  //lookahead spontaneous symbols (all except Epsilon)
            if item.lookahead <> nil then
              lookaheadHashSet.add(item.lookahead);  //lookahead propagate
          end;
          //for all rules X -> Gamma add X -> .Gamma as item
          ruleIndexSet := LRuleTable.getRulesByHead(X);
          if ruleIndexSet <> nil then begin
            ruleIndexSet.getNextReset;
            while ruleIndexSet.getNext(h) do begin
              new(newitem);
              newitem.lr0.rulenum := h;
              newitem.lr0.dot := 1;
  //            newitem.lookahead := nil;
  //            newitem.lookahead := TIntHashSet.create;//lookaheadHashSet.clone;
  //            newitem.lookahead.add(L_EOF);
              newitem.lookahead := TIntHashSet.create;
              newitem.lookahead.add(lookaheadHashSet);
              if item.lapropagate <> nil then
                newitem.lapropagate := item.lapropagate.clone
              else
                newitem.lapropagate := nil;
              newitem.bPropagated := item.bPropagated;
              I := result.FCount;
              b := result.add(newitem);
              if (b) then //and (result.FCount > I) then
                Q.pushEx(Integer(newitem))
  //            else
  //              dispose(newitem);
            end;
          end;
          lookaheadHashSet.free;
          restofRHS.free;
        end;
      end
      else if item.lr0.dot = rule.length then begin
        if rule.length = 1 then begin
          //S -> Epsilon aka S ::= .
        end;
      end
      else begin
        //!TODO! error invalid dot positon in rule
      end;
    end;

    Q.Free;

    //copy all from queue to result itemset
//      result.add(newitem);
  end;
end;

/// funkcia pocitajuca uzaver vstupnej mnoziny poloziek
/// vrati novu mnozinu poloziek tvoriacu uzaver vstupnej mnoziny (obvykle je vstupom jadro mnoziny)
function TParserBuilder.closure3(itemset: TItemSet): TItemSet;
var key,data: Integer;
    h: Integer;
    item,newitem: PLALRItem;
    Q: TQueueStack;
    I: Integer;
    X: Integer;
    rule: TRule;
    restofRHS: TQueueStack;
    b: Boolean;
    lookaheadHashSet: TIntHashSet;
    ruleIndexSet: TIntHashSet;
    counter: Integer;
begin
  if (itemset = nil) or (itemset.isEmpty) then begin
    result := nil;//TIntIntHashSet.create;
  end
  else begin
    result := TItemSet.create;
    Q := TQueueStack.create;
    itemset.getNextReset;
    while itemset.getNext(item) do begin
      new(newitem);
      copyLALR(item,newitem);
//      newitem.bPropagated := false;
      Q.push(Integer(newitem));
      result.add(newitem);
    end;
    //process all LALR items in FIFO order and add new at the back
    counter := 0;
//    Q.peekNextReset;
    while not Q.isEmpty do begin//Q.peekNext(I) do begin
      I := Q.dequeue;
      counter := counter + 1;
//      writeln('Count: ' + IntToStr(counter));
//      writeln('###now###' + EOL + result.toString(true,LTokenNameTypeTable,LStateNameTypeTable,LRuleTable));
      item := PLALRItem(I);
      rule := LRuleTable.rules[item.lr0.rulenum];
      if item.lr0.dot < rule.length then begin
        X := rule.symbols[item.lr0.dot];
        if X > MAX_TERMINAL then begin
          restofRHS := TQueueStack.create;
          restofRHS.push(rule.symbols,Succ(item.lr0.dot),Pred(rule.length));
          if not restofRHS.isEmpty then begin
            lookaheadHashSet := First(restofRHS);  //lookahead spontaneous symbols (all except Epsilon)
            if lookaheadHashSet.contains(L_EPSILON) then begin  //lookahead propagate (& spontaneous all except Epsilon)
              lookaheadHashSet.remove(L_EPSILON);
              if item.lookahead <> nil then
                lookaheadHashSet.add(item.lookahead);
            end
            else begin  //lookahead spontaneous
            end;
          end
          else begin
            lookaheadHashSet := First(restofRHS);  //lookahead spontaneous symbols (all except Epsilon)
            if item.lookahead <> nil then
              lookaheadHashSet.add(item.lookahead);  //lookahead propagate
          end;
          //for all rules X -> Gamma add X -> .Gamma as item
          ruleIndexSet := LRuleTable.getRulesByHead(X);
          if ruleIndexSet <> nil then begin
            ruleIndexSet.getNextReset;
            while ruleIndexSet.getNext(h) do begin
              new(newitem);
              newitem.lr0.rulenum := h;
              newitem.lr0.dot := 1;
  //            newitem.lookahead := nil;
  //            newitem.lookahead := TIntHashSet.create;//lookaheadHashSet.clone;
  //            newitem.lookahead.add(L_EOF);
              newitem.lookahead := TIntHashSet.create;
              newitem.lookahead.add(lookaheadHashSet);
              if item.lapropagate <> nil then
                newitem.lapropagate := item.lapropagate.clone
              else
                newitem.lapropagate := nil;
              newitem.bPropagated := item.bPropagated;
              I := result.FCount;
              b := result.add(newitem);
              if (b) then //and (result.FCount > I) then
                Q.pushEx(Integer(newitem));
  //            else
  //              dispose(newitem);
            end;
          end;
          lookaheadHashSet.free;
          restofRHS.free;
        end;
      end
      else if item.lr0.dot = rule.length then begin
        if rule.length = 1 then begin
          //S -> Epsilon aka S ::= .
        end;
      end
      else begin
        //!TODO! error invalid dot positon in rule
      end;
    end;

    Q.Free;

    //copy all from queue to result itemset
//      result.add(newitem);
  end;
end;

/// funkcia pocitajuca uzaver vstupnej mnoziny poloziek
/// vrati novu mnozinu poloziek tvoriacu uzaver vstupnej mnoziny (obvykle je vstupom jadro mnoziny)
function TParserBuilder.closure2(itemset: TItemSet): TItemSet;
var key,data: Integer;
    h: Integer;
    item,newitem: PLALRItem;
    Q: TQueueStack;
    I: Integer;
    X: Integer;
    rule: TRule;
    restofRHS: TQueueStack;
    b: Boolean;
    lookaheadHashSet: TIntHashSet;
    ruleIndexSet: TIntHashSet;
    bAdded: Boolean;
    counter: Integer;
begin
  if (itemset = nil) or (itemset.isEmpty) then begin
    result := nil;//TIntIntHashSet.create;
  end
  else begin
    result := TItemSet.create;
//    Q := TQueueStack.create;
    itemset.getNextReset;
    while itemset.getNext(item) do begin
      new(newitem);
      copyLALR(item,newitem);
      newitem.bPropagated := false;
//      Q.push(Integer(newitem));
      result.add(newitem);
    end;
    //process all LALR items in FIFO order and add new at the back
//    Q.peekNextReset;
    counter := 0;
    I := 0;
    repeat
    I := I + 1;
(*    writeln('Repeat #' + IntToStr(I));
    writeln('Repeat #' + IntToStr(I));
    writeln('Repeat #' + IntToStr(I));
    writeln('Repeat #' + IntToStr(I));
    writeln('Repeat #' + IntToStr(I));*)
    bAdded := false;
    result.getNextReset;
    while result.getNext(item) do begin
      counter := counter + 1;
{$IFDEF DEBUG1}
      writeln('Count: ' + IntToStr(counter));
{$ENDIF}
//      writeln(result.toString(true,LTokenNameTypeTable,LStateNameTypeTable,LRuleTable));
//      item := PLALRItem(I);
      rule := LRuleTable.rules[item.lr0.rulenum];
      if item.lr0.dot < rule.length then begin
        X := rule.symbols[item.lr0.dot];
        restofRHS := TQueueStack.create;
        restofRHS.push(rule.symbols,Succ(item.lr0.dot),Pred(rule.length));
        if not restofRHS.isEmpty then begin
//          lookaheadHashSet := TIntHashSet.create;
//          lookaheadHashSet.add(First(restofRHS));
          lookaheadHashSet := First(restofRHS);  //lookahead spontaneous symbols (all except Epsilon)
          if lookaheadHashSet.contains(L_EPSILON) then begin  //lookahead propagate (& spontaneous all except Epsilon)
            lookaheadHashSet.remove(L_EPSILON);
            if item.lookahead <> nil then
              lookaheadHashSet.add(item.lookahead);
          end
          else begin  //lookahead spontaneous
          end;
        end
        else begin
//          lookaheadHashSet := TIntHashSet.create;
          lookaheadHashSet := First(restofRHS);  //lookahead spontaneous symbols (all except Epsilon)
          if item.lookahead <> nil then
            lookaheadHashSet.add(item.lookahead);  //lookahead propagate
        end;
        if X > MAX_TERMINAL then begin
          //for all rules X -> Gamma add X -> .Gamma as item
          ruleIndexSet := LRuleTable.getRulesByHead(X);
          if ruleIndexSet <> nil then begin
            ruleIndexSet.getNextReset;
            while ruleIndexSet.getNext(h) do begin
              new(newitem);
              newitem.lr0.rulenum := h;
              newitem.lr0.dot := 1;
  //            newitem.lookahead := nil;
  //            newitem.lookahead := TIntHashSet.create;//lookaheadHashSet.clone;
  //            newitem.lookahead.add(L_EOF);
              newitem.lookahead := TIntHashSet.create;
              newitem.lookahead.add(lookaheadHashSet);
              if item.lapropagate <> nil then
                newitem.lapropagate := item.lapropagate.clone
              else
                newitem.lapropagate := nil;
              newitem.bPropagated := false;
              b := result.add(newitem);
  //            if b then
  //              Q.push(Integer(newitem))
  //            else
  //              dispose(newitem);
              newitem := nil;
              bAdded := bAdded or b;
            end;
          end;
        end;
        lookaheadHashSet.free;
        restofRHS.free;
      end
      else if item.lr0.dot = rule.length then begin
        if rule.length = 1 then begin
          //S -> Epsilon aka S ::= .
        end;
      end
      else begin
        //!TODO! error invalid dot positon in rule
      end;
    end;
    until (bAdded = false)
//    Q.Free;

    //copy all from queue to result itemset
//      result.add(newitem);
  end;
end;

/// funkcia vracajuca vybudovany LALR parser
function TParserBuilder.getParser: TParser;
begin
  result := FParser;
end;

{ TItemSet }

/// kostruktor na vytvornie objektu popisujuceho mnozinu poloziek so vstupnymi parametrami;
/// cap je mnozstvo bucketov v internej hasovacej tabulke, icap je velkost jedneho bucketu
constructor TItemSet.create(cap, icap: Integer);
var I: Integer;
begin
  inherited create;
  FCount := 0;
  FCapacity := cap;
  SetLength(FBuckets,FCapacity);
  for I := 0 to Pred(FCapacity) do begin
    FBuckets[I] := nil;
  end;
end;

/// kostruktor na vytvornie objektu popisujuceho mnozinu poloziek
constructor TItemSet.create;
begin
  create(16,4);
end;

/// funkcia na pridanie LR0 polozky do mnoziny poloziek,
/// vrati true ak v nej taka LR0 polozka este nebola, inak false
function TItemSet.add(lr0: TLR0Item): Boolean;
var h: Integer;
    item: PItemSetItem;
begin
  result := false;
  h := hash(lr0);
  item := FBuckets[h];
  if item <> nil then begin
    while (item.next <> nil) and ((item.lalr.lr0.rulenum <> lr0.rulenum) or (item.lalr.lr0.dot <> lr0.dot)) do begin
      item := item.next;
    end;
    if (item.lalr.lr0.rulenum <> lr0.rulenum) or (item.lalr.lr0.dot <> lr0.dot) then begin
      item.next := new(PItemSetItem);
      item.next.lalr := new(PLALRItem);
      item.next.lalr.lr0 := lr0;
      item.next.lalr.lookahead := nil;
      item.next.lalr.lapropagate := nil;
      item.next.lalr.bPropagated := false;
      item.next.next := nil;
      FCount := FCount + 1;
      result := true;
    end
    else begin
      result := false;  //taky uz mame
    end;
  end
  else begin
    FBuckets[h] := new(PItemSetItem);
    FBuckets[h].lalr := new(PLALRItem);
    FBuckets[h].lalr.lr0 := lr0;
    FBuckets[h].lalr.lookahead := nil;
    FBuckets[h].lalr.lapropagate := nil;
    FBuckets[h].lalr.bPropagated := false;
    FBuckets[h].next := nil;
    FCount := FCount + 1;
    result := true;
  end;
end;

/// funkcia na odstranenie LR0 polozky z mnoziny poloziek,
/// vrati true ak tam taka polozka bola a bola odstranena, inak false
function TItemSet.remove(lr0: TLR0Item): Boolean;
var h: Integer;
    previtem,item: PItemSetItem;
begin
  result := false;
  h := hash(lr0);
  previtem := nil;
  item := FBuckets[h];
  if item <> nil then begin
    while (item.next <> nil) and ((item.lalr.lr0.rulenum <> lr0.rulenum) or (item.lalr.lr0.dot <> lr0.dot)) do begin
      previtem := item;
      item := item.next;
    end;
    if (item.lalr.lr0.rulenum <> lr0.rulenum) or (item.lalr.lr0.dot <> lr0.dot) then begin
    end
    else begin
      if previtem = nil then
        FBuckets[h] := item.next
      else begin
        previtem.next := item.next;
      end;
      if item.lalr.lookahead <> nil then
        item.lalr.lookahead.Free;
      if item.lalr.lapropagate <> nil then
        item.lalr.lapropagate.Free;
      if item = FCurrentItem then
        FCurrentItem := nil;
      dispose(item.lalr);
      dispose(item);
      FCount := FCount - 1;
      result := true;  //taky uz mame
    end;
  end;
end;

/// funkcia na pridanie LALR polozky do mnoziny poloziek,
/// vrati true ak v nej taka LALR polozka este nebola, inak false
function TItemSet.add(lalr: TLALRItem): Boolean;
var h: Integer;
    item: PItemSetItem;
    b: Boolean;
begin
  result := false;
  h := hash(lalr.lr0);
  item := FBuckets[h];
  if item <> nil then begin
    while (item.next <> nil) and ((item.lalr.lr0.rulenum <> lalr.lr0.rulenum) or (item.lalr.lr0.dot <> lalr.lr0.dot)) do begin
      item := item.next;
    end;
    if (item.lalr.lr0.rulenum <> lalr.lr0.rulenum) or (item.lalr.lr0.dot <> lalr.lr0.dot) then begin
      item.next := new(PItemSetItem);
      item.next.lalr.lr0 := lalr.lr0;
      if lalr.lookahead <> nil then
        item.next.lalr.lookahead := lalr.lookahead.clone
      else
        item.next.lalr.lookahead := nil;
      if lalr.lapropagate <> nil then
        item.next.lalr.lapropagate := lalr.lapropagate.clone
      else
        item.next.lalr.lapropagate := nil;
      item.next.lalr.bPropagated := lalr.bPropagated;
      item.next.next := nil;
      FCount := FCount + 1;
      result := true;
    end
    else begin
      b := false;
      if lalr.lookahead <> nil then begin
        if item.next.lalr.lookahead <> nil then begin
          b := item.next.lalr.lookahead.add(lalr.lookahead);
        end
        else begin
          item.next.lalr.lookahead := lalr.lookahead.clone;
          b := true;
        end;
      end;
      if lalr.lapropagate <> nil then begin
        if item.next.lalr.lapropagate <> nil then
          b := item.next.lalr.lapropagate.add(lalr.lapropagate) or b
        else begin
          item.next.lalr.lapropagate := lalr.lapropagate.clone;
          b := true;
        end;
      end;
      item.next.lalr.bPropagated := lalr.bPropagated;
      result := b;
    end;
  end
  else begin
    FBuckets[h] := new(PItemSetItem);
    FBuckets[h].lalr := new(PLALRItem);
    FBuckets[h].lalr.lr0 := lalr.lr0;
      if lalr.lookahead <> nil then
        FBuckets[h].lalr.lookahead := lalr.lookahead.clone
      else
        FBuckets[h].lalr.lookahead := nil;
      if lalr.lapropagate <> nil then
        FBuckets[h].lalr.lapropagate := lalr.lapropagate.clone
      else
        FBuckets[h].lalr.lapropagate := nil;
    FBuckets[h].lalr.bPropagated := lalr.bPropagated;
    FBuckets[h].next := nil;
    FCount := FCount + 1;
    result := true;
  end;
end;

(*
function TItemSet.add2(lalr: TLALRItem): Boolean;
var h: Integer;
    item: PItemSetItem;
    b: Boolean;
begin
  result := false;
  h := hash(lalr.lr0);
  item := FBuckets[h];
  if item <> nil then begin
    while (item.next <> nil) and ((item.lalr.lr0.rulenum <> lalr.lr0.rulenum) or (item.lalr.lr0.dot <> lalr.lr0.dot)) do begin
      item := item.next;
    end;
    if (item.lalr.lr0.rulenum <> lalr.lr0.rulenum) or (item.lalr.lr0.dot <> lalr.lr0.dot) then begin
      item.next := new(PItemSetItem);
      item.next.lalr.lr0 := lalr.lr0;
      if lalr.lookahead <> nil then
        item.next.lalr.lookahead := lalr.lookahead.clone
      else
        item.next.lalr.lookahead := nil;
      if lalr.lapropagate <> nil then
        item.next.lalr.lapropagate := lalr.lapropagate.clone
      else
        item.next.lalr.lapropagate := nil;
      item.next.lalr.bPropagated := lalr.bPropagated;
      item.next.next := nil;
      FCount := FCount + 1;
      result := true;
    end
    else begin
      b := false; //ignore lookahead > MAX_TERMINAL changes when declaring if added something
      if lalr.lookahead <> nil then begin
        if item.next.lalr.lookahead <> nil then begin
//          b := item.next.lalr.lookahead.add(lalr.lookahead)
          b := false;
          lalr.lookahead.getNextReset;
          while lalr.lookahead.getNext(h) do begin
            if item.next.lalr.lookahead.add(h) then begin
              if h <= MAX_TERMINAL then begin
                b := true;
              end;
            end;
          end;
        end
        else begin
//          item.next.lalr.lookahead := lalr.lookahead.clone;
//          b := true;
          item.next.lalr.lookahead := TIntHashSet.create;
          b := false;
          lalr.lookahead.getNextReset;
          while lalr.lookahead.getNext(h) do begin
            if item.next.lalr.lookahead.add(h) then begin
              if h <= MAX_TERMINAL then begin
                b := true;
              end;
            end;
          end;
        end;
      end;
      result := b;  //ignore lapropagate changes when declaring if added something
      if lalr.lapropagate <> nil then begin
        if item.next.lalr.lapropagate <> nil then
          b := item.next.lalr.lapropagate.add(lalr.lapropagate) or b
        else begin
          item.next.lalr.lapropagate := lalr.lapropagate.clone;
          b := true;
        end;
      end;
      item.next.lalr.bPropagated := lalr.bPropagated;
//      result := b;
    end;
  end
  else begin
    FBuckets[h] := new(PItemSetItem);
    FBuckets[h].lalr := new(PLALRItem);
    FBuckets[h].lalr.lr0 := lalr.lr0;
      if lalr.lookahead <> nil then
        FBuckets[h].lalr.lookahead := lalr.lookahead.clone
      else
        FBuckets[h].lalr.lookahead := nil;
      if lalr.lapropagate <> nil then
        FBuckets[h].lalr.lapropagate := lalr.lapropagate.clone
      else
        FBuckets[h].lalr.lapropagate := nil;
    FBuckets[h].lalr.bPropagated := lalr.bPropagated;
    FBuckets[h].next := nil;
    FCount := FCount + 1;
    result := true;
  end;
end;
*)

/// funkcia na pridanie LALR polozky do mnoziny poloziek (vstupom je pointer na tuto polozku),
/// vrati true ak v nej presne taka LALR polozka este nebola (staci rozdiel v lookaheade), inak false;
/// v premennej lalr bude pointer na tuto polozky, resp. na uz existujucu, ak tam taka uz bola
function TItemSet.add(var lalr: PLALRItem): Boolean;
var h: Integer;
    item: PItemSetItem;
    b: Boolean;
begin
  result := false;
  h := hash(lalr.lr0);
  item := FBuckets[h];
  if item <> nil then begin
    while (item.next <> nil) and ((item.lalr.lr0.rulenum <> lalr.lr0.rulenum) or (item.lalr.lr0.dot <> lalr.lr0.dot)) do begin
      item := item.next;
    end;
    if (item.lalr.lr0.rulenum <> lalr.lr0.rulenum) or (item.lalr.lr0.dot <> lalr.lr0.dot) then begin
      item.next := new(PItemSetItem);
      item.next.lalr := lalr;
      item.next.next := nil;
      FCount := FCount + 1;
      result := true;
    end
    else begin
      b := false;
      if lalr.lookahead <> nil then begin
        if item.lalr.lookahead <> nil then begin
          b := item.lalr.lookahead.add(lalr.lookahead);
          lalr.lookahead.Free;
//          lalr.lookahead := item.lalr.lookahead;
        end
        else begin
          item.lalr.lookahead := lalr.lookahead;
          b := true;
        end;
      end;
      if lalr.lapropagate <> nil then begin
        if item.lalr.lapropagate <> nil then begin
          b := item.lalr.lapropagate.add(lalr.lapropagate) or b;
          lalr.lapropagate.free;
//          lalr.lapropagate := item.lalr.lapropagate;
        end
        else begin
          item.lalr.lapropagate := lalr.lapropagate;
          b := true;
        end;
      end;
      item.lalr.bPropagated := lalr.bPropagated;
      result := b;
      dispose(lalr);
      lalr := item.lalr;
    end;
  end
  else begin
    FBuckets[h] := new(PItemSetItem);
    FBuckets[h].lalr := lalr;
    FBuckets[h].next := nil;
    FCount := FCount + 1;
    result := true;
  end;
end;

/// funkcia na pridanie LALR polozky do mnoziny poloziek (vstupom je pointer na tuto polozku),
/// vrati true ak v nej LALR polozka s takym jadrom este nebola, inak false;
/// v premennej lalr bude pointer na tuto polozky, resp. na uz existujucu, ak tam taka uz bola
function TItemSet.add2(var lalr: PLALRItem): Boolean;
var h: Integer;
    item: PItemSetItem;
    b: Boolean;
begin
  result := false;
  h := hash(lalr.lr0);
  item := FBuckets[h];
  if item <> nil then begin
    while (item.next <> nil) and ((item.lalr.lr0.rulenum <> lalr.lr0.rulenum) or (item.lalr.lr0.dot <> lalr.lr0.dot)) do begin
      item := item.next;
    end;
    if (item.lalr.lr0.rulenum <> lalr.lr0.rulenum) or (item.lalr.lr0.dot <> lalr.lr0.dot) then begin
      item.next := new(PItemSetItem);
      item.next.lalr := lalr;
      item.next.next := nil;
      FCount := FCount + 1;
      result := true;
    end
    else begin
      b := false;  //ignore lookahead > MAX_TERMINAL changes when declaring if added something
      if lalr.lookahead <> nil then begin
        if item.lalr.lookahead <> nil then begin
//          b := item.lalr.lookahead.add(lalr.lookahead)
          b := false;
          lalr.lookahead.getNextReset;
          while lalr.lookahead.getNext(h) do begin
            if item.lalr.lookahead.add(h) then begin
              if h <= MAX_TERMINAL then begin
                b := true;
              end;
            end;
          end;
          lalr.lookahead.Free;
        end
        else begin
//          item.lalr.lookahead := lalr.lookahead;
//          b := true;
          item.lalr.lookahead := TIntHashSet.create;
          b := false;
          lalr.lookahead.getNextReset;
          while lalr.lookahead.getNext(h) do begin
            if item.lalr.lookahead.add(h) then begin
              if h <= MAX_TERMINAL then begin
                b := true;
              end;
            end;
          end;
          lalr.lookahead.Free;
        end;
      end;
      result := b;  //ignore lapropagate changes when declaring if added something
      if lalr.lapropagate <> nil then begin
        if item.lalr.lapropagate <> nil then begin
          b := item.lalr.lapropagate.add(lalr.lapropagate) or b;
          lalr.lapropagate.Free;
        end
        else begin
          item.lalr.lapropagate := lalr.lapropagate;
          b := true;
        end;
      end;
      item.lalr.bPropagated := lalr.bPropagated;
//      result := b;


(*
      b := false;
      if lalr.lookahead <> nil then begin
        if item.lalr.lookahead <> nil then begin
          b := item.lalr.lookahead.add(lalr.lookahead);
          lalr.lookahead.Free;
//          lalr.lookahead := item.lalr.lookahead;
        end
        else begin
          item.lalr.lookahead := lalr.lookahead;
          b := true;
        end;
      end;
      if lalr.lapropagate <> nil then begin
        if item.lalr.lapropagate <> nil then begin
          b := item.lalr.lapropagate.add(lalr.lapropagate) or b;
          lalr.lapropagate.free;
//          lalr.lapropagate := item.lalr.lapropagate;
        end
        else begin
          item.lalr.lapropagate := lalr.lapropagate;
          b := true;
        end;
      end;

      item.lalr.bPropagated := lalr.bPropagated;
      result := b;
*)
      dispose(lalr);
      lalr := item.lalr;
    end;
  end
  else begin
    FBuckets[h] := new(PItemSetItem);
    FBuckets[h].lalr := lalr;
    FBuckets[h].next := nil;
    FCount := FCount + 1;
    result := true;
  end;
end;

/// funkcia vracajuca hasovaciu hodnotu pre danu LR0 polozku
function TItemSet.hash(lr0: TLR0Item): DWORD;
const
  A = 0.6180339887; // (sqrt(5) - 1) / 2
begin
  Result := Abs(Trunc(FCapacity * (Frac((lr0.rulenum + lr0.dot shl 16) * A))));
end;

/// funkcia zistujuca ci sa dana LR0 polozka nachadza v tejto mnozine poloziek
function TItemSet.contains(lr0: TLR0Item): Boolean;
var plalr: PLALRItem;
begin
  result := get(lr0,plalr);
end;

/// funkcia vracajuca pointer na LALR polozku (plalritem) podla jej LR0 jadra (lr0),
/// vrati ci bola takato polozka v mnozine najdena
function TItemSet.get(lr0: TLR0Item; var plalritem: PLALRItem): Boolean;
var h: Integer;
    item: PItemSetItem;
    b: Boolean;
begin
  result := false;
  h := hash(lr0);
  item := FBuckets[h];
  if item <> nil then begin
    while (item.next <> nil) and ((item.lalr.lr0.rulenum <> lr0.rulenum) or (item.lalr.lr0.dot <> lr0.dot)) do begin
      item := item.next;
    end;
    if (item.lalr.lr0.rulenum = lr0.rulenum) and (item.lalr.lr0.dot = lr0.dot) then begin
      plalritem := item.lalr;
      result := true;
    end;
  end;
end;

/// funkcia vracajuca LALR polozku (lalritem) podla jej LR0 jadra (lr0),
/// vrati ci bola takato polozka v mnozine najdena
function TItemSet.get(lr0: TLR0Item; var lalritem: TLALRItem): Boolean;
var h: Integer;
    item: PItemSetItem;
    b: Boolean;
begin
  result := false;
  h := hash(lr0);
  item := FBuckets[h];
  if item <> nil then begin
    while (item.next <> nil) and ((item.lalr.lr0.rulenum <> lr0.rulenum) or (item.lalr.lr0.dot <> lr0.dot)) do begin
      item := item.next;
    end;
    if (item.lalr.lr0.rulenum = lr0.rulenum) and (item.lalr.lr0.dot = lr0.dot) then begin
      lalritem := item.lalr^;
      result := true;
    end;
  end;
end;

/// funkcia vracajuca prvu LALR polozku v mnozine (teda pointer na nu)
/// vrati ci bola takato polozka v mnozine najdena
function TItemSet.getFirst(var plalritem: PLALRItem): Boolean;
var I: Integer;
    item: PItemSetItem;
begin
  FCurrentItem := nil;
  FCurrentBucket := -1;
  result := false;

  for I := 0 to Pred(FCapacity) do begin
    item := FBuckets[I];
    if (item <> nil) then begin
      FCurrentItem := item;
      FCurrentBucket := I;
      plalritem := FCurrentItem.lalr;
      result := true;
      exit;
    end;
  end;
end;

/// funkcia vracajuca nasledujucu LALR polozku v mnozine (teda pointer na nu)
/// vrati ci bola takato polozka v mnozine najdena
function TItemSet.getNext(var plalritem: PLALRItem): Boolean;
var I: Integer;
    item: PItemSetItem;
begin
  result := false;

  if (FCurrentItem = nil) or (FCurrentBucket = -1) then begin
    result := getFirst(plalritem);
    exit;
  end
  else if FCurrentItem.next <> nil then begin
    FCurrentItem := FCurrentItem.next;
    plalritem := FCurrentItem.lalr;
    result := true;
    exit;
  end
  else begin
    for I := FCurrentBucket + 1 to Pred(FCapacity) do begin
      item := FBuckets[I];
      if item <> nil then begin
        FCurrentItem := item;
        FCurrentBucket := I;
        plalritem := FCurrentItem.lalr;
        result := true;
        exit;
      end;
    end;
  end;
end;

/// funkcia vracajuca velkost tejto mnoziny poloziek
function TItemSet.getSize: Integer;
begin
  result := FCount;
end;

/// funkcia vracajuca, ci je tato mnozina prazdna
function TItemSet.isEmpty: Boolean;
begin
  result := getSize = 0;
end;

/// funkcia resetujuca iterovanie cez polozky (nasledujuca bude prva polozka v mnozine)
procedure TItemSet.getNextReset;
begin
  FCurrentItem := nil;
  FCurrentBucket := -1;
end;

/// funkcia vypisujuca mnozinu poloziek do vystupneho stringu; bComplex urcuje ci chceme komplexnejsi vypis
/// vstupne tabulky sluzia na transformovanie identifikatorov a indexov na mena terminalov, neterminalov a pravidiel
function TItemSet.toString(bComplex: Boolean = false; tnametable: TStrIntHashMap = nil; snametable: TStrIntHashMap = nil; ruletable: TRuleTable = nil): xString;
var h: Integer;
    I,index: Integer;
    item: PItemSetItem;
    S: xString;
    len: Integer;
    J: Integer;
    b: Boolean;
    plalr: PLALRItem;
begin
  result := '';
  for h := 0 to Pred(FCapacity) do begin
//    if bComplex then
//      result := result + '[' + IntToStr(h) + ']' + EOL;
    item := FBuckets[h];
    index := 0;
    while item <> nil do begin
      if bComplex then begin
//        result := result + '  (' + IntToStr(index) + ')  ';
        result := result + '  ';
        if (tnametable = nil) or (snametable = nil) or (ruletable = nil) then begin
          result := result + IntToStr(item.lalr.lr0.rulenum) + ' .' + IntToStr(item.lalr.lr0.dot);
          if item.lalr.lookahead <> nil then
            result := result + ' ;' + item.lalr.lookahead.toString;
          if item.lalr.bPropagated then
            result := result + ' TRUE'
          else
            result := result + ' FALSE';
          if item.lalr.lapropagate <> nil then
            result := result + ' #' + item.lalr.lapropagate.toString;
          result := result + EOL;
        end
        else begin
          len := ruletable.getRuleByIndex(item.lalr.lr0.rulenum).length;
          for I := 0 to Pred(len) do begin
            if I = item.lalr.lr0.dot then
              result := result + '.';
            J := ruletable.getRuleByIndex(item.lalr.lr0.rulenum).symbols[I];
            if J > MAX_TERMINAL then
              S := '<' + snametable.findByValue(J,b) + '>'
            else
              S := tnametable.findByValue(J,b);
            if b then
              result := result + S
            else
              result := result + IntToStr(J);
            if I = 0 then
              result := result + ' -> ';
          end;
          if len = item.lalr.lr0.dot then
            result := result + '.';

          if item.lalr.lookahead <> nil then begin
            result := result + ' ;';
            item.lalr.lookahead.getNextReset;
            while item.lalr.lookahead.getNext(I) do begin
              if I <= MAX_TERMINAL then begin
                S := tnametable.findByValue(I,b);
                if b then
                  result := result + S + ' '
                else
                  result := result + '?' + IntToStr(I) + '? ';
              end
              else begin
                result := result + '(' + IntToStr(I) + ') ';
              end;
            end;
//            result := result + item.lalr.lookahead.toString;
          end;
          if item.lalr.bPropagated then
            result := result + ' TRUE'
          else
            result := result + ' FALSE';
          if item.lalr.lapropagate <> nil then begin
            item.lalr.lapropagate.getNextReset;
            while item.lalr.lapropagate.getNext(Integer(plalr)) do begin
              result := result + EOL;
              for I := 1 to 4 do
                result := result + ' ';
              result := result + '#';
              len := ruletable.getRuleByIndex(plalr.lr0.rulenum).length;
              for I := 0 to Pred(len) do begin
                if I = plalr.lr0.dot then
                  result := result + '.';
                J := ruletable.getRuleByIndex(plalr.lr0.rulenum).symbols[I];
                if J > MAX_TERMINAL then
                  S := '<' + snametable.findByValue(J,b) + '>'
                else
                  S := tnametable.findByValue(J,b);
                if b then
                  result := result + S
                else
                  result := result + IntToStr(J);
                if I = 0 then
                  result := result + ' -> ';
              end;
              if len = plalr.lr0.dot then
                result := result + '.';
            end;
//            result := result + item.lalr.lapropagate.toString;
          end;
          result := result + EOL;
        end;
      end
      else begin
        result := result + IntToStr(item.lalr.lr0.rulenum) + ' .' + IntToStr(item.lalr.lr0.dot);
        if item.lalr.lookahead <> nil then
          result := result + ' ;' + item.lalr.lookahead.toString;
        if item.lalr.bPropagated then
          result := result + ' TRUE'
        else
          result := result + ' FALSE';
        if item.lalr.lapropagate <> nil then
          result := result + ' #' + item.lalr.lapropagate.toString;
        result := result + ' , ';
      end;
      item := item.next;
      index := index + 1;
    end;
  end;
end;

/// funkcia vypisujuca mnozinu poloziek do vystupneho zoznamu stringov (pouzivane pri zobrazovani LALR tabulky v LALR Table okne v programe);
/// bComplex urcuje ci chceme komplexnejsi vypis,
/// vstupne tabulky sluzia na transformovanie identifikatorov a indexov na mena terminalov, neterminalov a pravidiel
function TItemSet.toStringList(bComplex: Boolean = false; tnametable: TStrIntHashMap = nil; snametable: TStrIntHashMap = nil; ruletable: TRuleTable = nil): TStringList;
var h: Integer;
    I,index: Integer;
    item: PItemSetItem;
    Sresult, Slookahead, S: xString;
    len: Integer;
    J: Integer;
    b: Boolean;
    plalr: PLALRItem;
begin
  result := TStringList.create;
  Sresult := Sresult + '';
  Slookahead := '';
  for h := 0 to Pred(FCapacity) do begin
//    if bComplex then
//      Sresult := Sresult + '[' + IntToStr(h) + ']' + EOL;
    item := FBuckets[h];
    index := 0;
    while item <> nil do begin
      if bComplex then begin
//        Sresult := Sresult + '  (' + IntToStr(index) + ')  ';
(*        if (tnametable = nil) or (snametable = nil) or (ruletable = nil) then begin
          Sresult := Sresult + IntToStr(item.lalr.lr0.rulenum) + ' .' + IntToStr(item.lalr.lr0.dot);
          if item.lalr.lookahead <> nil then
            Sresult := Sresult + ' ;' + item.lalr.lookahead.toString;
          if item.lalr.bPropagated then
            Sresult := Sresult + ' TRUE'
          else
            Sresult := Sresult + ' FALSE';
          if item.lalr.lapropagate <> nil then
            Sresult := Sresult + ' #' + item.lalr.lapropagate.toString;
          Sresult := Sresult + EOL;
        end
        else*)
        begin
          len := ruletable.getRuleByIndex(item.lalr.lr0.rulenum).length;
          for I := 0 to Pred(len) do begin
            if I = item.lalr.lr0.dot then
              Sresult := Sresult + ' . ';
            J := ruletable.getRuleByIndex(item.lalr.lr0.rulenum).symbols[I];
            if J > MAX_TERMINAL then
              S := '<' + snametable.findByValue(J,b) + '>'
            else
              S := tnametable.findByValue(J,b);
            if b then
              Sresult := Sresult + S
            else
              Sresult := Sresult + IntToStr(J);
            if I = 0 then
              Sresult := Sresult + ' -> ';
          end;
          if len = item.lalr.lr0.dot then
            Sresult := Sresult + ' . ';

          if item.lalr.lookahead <> nil then begin
            item.lalr.lookahead.getNextReset;
            while item.lalr.lookahead.getNext(I) do begin
              if I <= MAX_TERMINAL then begin
                S := tnametable.findByValue(I,b);
                if b then
                  Slookahead := Slookahead + S + ' '
                else
                  Slookahead := Slookahead + '?' + IntToStr(I) + '? ';
              end
              else begin
                Slookahead := Slookahead + '(' + IntToStr(I) + ') ';
              end;
            end;
//            Sresult := Sresult + item.lalr.lookahead.toString;
          end;
          result.AddObject(Slookahead,TObject(item.lalr.lr0.rulenum));
          Slookahead := '';
//          if item.lalr.bPropagated then
//            Sresult := Sresult + ' TRUE'
//          else
//            Sresult := Sresult + ' FALSE';
          if item.lalr.lapropagate <> nil then begin
            item.lalr.lapropagate.getNextReset;
            while item.lalr.lapropagate.getNext(Integer(plalr)) do begin
              result.Add(Sresult);
              Sresult := '';
//              for I := 1 to 4 do
//                Sresult := Sresult + ' ';
              Sresult := Sresult + '~~ ';
              len := ruletable.getRuleByIndex(plalr.lr0.rulenum).length;
              for I := 0 to Pred(len) do begin
                if I = plalr.lr0.dot then
                  Sresult := Sresult + ' . ';
                J := ruletable.getRuleByIndex(plalr.lr0.rulenum).symbols[I];
                if J > MAX_TERMINAL then
                  S := '<' + snametable.findByValue(J,b) + '>'
                else
                  S := tnametable.findByValue(J,b);
                if b then
                  Sresult := Sresult + S
                else
                  Sresult := Sresult + IntToStr(J);
                if I = 0 then
                  Sresult := Sresult + ' -> ';
              end;
              if len = plalr.lr0.dot then
                Sresult := Sresult + ' . ';

              if plalr.lookahead <> nil then begin
                plalr.lookahead.getNextReset;
                while plalr.lookahead.getNext(I) do begin
                  if I <= MAX_TERMINAL then begin
                    S := tnametable.findByValue(I,b);
                    if b then
                      Slookahead := Slookahead + S + ' '
                    else
                      Slookahead := Slookahead + '?' + IntToStr(I) + '? ';
                  end
                  else begin
                    Slookahead := Slookahead + '(' + IntToStr(I) + ') ';
                  end;
                end;
              end;
              result.AddObject(Slookahead,TObject(plalr.lr0.rulenum));
              Slookahead := '';
            end;
//            Sresult := Sresult + item.lalr.lapropagate.toString;
          end;
//          Sresult := Sresult + EOL;
          result.Add(Sresult);
          Sresult := '';
        end;
      end;
(*      else begin
        Sresult := Sresult + IntToStr(item.lalr.lr0.rulenum) + ' .' + IntToStr(item.lalr.lr0.dot);
        if item.lalr.lookahead <> nil then
          Sresult := Sresult + ' ;' + item.lalr.lookahead.toString;
        if item.lalr.bPropagated then
          Sresult := Sresult + ' TRUE'
        else
          Sresult := Sresult + ' FALSE';
        if item.lalr.lapropagate <> nil then
          Sresult := Sresult + ' #' + item.lalr.lapropagate.toString;
        Sresult := Sresult + ' , ';
      end;*)
      item := item.next;
      index := index + 1;
    end;
  end;
end;

/// procedura na vymazanie vsetkych poloziek z mnoziny
procedure TItemSet.clear;
var I: Integer;
    item,previtem: PItemSetItem;
begin
  for I := 0 to Pred(FCapacity) do begin
    item := FBuckets[I];
    while (item <> nil) do begin
      previtem := item;
      item := item.next;

      previtem.lalr.lookahead.Free;
      previtem.lalr.lapropagate.Free;
      dispose(previtem.lalr);

      dispose(previtem);
      FCount := FCount - 1;
    end;
    FBuckets[I] := nil;
  end;
end;

/// destruktor na uvolnenie tohto objektu z pamate
destructor TItemSet.Destroy;
begin
  clear;
  FBuckets := nil;

  inherited;
end;

{ TSetOfItemSets }

/// kostruktor na vytvornie objektu popisujuceho mnozinu mnozin poloziek so vstupnym parametrom;
/// size je velkost interneho pola majuceho pointre na vsetky mnoziny poloziek
constructor TSetOfItemSets.create(size: Integer = 64);
var I: Integer;
begin
  inherited create;
  if size < 1 then
    size := 1;
  FAllocated := size;
  SetLength(a,FAllocated);
  for I := 0 to Pred(FAllocated) do begin
    a[I] := nil;
  end;
  items := TIntIntHashSet.create(true,ItemSet_kernelhash,ItemSet_kernelequal);
//  items.add(7,10);
//  writeln(items.toString(true));
end;

/// funkcia na pridanie vstupnej mnoziny poloziek (itemset) do tejto mnoziny mnozin,
/// vrati true, ak sa dana mnozina uspesne pridala, inak false
/// v outindex sa nachadza index pridanej, resp. najdenej mnoziny
function TSetOfItemSets.add(itemset: TItemSet; var outindex: Integer): Boolean;
var I: Integer;
begin
  I := FCount;
  if items.addorget(Integer(itemset),I) then begin
    result := true;
    outindex := FCount;
    a[FCount] := itemset;
    FCount := FCount + 1;
    if FAllocated = FCount then begin
      FAllocated := FAllocated * 2;
      SetLength(A,FAllocated);
    end;
  end
  else begin
    result := false;
    outindex := I;
  end;
(*
  result := false;
  if items.get(Integer(itemset),I) then begin
    outindex := I;
  end
  else begin
    if items.add(Integer(itemset),FCount) then begin
      result := true;
      outindex := FCount;
      a[FCount] := itemset;
      FCount := FCount + 1;
      if FAllocated = FCount then begin
        FAllocated := FAllocated * 2;
        SetLength(A,FAllocated);
      end;
    end
    else begin
      outindex := -1;
    end;
  end;
*)
end;

/// destruktor na uvolnenie tejto mnoziny mnozin poloziek z pamate
destructor TSetOfItemSets.Destroy;
var I: Integer;
begin
  for I := 0 to Pred(FCount) do begin
    A[I].Free;
  end;
  A := nil;
  items.Free;
  inherited;
end;

/// funkcia vracajuca mnozinu podla vstupneho indexu mnoziny
function TSetOfItemSets.getByIndex(index: Integer): TItemSet;
begin
  if (index >= 0) and (index < FCount) then
    result := a[index]
  else
    result := nil;
end;

/// funkcia vracajuca index v mnozine mnozin danej vstupnej mnoziny
function TSetOfItemSets.getIndexByItemSet(itemset: TItemSet): Integer;
var I: Integer;
begin
  if items.get(Integer(ItemSet),I) then
    result := I
  else
    result := -1;
end;

function TSetOfItemSets.toString(bComplex: Boolean = false; tnametable: TStrIntHashMap = nil; snametable: TStrIntHashMap = nil; ruletable: TRuleTable = nil): xString;
var S: xString;
    I: Integer;
begin
  result := '';
  for I := 0 to Pred(FCount) do begin
    S := getByIndex(I).toString(true,tnametable,snametable,ruletable);
    result := result + '[Itemset ' + IntToStr(I) + ']' + EOL + S;
  end;
end;

/// funkcia vracajuca pocet prvkov (teda mnozin) mnoziny mnozin poloziek
function TSetOfItemSets.getSize: Integer;
begin
  result := FCount;
end;

{ TParser }

/// funkcia, ktora na zaklade parsovacich tabuliek preparsuje vstup a vytvori parsovaci strom s korenom root o velkosti size
/// algortimus je blizsie popisany v diplomovej praci
/// funkcia vrati true ak parser akceptoval cely vstup, inak vrati false
function TParser.parse(var root: PToken; var size: Integer): Boolean;
var stack,input: TQueueStack;
    pt: PToken;
    IIhashset: TIntIntHashSet;
    hashset: TIntHashSet;
    shift: Integer;
    state,ttype: Integer;
    rulenum: Integer;
    I,J: Integer;
//    WS,EOLNWS,CLINE,CLINEEND,CSTART,CSTART2,CEND,CEND2: Integer;
    WS,COMMENT1,COMMENT2,COMMENTLINE: Integer;
    bComments: Boolean;
    bNext,bAccept: Boolean;
    childToken,prevchildToken: PToken;
    startpos,endpos: Integer;
    num_of_inserted: Integer;
    lex_errors,parse_errors: Integer;
    num_of_nodes: Integer;
    rule: TRule;
    plalr: PLALRItem;
    b: Boolean;
    symbol,index: Integer;


  procedure _shift(src: Integer; token: PToken; dest: Integer);
  begin
    stack.push(Integer(token));
    stack.push(dest);
{$IFDEF DEBUG1}
    writeln('State ' + IntToStr(src) + ': shift on ' + IntToStr(token.tokentype)  + '(' + token.value + ')' + ' -> ' + IntToStr(dest));
{$ENDIF}
    bNext := true;
  end;
  procedure _reduce_genuine(src: Integer; token: PToken; rulenum: Integer);
  var I: Integer;
      st: Integer;
      parentToken: PToken;
  begin
    childToken := nil;
    prevchildToken := nil;
    startpos := -1;
    endpos := -1;
    for I := 1 to Pred(LRuleTable.rules[rulenum].length) do begin
      stack.pop;
      childToken := PToken(stack.pop);
      if endpos = -1 then
        endpos := childToken.endp;
      if childToken.startp <> -1 then
        startpos := childToken.startp;
      childToken.sibling := prevchildToken;
      prevchildToken := childToken;
      num_of_nodes := num_of_nodes + 1;
    end;

    st := stack.top;
    I := -1;
    if FgotoTable.get(st,Integer(IIhashset)) then begin
      new(parentToken);
      parentToken.tokentype := LRuleTable.rules[rulenum].symbols[0];
      parentToken.value := '';
      parentToken.startp := startpos;
      parentToken.endp := endpos;
      parentToken.child := childToken;
      parentToken.sibling := nil;
      IIhashset.get(parentToken.tokentype,I);
      stack.push(Integer(parentToken));
      stack.push(I);
    end;
{$IFDEF DEBUG1}
    writeln('State ' + IntToStr(src) + ': reduce by rule ' + IntToStr(rulenum) + ' , ' + IntToStr(token.tokentype)  + '(' + token.value + ')' + ' -> ' + IntToStr(I));
{$ENDIF}
    if rulenum = 0 then begin  //reduce by rule 0 => ACCEPT
      bAccept := true;
{$IFDEF DEBUG1}
      writeln('State ' + IntToStr(st) + ': ACCEPT');
{$ENDIF}
    end;
  end;
  procedure _reduce(src: Integer; token: PToken; rulenum: Integer);
  var I: Integer;
      st: Integer;
      parentToken: PToken;
      parentType: Integer;
      lastrulesymbolindex: Integer;
  begin
    childToken := nil;
    prevchildToken := nil;
    startpos := -1;
    endpos := -1;
    parentType := LRuleTable.rules[rulenum].symbols[0];
    lastrulesymbolindex := Pred(LRuleTable.rules[rulenum].length);
    for I := 1 to lastrulesymbolindex do begin
      repeat
        stack.pop;
        childToken := PToken(stack.pop);  //!TODO! ak chceme mat commentare v strome, tak tu ich vkladat do listov, ale nemenit pri tom I
        if endpos = -1 then
          endpos := childToken.endp;
        if childToken.startp <> -1 then
          startpos := childToken.startp;
  //      if (childToken.tokentype = parentType) then begin//and (childToken.child <> nil) then begin  //collapsing A -> (A "-> epsilon")
        if (childToken.tokentype = parentType) or ((lastrulesymbolindex = 1) and (childToken.child = nil) and (childToken.tokentype > MAX_TERMINAL)) then begin  //too much collapsing (collapsing substitution rules A -> B)
          if childToken.child <> nil then begin
            parentToken := childToken.child;
            while parentToken.sibling <> nil do begin
              parentToken := parentToken.sibling;
            end;
            parentToken.sibling := prevchildToken;
            prevchildToken := childToken.child;
          end;
          dispose(childToken);
          childToken := nil;
          num_of_nodes := num_of_nodes - 1;
        end
        else begin
          childToken.sibling := prevchildToken;
          prevchildToken := childToken;
        end;
        num_of_nodes := num_of_nodes + 1;
(*        if not ((childToken.tokentype <> COMMENTLINE) and (childToken.tokentype <> COMMENT1) and (childToken.tokentype <> COMMENT2)) then begin
          writeln('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
          writeln(childtoken.tokentype,':',childtoken.value);
        end;*)
      until (not bComments) or (childToken = nil) or ((childToken.tokentype <> COMMENTLINE) and (childToken.tokentype <> COMMENT1) and (childToken.tokentype <> COMMENT2));
    end;
    if (bComments) {and (lastrulesymbolindex = 0)} then begin
      if stack.getSize > 1 then begin
        while true do begin
          childToken := PToken(stack.peek(-2));
          if ((childToken.tokentype <> COMMENTLINE) and (childToken.tokentype <> COMMENT1) and (childToken.tokentype <> COMMENT2)) then begin
            break;
          end;
          stack.pop;
          childToken := PToken(stack.pop);
          if endpos = -1 then
            endpos := childToken.endp;
          if childToken.startp <> -1 then
            startpos := childToken.startp;
          childToken.sibling := prevchildToken;
          prevchildToken := childToken;
          num_of_nodes := num_of_nodes + 1;
          if stack.getSize < 2 then
            break;
        end;
      end;
    end;

    childToken := prevchildToken;

    st := stack.top;
    I := -1;
    if FgotoTable.get(st,Integer(IIhashset)) then begin
      new(parentToken);
      parentToken.tokentype := parentType;
      parentToken.value := '';
      parentToken.startp := startpos;
      parentToken.endp := endpos;
      parentToken.child := childToken;
      parentToken.sibling := nil;
      IIhashset.get(parentToken.tokentype,I);
      stack.push(Integer(parentToken));
      stack.push(I);
    end;
{$IFDEF DEBUG1}
    writeln('State ' + IntToStr(src) + ': reduce by rule ' + IntToStr(rulenum) + ' , ' + IntToStr(token.tokentype)  + '(' + token.value + ')' + ' -> ' + IntToStr(I));
    writeln('Number of states in stack: ' + IntToStr(stack.getsize));
{$ENDIF}
    if rulenum = 0 then begin  //reduce by rule 0 => ACCEPT
      bAccept := true;
{$IFDEF DEBUG1}
      writeln('State ' + IntToStr(st) + ': ACCEPT');
{$ENDIF}
    end;
  end;

begin
  result := false;

  if LLexer = nil then begin
    root := nil;
    size := 0;
    Exit;
  end;

  stack := TQueueStack.create;
  input := TQueueStack.create;

  stack.push(0);
  pt := nil;
  ttype := -1;
  bAccept := false;
  bNext := true;
  WS := LRuleTable.getWSTerminal;
  LRuleTable.getComments(COMMENTLINE,COMMENT1,COMMENT2);
  LRuleTable.getParseComments(bComments);
//  EOLNWS := LRuleTable.getEOLNWSTerminal;
 // LRuleTable.getCommentTerminals(CLINE,CLINEEND,CSTART,CEND,CSTART2,CEND2);
  LLexer.LFileReader.setLineEndChar(LRuleTable.getLEC);
  num_of_inserted := 0;
  parse_errors := 0;
  lex_errors := 0;
  num_of_nodes := 1;
  FLog.clearandreset;
  repeat
    Application.ProcessMessages;
    if FGlobalHalt then
      break;
    state := stack.top;
    if bNext then begin
      if (input.isEmpty) or (num_of_inserted >= MAX_NUMBER_OF_TURNS_FROM_INSERTED_SYMBOLS) then begin
        //get token from Lexer
        pt := LLexer.getToken;
        while pt.tokentype <> L_EOF do begin
          if (pt.tokentype = WS) then begin
            if pt <> nil then
              dispose(pt);
            pt := LLexer.getToken;
          end
          else if (pt.tokentype = COMMENTLINE) or (pt.tokentype = COMMENT1) or (pt.tokentype = COMMENT2) then begin
            if bComments then begin
              _shift(state,pt,state);
            end
            else begin
              if pt <> nil then
                dispose(pt);
            end;
            pt := LLexer.getToken;
          end
          else
            break;
        end;
        //got it
(*          else if (pt.tokentype = CSTART) then begin
            while (pt.tokentype <> CEND) and (pt.tokentype <> L_EOF) do begin
              if pt <> nil then
                dispose(pt);
              pt := LLexer.getToken;
            end;
            if pt.tokentype = CEND then begin
              if pt <> nil then
                dispose(pt);
              pt := LLexer.getToken;
            end;
          end
          else if (pt.tokentype = CSTART2) then begin
            while (pt.tokentype <> CEND2) and (pt.tokentype <> L_EOF) do begin
              if pt <> nil then
                dispose(pt);
              pt := LLexer.getToken;
            end;
            if pt.tokentype = CEND2 then begin
              if pt <> nil then
                dispose(pt);
              pt := LLexer.getToken;
            end;
          end
          else if (pt.tokentype = EOLNWS) then begin  //obycajne pri line commentoch, ak nechceme inak EOLN token (okrem konca line commentov)
            if pt <> nil then
              dispose(pt);
            pt := LLexer.getToken;
          end

          else
            break;*)
        num_of_inserted := 0;
        if not input.isEmpty then begin
          while not input.isEmpty do
            dispose(PToken(input.dequeue));
        end;
      end
      else begin
        pt := PToken(input.dequeue);
        num_of_inserted := num_of_inserted + 1;
      end;
      bNext := false;
    end;
    ttype := pt.tokentype;
    //input.enqueue(pt);
    if FgotoTable.get(state,Integer(IIhashset)) then begin
      if not IIhashset.get(pt.tokentype,shift) then begin
        shift := -1;
      end;
    end
    else
      shift := -1;

    if FreduceTable.get(state,Integer(IIhashset)) then begin
      if not IIhashset.get(pt.tokentype,Integer(hashset)) then begin
        hashset := nil;
      end;
    end
    else
      hashset := nil;

    if shift >= 0 then begin
      if (hashset = nil) or (hashset.isEmpty) then begin
        //normal shift
        _shift(state,pt,shift);
(*
        stack.push(pt.tokentype);
        stack.push(shift);
        writeln('State ' + IntToStr(state) + ': shift on ' + IntToStr(pt.tokentype) + ' -> ' + IntToStr(shift));
        bNext := true;
*)
      end
      else begin
        //shift/reduce conflict <- shift wins
        FLog.addSFR2(LLexer.LFileReader,-Length(pt.value),W_PARSING_SHIFT_REDUCE_SHIFT,IntToStr(state),LTokenNameTypeTable.findByValue(pt.tokentype,b),IntToStr(shift));
        hashset.getNextReset;
        while hashset.getNext(index) do begin
          rule := LRuleTable.getRuleByIndex(index);
          FLog.addSFR2(LLexer.LFileReader,-Length(pt.value),W_PARSING_SHIFT_REDUCE_REDUCE,IntToStr(state),LTokenNameTypeTable.findByValue(pt.tokentype,b),IntToStr(index));
        end;
{$IFDEF DEBUG1}
        writeln('SHIFT/REDUCE conflict');
{$ENDIF}
        _shift(state,pt,shift);
      end;
    end
    else begin
      if (hashset = nil) or (hashset.isEmpty) then begin
{$IFDEF DEBUG1}
        writeln('ERROR');
{$ENDIF}
        //!TODO! lexer error (i.e. pt.tokentype = -1) vs. parser error (i.e. no possible move or reduction)
        if (pt.tokentype = L_ERROR) or (parse_errors > PARSE_ERRORS_THRESHOLD) then begin
          lex_errors := lex_errors + 1;
          if parse_errors <= PARSE_ERRORS_THRESHOLD then begin
            FLog.addSFR2(LLexer.LFileReader,-Length(pt.value),E_PARSING_INVALID_TOKEN,IntToStr(state),LTokenNameTypeTable.findByValue(pt.tokentype,b),pt.value);
{$IFDEF DEBUG1}
            writeln('State ' + IntToStr(state) + ': lexer (invalid token) error on ' + IntToStr(pt.tokentype) + '(' + pt.value + ')');
{$ENDIF}
          end
          else begin
            FLog.addSFR2(LLexer.LFileReader,-Length(pt.value),E_PARSING_PARSE_ERRORS_THRESHOLD_EXCEEDED,IntToStr(state),LTokenNameTypeTable.findByValue(pt.tokentype,b),pt.value);
{$IFDEF DEBUG1}
            writeln('State ' + IntToStr(state) + ': lexer (parse errors threshold exceeded, removing virtual token from stack) error on ' + IntToStr(pt.tokentype) + '(' + pt.value + ')');
{$ENDIF}
          end;
          //remedy - go to next input token
          bNext := true;
          if pt <> nil then begin
            dispose(pt);
            pt := nil;
          end;
        end
        else begin
          parse_errors := parse_errors + 1;
          FLog.addSFR2(LLexer.LFileReader,-Length(pt.value),E_PARSING_NO_POSSIBLE_ACTION,IntToStr(state),LTokenNameTypeTable.findByValue(pt.tokentype,b),pt.value);
{$IFDEF DEBUG1}
          writeln('State ' + IntToStr(state) + ': parser (no possible action) error on ' + IntToStr(pt.tokentype) + '(' + pt.value + ')');
{$ENDIF}
      //remedy - add symbol to input and thus enable shift or reduce (hmm rather reduce to avoid loops)
          bNext := false;
          if (input.getSize < MAX_SIZE_OF_INSERTED_SYMBOLS_QUEUE) and (FreduceTable.get(state,Integer(IIhashset))) then begin
            IIhashset.getFirst(I,Integer(hashset));
            if I = L_EOF then begin
              if not IIhashset.getNext(I,Integer(hashset)) then begin
                I := L_EOF;
              end;
            end;
            if I <> L_EOF then begin
              input.addFirst(Integer(pt));
              new(pt);
              pt.tokentype := I;
              pt.value := '';
              pt.startp := -1;
              pt.endp := -1;
              pt.child := nil;
              pt.sibling := nil;
            end
            else begin
              bNext := true;
              if pt <> nil then begin  //last resort - only EOF reduction possible => go to next input token
                dispose(pt);
                pt := nil;
              end;
            end;
          end
          else begin  //last resort - no possible reduction from state => go to next input token
            bNext := true;
            if pt <> nil then begin
              dispose(pt);
              pt := nil;
            end;
          end;
        end;
      end
      else if hashset.getSize = 1 then begin
        //normal reduce
        hashset.getFirst(rulenum);
        _reduce(state,pt,rulenum);
      end
      else if hashset.getSize > 1 then begin
        //reduce/reduce conflict <- rule with longer RHS wins
        hashset.getNextReset;
        while hashset.getNext(index) do begin
          rule := LRuleTable.getRuleByIndex(index);
          //rule.pos - LLexer.LFileReader.getPosition
          FLog.addSFR2(LLexer.LFileReader,-Length(pt.value),W_PARSING_REDUCE_REDUCE_REDUCE,IntToStr(state),LTokenNameTypeTable.findByValue(pt.tokentype,b),IntToStr(index));
        end;
{$IFDEF DEBUG1}
        writeln('REDUCE/REDUCE conflict');
{$ENDIF}
//        hashset.getFirst(rulenum);
        J := 0;
        hashset.getNextReset;
        while hashset.getNext(I) do begin
          if LRuleTable.getRuleByIndex(I).length > J then begin
            J := LRuleTable.getRuleByIndex(I).length;
            rulenum := I;
          end;
        end;
        _reduce(state,pt,rulenum);
      end
    end;
  until (lex_errors > LEX_ERRORS_THRESHOLD) or ((ttype = L_EOF) and ((bAccept) or (bNext)));
  if pt <> nil then begin
    dispose(pt);
    pt := nil;
  end;
  if bAccept then begin
    result := true;
    root := PToken(stack.peek(-2));
//    stack.pop;
//    root := stack.pop;

(*    childToken := nil;
    prevchildToken := nil;
    startpos := -1;
    endpos := -1;
    stack.pop;
    while not stack.isEmpty do begin
      childToken := PToken(stack.pop);
      stack.pop;
      if endpos = -1 then
        endpos := childToken.endp;
      if childToken.startp <> -1 then
        startpos := childToken.startp;
      childToken.sibling := prevchildToken;
      prevchildToken := childToken;
      num_of_nodes := num_of_nodes + 1;
    end;*)
  end
  else begin
    childToken := nil;
    prevchildToken := nil;
    startpos := -1;
    endpos := -1;
    stack.pop;
    while not stack.isEmpty do begin
      childToken := PToken(stack.pop);
      stack.pop;
      if endpos = -1 then
        endpos := childToken.endp;
      if childToken.startp <> -1 then
        startpos := childToken.startp;
      childToken.sibling := prevchildToken;
      prevchildToken := childToken;
      num_of_nodes := num_of_nodes + 1;
    end;
    new(root);
    root.tokentype := L_SPECIAL;
    root.value := '';
    root.startp := startpos;
    root.endp := endpos;
    root.child := childToken;
    root.sibling := nil;
  end;


{$IFDEF DEBUG0}
  writeln('Number of lex errors: ' + IntToStr(lex_errors));
  writeln('Number of parse errors: ' + IntToStr(parse_errors));
  writeln('Parse tree contains ' + IntToStr(num_of_nodes) + ' nodes');
{$ENDIF}
{$IFDEF LOG0}
    if LogForm <> nil then begin
      LogForm.AddLine('Number of lex errors: ' + IntToStr(lex_errors));
      LogForm.AddLine('Number of parse errors: ' + IntToStr(parse_errors));
      LogForm.AddLine('Parse tree contains ' + IntToStr(num_of_nodes) + ' nodes');
    end;
{$ENDIF}


{$IFDEF DEBUG2}
  writeln(toStringSubTree(root,false,true,true,LTokenNameTypeTable,LStateNameTypeTable));
{$ENDIF}

(*
  stack.pop;  //kill topmost state
  while not stack.isEmpty do begin
    pt := PToken(stack.pop);  //pop next PToken
    stack.pop;  //kill another state
    freeSubTree(pt);  //freesubtree with root pt, including pt
  end;
*)
  size := num_of_nodes;

  stack.Free;
  input.Free;
end;

/// funkcia, ktora vypise tabulky LALR parsera do vystupneho stringu
function TParser.toString: xString;
var hashset: TIntHashSet;
    IIhashset: TIntIntHashSet;
    I,J,symbol: Integer;
begin
  result := '';
//  writeln(FgotoTable.toString(true));
  result := result + '###gotoTable###' + EOL;
  FgotoTable.getNextReset;
  while FgotoTable.getNext(I,Integer(IIhashset)) do begin
    IIhashset.getNextReset;
    while IIhashset.getNext(symbol,J) do begin
      result := result + '[itemset ' + IntToStr(I) + '] on ' + IntToStr(symbol) + ' (' + IntToStr(Integer(IIhashset)) + ') shifts to ' + IntToStr(J);
      result := result + EOL;
    end;
  end;

  //  writeln(FreduceTable.toString(true));
  result := result + '###reduceTable###' + EOL;
  FreduceTable.getNextReset;
  while FreduceTable.getNext(I,Integer(IIhashset)) do begin
//    writeln(IntToStr(I) + ' - ' + IntToStr(Integer(IIhashset)) + EOL + IIhashset.toString(true));
    IIhashset.getNextReset;
    while IIhashset.getNext(J,Integer(hashset)) do begin
      result := result + '[itemset ' + IntToStr(I) + '] on ' + IntToStr(J) + ' (' + IntToStr(Integer(hashset)) + ') reduces by rules ' + hashset.toString(false);
      result := result + EOL;
    end;
  end;
end;

/// konstuktor objektu type TParser, ktory implementuje LALR parsovanie na zaklade vstupny parsovacich tabuliek,
/// ktore boli vypocitane v TParserBuilder objekte
/// vstupom su hlavne dve tabulky  gotoTable a reduceTable, podla ktorych prebieha samotne parsovanie
/// ostatne tabulky su len pre vypis parsovacich tabuliek, podobne ako tomu bolo v TParserBuidler triede
/// ttable a stable su tabulky nazvov terminalov a neterminalov posielane najma kvoli vypisu parsovacich tabuliek;
/// lexer je pointer na lexikalny analyzator
constructor TParser.create(lexer: TLexer; ttable, stable: TStrIntHashMap; ruletable: TRuleTable; gototable, reducetable: TIntIntHashSet);
begin
  inherited create;
  LLexer := lexer;
  LTokenNameTypeTable := ttable;
  LStateNameTypeTable := stable;
  LRuleTable := ruletable;
  FGotoTable := gototable;
  FReduceTable := reducetable;
  FGlobalSet := nil;
  FConflictStates := nil;
  FLog := TLog.create;
end;

/// destruktor odstranujuci tento LALR parser z pamate
destructor TParser.Destroy;
var hashset: TIntHashSet;
    IIhashset: TIntIntHashSet;
    I,J: Integer;
begin
  FreduceTable.getNextReset;
  while FreduceTable.getNext(I,Integer(IIhashset)) do begin
    IIhashset.getNextReset;
    while IIhashset.getNext(J,Integer(hashset)) do begin
      hashset.Free;
    end;
    IIhashset.Free;
  end;
  FreduceTable.Free;

  FgotoTable.getNextReset;
  while FgotoTable.getNext(I,Integer(IIhashset)) do begin
    IIhashset.Free;
  end;
  FgotoTable.Free;

  if FGlobalSet <> nil then begin
    FGlobalSet.Free;
  end;

  if FConflictStates <> nil then begin
    FConflictStates.Free;
  end;

  FLog.Free;

  inherited;
end;

/// procedura na spojenie s inym lexikalnym analyzatorom
procedure TParser.linkwithLexer(lexer: TLexer);
begin
  LLexer := lexer;
end;

{ TDFA }

/// konstruktor vytvarajuci objekt, ktory je implementaciou deterministickeho konecneho automatu
constructor TDFA.create(NFA: TNFA; size: Integer = 256);
var I: Integer;
begin
  inherited create;
  if size < 1 then
    size := 1;
  LNFA := NFA;
  FAllocated := size;
  SetLength(a,FAllocated);
  for I := 0 to Pred(FAllocated) do begin
    a[I] := nil;
  end;
  FCount := 0;
  FCurrentState := 0;
  items := TIntIntHashSet.createEx(size,2,4,true,NStates_hash,NStates_equal);

  if LNFA <> nil then
    FCaseSensitive := LNFA.getCaseSensitivity
  else
    FCaseSensitive := true;  //aj tak sa prepise pri budovani DFA spravnou hodnotou
end;

/// funkcia na pridanie stavu do DFA, stav reprezentujuci Nstates mnozinu NFA stavov
function TDFA.add(Nstates: TIntHashSet; var index_out: Integer): Boolean;
var I: Integer;
begin
  I := FCount;
  if items.addorget(Integer(Nstates),I) then begin
    result := true;
    index_out := FCount;
//    a[FCount] := Nstates;
    a[FCount] := new(PDState);
    a[FCount].nstates := Nstates;
    a[FCount].moves := TIntIntHashSet.createEx(16);
    a[FCount].accepts := 0;
    FCount := FCount + 1;
    if FAllocated = FCount then begin
      FAllocated := FAllocated * 2;
      SetLength(A,FAllocated);
    end;
  end
  else begin
    result := false;
    index_out := I;
  end;
end;

/// procedura na nastavenie DFA do pociatocneho stavu
procedure TDFA.reset;
begin
  FCurrentState := 0;
end;

/// funkcia na simulaciu DFA pri citani znaku c,
/// vrati, ci je dany stav akcetacny (acc > 0, a konkretne cislo urcuje typ tokenu, ktory je akceptovany)
function TDFA.simulate(c: char): Integer;
var I: Integer;
begin
  if a[FCurrentState].moves.get(Integer(c),I) then begin
    FCurrentState := I;
    result := a[FCurrentState].accepts;
  end
  else begin
    result := -1;
  end;
end;

/// destruktor odstranujuci tento objekt z pamate
destructor TDFA.Destroy;
var I: Integer;
    hashset: TIntHashSet;
    data: Integer;
begin
  for I := 0 to Pred(FCount) do begin
    a[i].moves.Free;
    a[i].nstates.Free;
    dispose(a[I]);
  end;
  a := nil;
(*  items.getNextReset;
  while items.getNext(Integer(hashset),data) do begin
    hashset.Free;
  end;*)
  items.Free;
  inherited;
end;

(*
function TDFA.build: Boolean;
var index,aindex: Integer;
    state0: Integer;
    Nstates,closure,charmoves,setmoves,newNstates: TIntHashSet;
    acc: Integer;
    I: Integer;
    Q: TQueueStack;
    source,symbol: Integer;
    charset: TCharSet;
    relevantSetIndices: TIntHashSet;
begin
  result := false;
  if (LNFA = nil) or (LNFA.getSize = 0) then
    Exit;

  Q := TQueueStack.create;

  state0 := LNFA.states.peek;
  Nstates := TIntHashSet.create(state0);
  Q.push(-1);
  Q.push(-1);
  Q.push(Integer(Nstates));
  index := 0;
  repeat
    source := Q.dequeue;
    symbol := Q.dequeue;
    Nstates := TIntHashSet(Q.dequeue);
    acc := LNFA.epsilonClosure2(Nstates,closure, charmoves, setmoves);
    Nstates.Free;
{$IFDEF DEBUG1}
    writeln('Closure #: ' + IntToStr(index));
{$ENDIF}
    if not add(closure, aindex) then begin
      if source >= 0 then
        a[source].moves.add(symbol,aindex);
        closure.Free;
    end
    else begin
      if source >= 0 then
        a[source].moves.add(symbol,aindex);
      a[aindex].accepts := acc;

      if setmoves = nil then begin
        if charmoves <> nil then begin
          charmoves.getNextReset;
          while charmoves.getNext(I) do begin
            LNFA.move2(closure,I,newNstates);
            Q.push(aindex);
            Q.push(I);
            Q.push(Integer(newNstates));
          end;
        end;
      end
      else begin
        charset := TCharSet.create('temp');
        if charmoves <> nil then begin
          charmoves.getNextReset;
          while charmoves.getNext(I) do begin
            charset.add(char(I));
          end;
        end;
        setmoves.getNextReset;
        while setmoves.getNext(I) do begin
          charset.addCharSet(LNFA.FSetTable.getByIndex(I));
        end;
        for symbol := 0 to CHAR_HIGH do begin
          if charset.hasChar(char(symbol)) then begin

            relevantSetIndices := TIntHashSet.create;
            setmoves.getNextReset;
            while setmoves.getNext(I) do begin
              if LNFA.FSetTable.getByIndex(I).hasChar(char(symbol)) then begin
                relevantSetIndices.add(I);
              end;
            end;

            LNFA.move2(closure,symbol,newNstates,relevantSetIndices);

            relevantSetIndices.Free;

            Q.push(aindex);
            Q.push(symbol);
            Q.push(Integer(newNstates));
          end;
        end;
        charset.Free;
      end;
    end;
    charmoves.Free;
    setmoves.Free;

    index := index + 1;
  until Q.isEmpty;//index = FCount;
  Q.Free;

{$IFDEF DEBUG1}
  for I := 0 to Pred(FCount) do begin
    writeln('[Det State ' + IntToStr(I) + '] ' + '(accepts: ' + IntToStr(a[i].accepts) + ')');
    a[I].moves.getnextreset;
    while a[I].moves.getNext(symbol,aindex) do begin
      writeln('      ' + char(symbol) + ' -> ' + IntToStr(aindex));
    end;
  end;
{$ENDIF}


{$IFDEF DEBUG0}
  writeln('NFA closures generated: ' + IntToStr(index));
  writeln('DFA States: ' + IntToStr(FCount));
{$ENDIF}
{$IFDEF LOG0}
    if LogForm <> nil then begin
      LogForm.AddLine('NFA closures generated: ' + IntToStr(index));
      LogForm.AddLine('DFA States: ' + IntToStr(FCount));
    end;
{$ENDIF}

  result := true;
end;

function TDFA.build2: Boolean;
var index,aindex: Integer;
    state0: Integer;
    Nstates,closure,charmoves,setmoves,newNstates: TIntHashSet;
    acc: Integer;
    I,J: Integer;
    Q: TQueueStack;
    source,symbol: Integer;
    charset: TCharSet;
    relevantSetIndices: TIntHashSet;
    symbolSet: TIntHashSet;
    bFirst: Boolean;
begin
  result := false;
  if (LNFA = nil) or (LNFA.getSize = 0) then
    Exit;

  Q := TQueueStack.create;

  state0 := LNFA.states.peek;
  Nstates := TIntHashSet.create(state0);
  Q.push(-1);
  Q.push(Integer(TIntHashSet.create(-1)));
  Q.push(Integer(Nstates));
  index := 0;
  repeat
    source := Q.dequeue;
    symbolSet := TIntHashSet(Q.dequeue);
    Nstates := TIntHashSet(Q.dequeue);
    acc := LNFA.epsilonClosure2(Nstates,closure, charmoves, setmoves);
    Nstates.Free;
{$IFDEF DEBUG1}
    writeln('Closure #: ' + IntToStr(index));
{$ENDIF}
    if not add(closure, aindex) then begin
      if source >= 0 then begin
        symbolSet.getNextReset;
        while symbolSet.getNext(I) do begin
          a[source].moves.add(I,aindex);
{          if not a[source].moves.add(I,aindex) then begin
            a[source].moves.get(I,J);
            writeln(aindex, ': ', I, ' -> ', J, ' vs. ', I, ' -> ', aindex);
          end;}
        end;
      end;
      symbolSet.Free;
      closure.Free;
    end
    else begin
      if source >= 0 then begin
        while symbolSet.getNext(I) do begin
          a[source].moves.add(I,aindex);
{          if not a[source].moves.add(I,aindex) then begin
            a[source].moves.get(I,J);
            writeln(aindex, ': ', I, ' -> ', J, ' vs. ', I, ' -> ', aindex);
          end;}
        end;
      end;
      symbolSet.Free;
      a[aindex].accepts := acc;

      if setmoves = nil then begin
        if charmoves <> nil then begin
          charmoves.getNextReset;
          while charmoves.getNext(I) do begin
            LNFA.move2(closure,I,newNstates);
            Q.push(aindex);
            Q.push(Integer(TIntHashSet.create(I)));
            Q.push(Integer(newNstates));
          end;
        end;
      end
      else begin
        charset := TCharSet.create('temp');
        if charmoves <> nil then begin
          charmoves.getNextReset;
          while charmoves.getNext(I) do begin
            charset.add(char(I));
          end;
        end;
        setmoves.getNextReset;
        while setmoves.getNext(I) do begin
          charset.addCharSet(LNFA.FSetTable.getByIndex(I));
        end;
        bFirst := false;
        for symbol := 0 to CHAR_HIGH do begin
          if charset.hasChar(char(symbol)) then begin

            relevantSetIndices := TIntHashSet.create;
            setmoves.getNextReset;
            while setmoves.getNext(I) do begin
              if LNFA.FSetTable.getByIndex(I).hasChar(char(symbol)) then begin
                relevantSetIndices.add(I);
              end;
            end;

            LNFA.move2(closure,symbol,newNstates,relevantSetIndices);

            relevantSetIndices.Free;

            if (bFirst) and (not Q.isEmpty) and (NStates_equal(Integer(newNstates),Q.top)) then begin
              TIntHashSet(Q.peek(-2)).add(symbol);
              newNstates.Free;
            end
            else begin
              Q.push(aindex);
              Q.push(Integer(TIntHashSet.create(symbol)));
              Q.push(Integer(newNstates));
              bFirst := true;
            end;
          end;
        end;
        charset.Free;
      end;
    end;
    charmoves.Free;
    setmoves.Free;

    index := index + 1;
  until Q.isEmpty;//index = FCount;
  Q.Free;

{$IFDEF DEBUG1}
  for I := 0 to Pred(FCount) do begin
    writeln('[Det State ' + IntToStr(I) + '] ' + '(accepts: ' + IntToStr(a[i].accepts) + ')');
    a[I].moves.getnextreset;
    while a[I].moves.getNext(symbol,aindex) do begin
      writeln('      ' + char(symbol) + ' -> ' + IntToStr(aindex));
    end;
  end;
{$ENDIF}


{$IFDEF DEBUG0}
  writeln('NFA closures generated: ' + IntToStr(index));
  writeln('DFA States: ' + IntToStr(FCount));
{$ENDIF}
{$IFDEF LOG0}
    if LogForm <> nil then begin
      LogForm.AddLine('NFA closures generated: ' + IntToStr(index));
      LogForm.AddLine('DFA States: ' + IntToStr(FCount));
    end;
{$ENDIF}

  result := true;
end;
*)

/// funkcia na vypis DFA do stringu
function TDFA.toString: xString;
var I: Integer;
    symbol,aindex: Integer;
begin
  result := '';
  clearGCA;
  for I := 0 to Pred(FCount) do begin
    result := '[Det State ' + IntToStr(I) + '] ' + '(accepts: ' + IntToStr(a[i].accepts) + ')' + EOL;
    addStringtoGCA(result);
    a[I].moves.getnextreset;
    while a[I].moves.getNext(symbol,aindex) do begin
      if symbol < 256 then begin
        result := '      ' + char(symbol) + ' -> ' + IntToStr(aindex) + EOL;
        addStringtoGCA(result);
      end;
    end;
  end;
  toStringGCA(result);
end;

/// funkcia vracajuca pocet stavov DFA
function TDFA.getSize: Integer;
begin
  result := FCount;
end;

/// funkcia, ktora vytvara DFA z povodneho NFA
/// kazda mnozina stavov v ktorej sa povodny nedeterministicky konecny automat moze nachadzat je
/// reprezentovana jednym stavom v novom deterministickom konecnom automate
function TDFA.build3: Boolean;
var index,aindex: Integer;
    state0: Integer;
    Nstates,closure,newNstates: TIntHashSet;
    charmoves,setmoves: TIntIntHashSet;
    acc: Integer;
    I,J: Integer;
    Q: TQueueStack;
    source,symbol: Integer;
    charset: TCharSet;
    relevantSetIndices: TIntHashSet;
    hashset,symbolSet: TIntHashSet;
    bFirst,bCaseSensitive: Boolean;
    links_count: Integer;
    c1,c2: char;
    symbol2: Integer;
begin
  result := false;
  if (LNFA = nil) or (LNFA.getSize = 0) then
    Exit;

  bCaseSensitive := LNFA.getCaseSensitivity;
  FCaseSensitive := LNFA.getCaseSensitivity;

  Q := TQueueStack.create;
  charset := TCharSet.create('temp');
  state0 := LNFA.states.peek;
  Nstates := TIntHashSet.create(state0);
  Q.push(-1);
  Q.push(Integer(TIntHashSet.create(-1)));
  Q.push(Integer(Nstates));
  index := 0;
  links_count := 0;

  repeat
    Application.ProcessMessages;
    if FGlobalHalt then
      break;
    source := Q.dequeue;
    symbolSet := TIntHashSet(Q.dequeue);
    Nstates := TIntHashSet(Q.dequeue);
//    writeln(Integer(Nstates));
    acc := LNFA.epsilonClosure3(Nstates,closure, charmoves, setmoves);
    Nstates.Free;
{$IFDEF DEBUG1}
    writeln('Closure #: ' + IntToStr(index));
{$ENDIF}
    if not add(closure, aindex) then begin
      if source >= 0 then begin
        symbolSet.getNextReset;
        while symbolSet.getNext(I) do begin
          a[source].moves.add(I,aindex);
          links_count := links_count + 1;
(*          if not a[source].moves.add(I,aindex) then begin
            a[source].moves.get(I,J);
            writeln(aindex, ': ', I, ' -> ', J, ' vs. ', I, ' -> ', aindex);
          end;*)
        end;
      end;
      symbolSet.Free;
      closure.Free;
    end
    else begin
      if source >= 0 then begin
        while symbolSet.getNext(I) do begin
          a[source].moves.add(I,aindex);
          links_count := links_count + 1;
(*          if not a[source].moves.add(I,aindex) then begin
            a[source].moves.get(I,J);
            writeln(aindex, ': ', I, ' -> ', J, ' vs. ', I, ' -> ', aindex);
          end;*)
        end;
      end;
      symbolSet.Free;
      a[aindex].accepts := acc;

      if setmoves = nil then begin
        if charmoves <> nil then begin
          if bCaseSensitive then begin
            charmoves.getNextReset;
            while charmoves.getNext(I,Integer(newNstates)) do begin
  //            LNFA.move2(closure,I,newNstates);
              Q.push(aindex);
              Q.push(Integer(TIntHashSet.create(I)));
              Q.push(Integer(newNstates.clone));
            end;
          end
          else begin
            charset.clear;
            charmoves.getNextReset;
            while charmoves.getNext(I,Integer(newNstates)) do begin
              if not charset.hasChar(char(I)) then begin
                c1 := char(AnsiLowerCase(char(I))[1]);
                c2 := char(AnsiUpperCase(char(I))[1]);
                if c1 <> c2 then begin
                  symbolSet := TIntHashSet.create(Integer(c1));
                  symbolSet.add(Integer(c2));
//                  hashset := newNstates.clone;
                  if Integer(c1) = I then begin
                    if not charmoves.get(Integer(c2),Integer(hashset)) then
                      hashset := nil;
                  end
                  else begin
                    if not charmoves.get(Integer(c1),Integer(hashset)) then
                      hashset := nil;
                  end;
                  Nstates := newNstates.clone;
                  Nstates.add(hashset);
                  Q.push(aindex);
                  Q.push(Integer(symbolSet));
                  Q.push(Integer(Nstates));
                  charset.add(c1);
                  charset.add(c2);
                end
                else begin
                  Q.push(aindex);
                  Q.push(Integer(TIntHashSet.create(I)));
                  Q.push(Integer(newNstates.clone));
                end;
              end
              else begin
                //we have already put both cases into DFA transition table, nothing to do here
              end;
            end;
          end;
        end;
      end
      else begin
        charset.clear;
        if charmoves <> nil then begin
          charmoves.getNextReset;
          while charmoves.getNext(I,Integer(newNstates)) do begin
            charset.add(char(I));
          end;
        end;
        setmoves.getNextReset;
        while setmoves.getNext(I,Integer(newNstates)) do begin
          charset.addCharSet(LNFA.FSetTable.getByIndex(I));
        end;
        bFirst := false;
        if bCaseSensitive then begin
          for symbol := 0 to CHAR_HIGH do begin
            if charset.hasChar(char(symbol)) then begin

  //            relevantSetIndices := TIntHashSet.create;
              newNstates := nil;
              setmoves.getNextReset;
              while setmoves.getNext(I,Integer(hashset)) do begin
                if LNFA.FSetTable.getByIndex(I).hasChar(char(symbol)) then begin
                  if newNstates = nil then begin
                    newNstates := hashset.clone;
                  end
                  else
                    newNstates.add(hashset);
  //                relevantSetIndices.add(I);
                end;
              end;
              if charmoves <> nil then begin
                if charmoves.get(symbol,Integer(hashset)) then begin
                  if newNstates = nil then begin
                    newNstates := hashset.clone;
                  end
                  else
                    newNstates.add(hashset);
                end;
              end;

  //            if newNstates = nil then
  //              ShowMessage('Error');
  //            LNFA.move2(closure,symbol,newNstates,relevantSetIndices);
  //            relevantSetIndices.Free;

              if (bFirst) and (not Q.isEmpty) and (NStates_equal(Integer(newNstates),Q.top)) then begin
                TIntHashSet(Q.peek(-2)).add(symbol);
                newNstates.Free;
              end
              else begin
                Q.push(aindex);
                Q.push(Integer(TIntHashSet.create(symbol)));
                Q.push(Integer(newNstates));
                bFirst := true;
              end;
            end;
          end;
        end
        else begin
          for symbol := 0 to CHAR_HIGH do begin
            if charset.hasChar(char(symbol)) then begin
              c1 := char(AnsiLowerCase(char(symbol))[1]);
              c2 := char(AnsiUpperCase(char(symbol))[1]);
              if c1 <> c2 then begin

    //            relevantSetIndices := TIntHashSet.create;
                newNstates := nil;
                setmoves.getNextReset;
                while setmoves.getNext(I,Integer(hashset)) do begin
                  if LNFA.FSetTable.getByIndex(I).hasChar(char(symbol)) then begin
                    if newNstates = nil then begin
                      newNstates := hashset.clone;
                    end
                    else
                      newNstates.add(hashset);
    //                relevantSetIndices.add(I);
                  end;
                end;
                if charmoves <> nil then begin
                  if charmoves.get(symbol,Integer(hashset)) then begin
                    if newNstates = nil then begin
                      newNstates := hashset.clone;
                    end
                    else
                      newNstates.add(hashset);
                  end;
                end;

    //            if newNstates = nil then
    //              ShowMessage('Error');
    //            LNFA.move2(closure,symbol,newNstates,relevantSetIndices);
    //            relevantSetIndices.Free;
                if symbol = Integer(c1) then
                  symbol2 := Integer(c2)
                else
                  symbol2 := Integer(c1);

                setmoves.getNextReset;
                while setmoves.getNext(I,Integer(hashset)) do begin
                  if LNFA.FSetTable.getByIndex(I).hasChar(char(symbol2)) then begin
                    if newNstates = nil then begin
                      newNstates := hashset.clone;
                    end
                    else
                      newNstates.add(hashset);
    //                relevantSetIndices.add(I);
                  end;
                end;
                if charmoves <> nil then begin
                  if charmoves.get(symbol2,Integer(hashset)) then begin
                    if newNstates = nil then begin
                      newNstates := hashset.clone;
                    end
                    else
                      newNstates.add(hashset);
                  end;
                end;

                charset.sub(c1);
                charset.sub(c2);

                if (bFirst) and (not Q.isEmpty) and (NStates_equal(Integer(newNstates),Q.top)) then begin
                  TIntHashSet(Q.peek(-2)).add(symbol);
                  TIntHashSet(Q.peek(-2)).add(symbol2);
                  newNstates.Free;
                end
                else begin
                  symbolSet := TIntHashSet.create(symbol);
                  symbolSet.add(symbol2);
                  Q.push(aindex);
                  Q.push(Integer(symbolSet));
                  Q.push(Integer(newNstates));
                  bFirst := true;
                end;
              end
              else begin
    //            relevantSetIndices := TIntHashSet.create;
                newNstates := nil;
                setmoves.getNextReset;
                while setmoves.getNext(I,Integer(hashset)) do begin
                  if LNFA.FSetTable.getByIndex(I).hasChar(char(symbol)) then begin
                    if newNstates = nil then begin
                      newNstates := hashset.clone;
                    end
                    else
                      newNstates.add(hashset);
    //                relevantSetIndices.add(I);
                  end;
                end;
                if charmoves <> nil then begin
                  if charmoves.get(symbol,Integer(hashset)) then begin
                    if newNstates = nil then begin
                      newNstates := hashset.clone;
                    end
                    else
                      newNstates.add(hashset);
                  end;
                end;

    //            if newNstates = nil then
    //              ShowMessage('Error');
    //            LNFA.move2(closure,symbol,newNstates,relevantSetIndices);
    //            relevantSetIndices.Free;

                if (bFirst) and (not Q.isEmpty) and (NStates_equal(Integer(newNstates),Q.top)) then begin
                  TIntHashSet(Q.peek(-2)).add(symbol);
                  newNstates.Free;
                end
                else begin
                  Q.push(aindex);
                  Q.push(Integer(TIntHashSet.create(symbol)));
                  Q.push(Integer(newNstates));
                  bFirst := true;
                end;

              end;
            end;
          end;
        end;
      end;
    end;

    if charmoves <> nil then begin
      charmoves.getNextReset;
      while charmoves.getNext(I,Integer(hashset)) do begin
        hashset.Free;
      end;
      charmoves.Free;
    end;
    if setmoves <> nil then begin
      setmoves.getNextReset;
      while setmoves.getNext(I,Integer(hashset)) do begin
        hashset.Free;
      end;
      setmoves.Free;
    end;
    index := index + 1;
  until Q.isEmpty;//index = FCount;

  charset.Free;
  Q.Free;

{$IFDEF DEBUG1}
  for I := 0 to Pred(FCount) do begin
    writeln('[Det State ' + IntToStr(I) + '] ' + '(accepts: ' + IntToStr(a[i].accepts) + ')');
    a[I].moves.getnextreset;
    while a[I].moves.getNext(symbol,aindex) do begin
      writeln('      ' + char(symbol) + ' -> ' + IntToStr(aindex));
    end;
  end;
{$ENDIF}

{$IFDEF DEBUG0}
  writeln('NFA closures generated: ' + IntToStr(index));
  writeln('DFA States: ' + IntToStr(FCount));
  writeln('DFA Links: ' + IntToStr(links_count));
{$ENDIF}
{$IFDEF LOG0}
  LogForm.AddLine('NFA closures generated: ' + IntToStr(index));
  LogForm.AddLine('DFA States: ' + IntToStr(FCount));
  LogForm.AddLine('DFA Links: ' + IntToStr(links_count));
{$ENDIF}
  result := true;
end;

/// procedura na nastavanie citlivosti na velke a male pismena
procedure TDFA.setCaseSensitivity(bCaseSensitive: Boolean);
begin
  FCaseSensitive := bCaseSensitive;
end;

/// procedura vracajuca momentalne nastavanie citlivosti na velke a male pismena
function TDFA.getCaseSensitivity: Boolean;
begin
  result := FCaseSensitive;
end;

/// procedura na nastavanie citlivosti na velke a male pismena (nadradene konkretnym automatom ak true)
procedure TLexer.setCaseSensitivity(bCaseSensitive: Boolean);
begin
  FCaseSense := bCaseSensitive;
end;

/// procedura vracajuca momentalne nastavanie citlivosti na velke a male pismena (nadradene konkretnym automatom ak true)
function TLexer.getCaseSensitivity: Boolean;
begin
  result := FCaseSense;
end;

end.

