#define MIX_TOKENIZER_CPP_

#include "Tokenizer.h"
#include "DefaultTraits.h"
#include "DefaultXMLTraits.h"

#include <algorithm>

namespace MiX{

  //トークン
  template<class charT,class char_traits,class xml_traits>
  const charT Tokenizer<charT,char_traits,xml_traits>::tokens_[15] = {
    xml_traits::null(),
    xml_traits::crlf(),
    xml_traits::sp(),
    xml_traits::tab(),
    xml_traits::lt(),
    xml_traits::eq(),
    xml_traits::gt(),
    xml_traits::slash(),
    xml_traits::hyphen(),
    xml_traits::semicolon(),
    xml_traits::exclamation(),
    xml_traits::question(),
    xml_traits::amp(),
    xml_traits::dblquote(),
    xml_traits::quote(), 
  };

  template <class charT,class char_traits,class xml_traits>
  bool Tokenizer<charT,char_traits,xml_traits>::fillUpBuffer(){
    string_type line;
    if (is_ != NULL && std::getline(*is_, line)) {
      if(!is_->eof()) line+=xml_traits::crlf();
      line+=xml_traits::null();
      if(line_) delete line_;
      line_ = new charT[line.size()];
      std::copy(line.begin(),line.end(),line_);
      return injectString(line_);
    } else {
      return false;
    }
  }
  
  template <class charT,class char_traits,class xml_traits>
  bool Tokenizer<charT,char_traits,xml_traits>::ejectToken(XMLToken<charT,char_traits,xml_traits>& dest){
    //スタックに詰め込まれたトークンが残っていれば、それを放出する。
    if(!stack_.empty()){
      dest = stack_.top(); 
      stack_.pop();
      return true;
    }

    while(*current_==xml_traits::null()){
      if(!fillUpBuffer()){
	token_type tok(current_,current_+1,Token_null,current_-start_);
	dest = tok;
	return false;
      }
    }
    TokenType type;
    if(Token_text!=(type=isToken(*current_))){
      token_type tok(current_,current_+1,type,current_-start_);
      current_++;
      dest = tok;
      return true;
    }
    const charT* mark = current_;
    while(Token_text==isToken(*(current_+1))){
      current_++;
    }
    long size = current_-mark+1;
    token_type tok(mark,mark+size,Token_text,mark-start_);

    current_++;
    dest = tok;
    return true;
  }
    
  template<class charT,class char_traits,class xml_traits>
  TokenType Tokenizer<charT,char_traits,xml_traits>::isToken(charT c){
    int n = 15;//sizeof(tokens_) / sizeof(Char);
    for(int i=0;i<n;++i)
      if(tokens_[i]==c) return static_cast<TokenType>(i);
    return Token_text;
  }
}
