/* Copyright (c) 1991-2002 Doshita Lab. Speech Group, Kyoto University */
/* Copyright (c) 2000-2002 Speech and Acoustics Processing Lab., NAIST */
/*   All rights reserved   */

/* beam.c --- frame-synchronous beam search on 1st pass */
/* use (1-best|word-pair) with static tree structure */

/* $Id: beam.c,v 1.9 2002/09/11 22:02:33 ri Exp $ */

#include <julius.h>

#undef DEBUG

static boolean idc_on;
static int progout_interval_frame;


/* -------------------------------------------------------------------- */
/*                     第１パスの結果出力と終了処理                     */
/*              result output and end procedure of 1st pass             */
/* -------------------------------------------------------------------- */

#ifdef WORD_GRAPH
/* 第１パス結果から単語グラフを生成
   実際には単語トレリス上でグラフ上に残るものにマークを付け,
   (第2パスではマークのついた単語のみ展開する) */
/* generate word graphs from result of 1st pass (=backtrellis) */

static int glevel;		/* depth of generate_lattice() call */
static int gnodes;		/* node num */
static int garcs;		/* arc (=word) num */

/* lookup word on trellis and traverse backward (recursive)
   to mark which trellis words are on the graph */
static void
generate_lattice(
     int endtime,		/* word end frame to lookup */
     WORD_ID wid,		/* word ID to lookup */
     BACKTRELLIS *bt,		/* backtrellis structure */
     WORD_INFO winfo		/* used dictionary (for word output) */
     )
{
  TRELLIS_ATOM *ta, *ta_last;
  int i,j;
  boolean new_node = FALSE;

  glevel++;

  if (endtime >= 0) {
    for (i=0;i<bt->num[endtime];i++) {
      ta = bt->rw[endtime][i];
      if (ta->wid == wid) {
	if (ta->within_wordgraph) {
	  /* already marked */
	} else {
	  /* mark  */
	  ta->within_wordgraph = TRUE;
	  garcs++;
	  new_node = TRUE;	/* new node mark */
	  ta_last = ta->last_tre;
	  if (debug2_flag) {
	    for(j=0;j<glevel;j++) j_printf(" ");
	    j_printf("%s: %d->", winfo->wname[ta->wid], ta->endtime);
	    if (ta_last->wid == WORD_INVALID) {
	      j_printf("%d(WORD_INVALID)\n", ta_last->endtime);
	    } else {
	      j_printf("%d(%s)\n", ta_last->endtime, winfo->wname[ta_last->wid]);
	    }
	  }
	  /* recursive call */
	  generate_lattice(ta_last->endtime, ta_last->wid, bt, winfo);
	}
      }
    }
  }
  if (new_node) {
    gnodes++;
  }
  glevel--;
}
#endif

/* トレリス単語情報を出力 (デバッグ用) */
/* output hypothesis word (= trellis atom) info (for debug) */
static void
put_atom(TRELLIS_ATOM *atom, WORD_INFO *winfo)
{
  int i;
  j_printf("%3d,%3d %f %16s (id=%5d)", atom->begintime, atom->endtime,
	 atom->backscore, winfo->wname[atom->wid], atom->wid);
  for (i=0;i<winfo->wlen[atom->wid]; i++) {
    j_printf(" %s",winfo->wseq[atom->wid][i]->name);
  }
  j_printf("\n");
}

/* 与えられたトレリス単語から始端に向かって単語トレリス集合を
   トレースバックし, そこからの最尤単語系列候補を返す */
/* backtrace the word trellis (=result of 1st pass) from specified
   trellis atom, and return the best sequence
 */

static LOGPROB			/* return value (total language score) */
trace_backptr(WORD_ID wordseq_rt[MAXSEQNUM], /* word sequence (return) */
	      int *rt_wordlen,	/* length of above (return) */
	      TRELLIS_ATOM *atom, /* start backtrace from this atom */
	      BACKTRELLIS *backtrellis,	/* whole backward trellis */
	      WORD_INFO *winfo)	/* used word info (for word output) */
{
  int wordlen = 0;		/* word length of best sentence hypothesis */
  TRELLIS_ATOM *tretmp;
  LOGPROB langscore = 0.0;
  static wordseq[MAXSEQNUM];	/* temporal: in reverse order */
  int i;
  
  /* initialize */
  wordseq[0] = atom->wid;	/* start from specified atom */
  wordlen = 1;
  tretmp = atom;
#ifdef USE_NGRAM
  langscore += tretmp->lscore;
#endif
  if (debug2_flag) {
    put_atom(tretmp, winfo);
  }
  
  /* trace the backtrellis */
  while (tretmp->begintime > 0) {/* until beginning of input */
    tretmp = tretmp->last_tre;
/*    t = tretmp->boundtime - 1;
    tretmp = bt_binsearch_atom(backtrellis, tretmp->boundtime-1, tretmp->last_wid);*/
    if (tretmp == NULL) {	/* should not happen */
      j_error("ERROR: BackTrellis Pass missing??\n");
    }
#ifdef USE_NGRAM
    langscore += tretmp->lscore;
#endif
    wordseq[wordlen] = tretmp->wid;
    wordlen++;
    if (debug2_flag) {
      put_atom(tretmp, winfo);
    }
    if (wordlen >= MAXSEQNUM) {
      j_error("sentence length exceeded ( > %d)\n",MAXSEQNUM);
    }
  }
  *rt_wordlen = wordlen;
  /* reverse order -> normal order */
  for(i=0;i<wordlen;i++) wordseq_rt[i] = wordseq[wordlen-i-1];
  return(langscore);
}

/* 第１パス終了: 結果から最尤仮説のパスを決定, 表示し, pass1_wseq[] に格納 */
/* 返り値: 最尤仮説のスコア */
/* 表示する関数 result_pass1_*() は result_*.c で定義 */

/* end of the 1st pass: determine best hypothesis from resulting trellis,
   output it, and store it to global val 'pass1_wseq[]'.
   real text output functions result_pass1_*() is in result_*.c */
/* return value: score of the hypothesis */
static LOGPROB
print_1pass_result(
     BACKTRELLIS *backtrellis,	/* word trellis */
     int framelen,		/* where 1st pass search ends */
     WORD_INFO *winfo)		/* word info (for word output) */
{
  WORD_ID wordseq[MAXSEQNUM];
  int wordlen;
  int i;
  TRELLIS_ATOM *best;
  int last_time;
  LOGPROB total_lscore;
  LOGPROB maxscore;
  TRELLIS_ATOM *tmp;

  /* look for the first trellis word */
  for (last_time = framelen - 1; last_time >= 0; last_time--) {
#ifdef USE_NGRAM
#ifdef SP_BREAK_CURRENT_FRAME
    /* 最終フレームに残った最大スコアの単語 */
    /* it should be the best trellis word on the last frame */
    maxscore = LOG_ZERO;
    for (i=0;i<backtrellis->num[last_time];i++) {
      tmp = backtrellis->rw[last_time][i];
      if (maxscore < tmp->backscore) {
	maxscore = tmp->backscore;
	best = tmp;
      }
    }
    if (maxscore != LOG_ZERO) break;
#else
    /* 最終単語は winfo->tail_silwid に固定 */
    /* it is fixed to the tail silence model (winfo->tail_silwid) */
    best = bt_binsearch_atom(backtrellis, last_time, winfo->tail_silwid);
    if (best != NULL) break;
#endif
#else  /* USE_DFA */
    /* 末尾に残った単語の中で最大スコアの単語(cp_endは使用しない) */
    /* the best trellis word on the last frame (not use cp_end[]) */
    maxscore = LOG_ZERO;
    for (i=0;i<backtrellis->num[last_time];i++) {
      tmp = backtrellis->rw[last_time][i];
      /*      if (dfa->cp_end[winfo->wton[tmp->wid]] == TRUE) {*/
	if (maxscore < tmp->backscore) {
	  maxscore = tmp->backscore;
	  best = tmp;
	}
	/*      }*/
    }
    if (maxscore != LOG_ZERO) break;
#endif /* USE_NGRAM */
  }
  if (last_time < 0) {		/* not found */
    j_printerr("[no sentence-end word survived on last beam]\n");
    /* print null result */
    result_pass1_final(NULL, 0, LOG_ZERO, LOG_ZERO, winfo);
    return(LOG_ZERO);
  }
  
  /* traceback word trellis from the best word */
  total_lscore = trace_backptr(wordseq, &wordlen, best, backtrellis, winfo);

  if (progout_flag) {		/* just flush last progress output */
    result_pass1_current(last_time, wordseq, wordlen, best->backscore, total_lscore, winfo);
  }

  /* output 1st pass result */    
  if (verbose_flag || !progout_flag) {
    result_pass1_final(wordseq, wordlen, best->backscore, total_lscore, winfo);
  }
  result_pass1_end();
  
  /* store the result to global val (notice: in reverse order) */
  for(i=0;i<wordlen;i++) pass1_wseq[i] = wordseq[i];
  pass1_wnum = wordlen;
  pass1_score = best->backscore;

#ifdef WORD_GRAPH
  /* 単語トレリスから，ラティスを生成する */
  /* 実際には within_wordgraph を on にするだけ */
  /* generate word graph from the word trellis */
  /* actually generate_lattice() does not construct graph:
     it only marks words in trellis that are on the word graph */
  glevel = 0;
  gnodes = 1;			/* frame = 0 の分 */
  garcs = 0;
  generate_lattice(last_time, winfo->tail_silwid, backtrellis, winfo);
  if (verbose_flag) j_printf("word graph generated (nodes=%d,arcs=%d)\n",gnodes, garcs);
#endif

  /* return maximum score */
  return(best->backscore);
}

/* 時間 t でのベストバスを表示 */
/* output best part-of-sentence hypothesis on time t */
static void
bt_current_max(BACKTRELLIS *bt, int t, WORD_INFO *winfo)
{
  static WORD_ID wordseq[MAXSEQNUM];
  int wordlen;
  TRELLIS_ATOM *tre;
  TRELLIS_ATOM *tremax;
  LOGPROB maxscore;
  LOGPROB lscore;

  /* bt->list is ordered by time frame */
  maxscore = LOG_ZERO;
  tremax = NULL;
  tre = bt->list;
  while (tre != NULL && tre->endtime == t) {
    if (maxscore < tre->backscore) {
      maxscore = tre->backscore;
      tremax = tre;
    }
    tre = tre->next;
  }

  if (maxscore != LOG_ZERO) {
    lscore = trace_backptr(wordseq, &wordlen, tremax, bt, winfo);
    result_pass1_current(t, wordseq, wordlen, tremax->backscore, lscore, winfo);
  } else {
    wordlen = 0;
    result_pass1_current(t, wordseq, wordlen, LOG_ZERO, LOG_ZERO, winfo);
  }
}

/* for debug: it only  */
static void
bt_current_max_word(BACKTRELLIS *bt, int t, WORD_INFO *winfo)
{
  TRELLIS_ATOM *tre;
  TRELLIS_ATOM *tremax;
  LOGPROB maxscore;
  WORD_ID w;

  /* bt->list は時間順に格納されている */
  maxscore = LOG_ZERO;
  tremax = NULL;
  tre = bt->list;
  while (tre != NULL && tre->endtime == t) {
    if (maxscore < tre->backscore) {
      maxscore = tre->backscore;
      tremax = tre;
    }
    tre = tre->next;
  }

  if (maxscore != LOG_ZERO) {
    j_printf("%3d: ",t);
    w = tremax->wid;
    j_printf("\"%s [%s]\"(id=%d)",
	   winfo->wname[w], winfo->woutput[w], w);
    j_printf(" [%d-%d] %f <- ", tremax->begintime, t, tremax->backscore);
    w = tremax->last_tre->wid;
    if (w != WORD_INVALID) {
      j_printf("\"%s [%s]\"(id=%d)\n",
	     winfo->wname[w], winfo->woutput[w], w);
    } else {
      j_printf("bgn\n");
    }
  }
}

/* -------------------------------------------------------------------- */
/*                 ビーム探索中のトークンを扱うサブ関数                 */
/*                functions to handle hypothesis tokens                  */
/* -------------------------------------------------------------------- */

/* Token structure:
   
/* How tokens are managed:
   o  tlist[][] is a token stocker.  It holds all tokens in sequencial
      buffer.  They are malloced first on startup, and refered by ID while
      Viterbi procedure.  In word-pair mode, each token also has a link to
      another token to allow a node to have more than 1 token.
      
   o  token[n] holds the current ID number of a token associated to a
      lexicon tree node 'n'.

 */

static TOKEN2 *tlist[2];	/* sequencial list of all tokens */
static TOKENID *tindex[2];	/* index for sort */
static int tnum[2];		/* current number of tokens used in tlist */
static int maxtnum;		/* maximum number of tokens */
static int expand_step;		/* step to expand token */
static TOKENID *token;		/* sequencial list per tree node */
static int totalnodenum;	/* allocated node num of above */
static int tl,tn;		/* node id flip/flop */
static int n_start, n_end;	/* (for heap sort) */

static TRELLIS_ATOM bos;

/* malloc work area */
static void
malloc_nodes(int n)
{
  totalnodenum = n;
  token        = mymalloc(sizeof(TOKENID)*totalnodenum);
  maxtnum = n;
  tlist[0]     = mymalloc(sizeof(TOKEN2)*maxtnum);
  tlist[1]     = mymalloc(sizeof(TOKEN2)*maxtnum);
  tindex[0]     = mymalloc(sizeof(TOKENID)*maxtnum);
  tindex[1]     = mymalloc(sizeof(TOKENID)*maxtnum);
  tnum[0] = tnum[1] = 0;
  expand_step = n;
}
/* expand work area */
static void
expand_tlist()
{
  maxtnum += expand_step;
  tlist[0]     = myrealloc(tlist[0],sizeof(TOKEN2)*maxtnum);
  tlist[1]     = myrealloc(tlist[1],sizeof(TOKEN2)*maxtnum);
  tindex[0]     = myrealloc(tindex[0],sizeof(TOKENID)*maxtnum);
  tindex[1]     = myrealloc(tindex[1],sizeof(TOKENID)*maxtnum);
  j_printerr("warn: token space expanded\n");
}
/* free work area */
static void
free_nodes()
{
  free(token);
  free(tlist[0]);
  free(tlist[1]);
  free(tindex[0]);
  free(tindex[1]);
}

/* initialize token buffer */
static void
clear_tlist(int tt)
{
  tnum[tt] = 0;
}
/* initialize token buffer: only clear ones used in the last call */
static void
clear_tokens(int tt)
{
  int j;
  for (j=0; j<tnum[tt]; j++) {
    token[tlist[tt][j].node] = TOKENID_UNDEFINED;
  }
}
/* return a new token ID */
static TOKENID
create_token()
{
  TOKENID newid;
  newid = tnum[tn];
  tindex[tn][newid] = newid;
  tnum[tn]++;
  if (tnum[tn]>maxtnum) expand_tlist();
#ifdef WPAIR
  tlist[tn][newid].next = TOKENID_UNDEFINED;
#endif
  return(newid);
}

/* assign token to a node */
static void
node_assign_token(int node, TOKENID tkid)
{
#ifdef WPAIR
  /* add to list */
  tlist[tn][tkid].next = token[node];
#endif
  token[node] = tkid;
  tlist[tn][tkid].node = node;
}

/* check if token with last word='wid' exist on 'node' */
/* return the to-be-replaced token ID (TOKENID_UNDEFINED if not found) */
static TOKENID
node_exist_token(int tt, int node, WORD_ID wid)
{
#ifdef WPAIR
  /* In word-pair mode, multiple tokens are assigned to a node as a list.
     so we have to search for tokens with same last word ID */
#ifdef WPAIR_KEEP_NLIMIT
  /* 1ノードごとに保持するtoken数の上限を設定 */
  /* tokenが無いが上限に達しているときは一番スコアの低いtokenを上書きする */
  /* N-best: limit number of assigned tokens to a node */
  int i = 0;
  TOKENID lowest_token = TOKENID_UNDEFINED;
#endif
  TOKENID tmp;
  for(tmp=token[node]; tmp != TOKENID_UNDEFINED; tmp=tlist[tt][tmp].next) {
    if (tlist[tt][tmp].last_tre->wid == wid) {
      return(tmp);
    }
#ifdef WPAIR_KEEP_NLIMIT
    if (lowest_token == TOKENID_UNDEFINED ||
	tlist[tt][lowest_token].score < tlist[tt][tmp].score)
      lowest_token = tmp;
    if (++i >= wpair_keep_nlimit) break;
#endif
  }
#ifdef WPAIR_KEEP_NLIMIT
  if (i >= wpair_keep_nlimit) { /* overflow, overwrite lowest score */
    return(lowest_token);
  } else {
    return(TOKENID_UNDEFINED);
  }
#else 
  return(TOKENID_UNDEFINED);
#endif
  
#else  /* not WPAIR */
  /* 1つだけ保持,これを常に上書き */
  /* Only one token is kept in 1-best mode (default), so
     simply return the ID */
  return(token[node]);
#endif
}

#ifdef DEBUG
/* tlist と token の対応をチェックする(debug) */
/* for debug: check tlist <-> token correspondence
   where  tlist[tt][tokenID].node = nodeID and
          token[nodeID] = tokenID
 */
static void
node_check_token(int tt)
{
  int i;
  for(i=0;i<tnum[tt];i++) {
    if (node_exist_token(tt, tlist[tt][i].node, tlist[tt][i].last_tre->wid) != i) {
      j_printerr("token %d not found on node %d\n", i, tlist[tt][i].node);
    }
  }
}
#endif



/* -------------------------------------------------------------------- */
/*       トークンをソートし 上位 N トークンを判別する (heap sort)       */
/*        Sort generated tokens and get N-best (use heap sort)          */
/* -------------------------------------------------------------------- */
/* ビームの閾値として上位 N 番目のスコアが欲しいだけであり，実際にソート
   される必要はない */
/* we only want to know the N-th score for determining beam threshold, so
   order is not considered here */

#define SD(A) tindex[tn][A-1]
#define SCOPY(D,S) D = S
#define SVAL(A) (tlist[tn][tindex[tn][A-1]].score)
#define STVAL (tlist[tn][s].score)

/* sort tokens upward to find 'neednum'-best score */
/* 大きい順にソートして上位 neednum 個が見つかったら終わり */
static void
sort_token_upward(int neednum, int totalnum)
{
  int n,root,child,parent;
  TOKENID s;
  for (root = totalnum/2; root >= 1; root--) {
    SCOPY(s, SD(root));
    parent = root;
    while ((child = parent * 2) <= totalnum) {
      if (child < totalnum && SVAL(child) < SVAL(child+1)) {
	child++;
      }
      if (STVAL >= SVAL(child)) {
	break;
      }
      SCOPY(SD(parent), SD(child));
      parent = child;
    }
    SCOPY(SD(parent), s);
  }
  n = totalnum;
  while ( n > totalnum - neednum) {
    SCOPY(s, SD(n));
    SCOPY(SD(n), SD(1));
    n--;
    parent = 1;
    while ((child = parent * 2) <= n) {
      if (child < n && SVAL(child) < SVAL(child+1)) {
	child++;
      }
      if (STVAL >= SVAL(child)) {
	break;
      }
      SCOPY(SD(parent), SD(child));
      parent = child;
    }
    SCOPY(SD(parent), s);
  }
}

/* sort tokens downward to find 'neednum'-lowest score */
/* 小さい順にソートして下位 neednum 個が見つかったら終わり */
static void
sort_token_downward(int neednum, int totalnum)
{
  int n,root,child,parent;
  TOKENID s;
  for (root = totalnum/2; root >= 1; root--) {
    SCOPY(s, SD(root));
    parent = root;
    while ((child = parent * 2) <= totalnum) {
      if (child < totalnum && SVAL(child) > SVAL(child+1)) {
	child++;
      }
      if (STVAL <= SVAL(child)) {
	break;
      }
      SCOPY(SD(parent), SD(child));
      parent = child;
    }
    SCOPY(SD(parent), s);
  }
  n = totalnum;
  while ( n > totalnum - neednum) {
    SCOPY(s, SD(n));
    SCOPY(SD(n), SD(1));
    n--;
    parent = 1;
    while ((child = parent * 2) <= n) {
      if (child < n && SVAL(child) > SVAL(child+1)) {
	child++;
      }
      if (STVAL <= SVAL(child)) {
	break;
      }
      SCOPY(SD(parent), SD(child));
      parent = child;
    }
    SCOPY(SD(parent), s);
  }
}

/* ソートする（上向き・下向きで早い方を使う） */
/* sort tokens up to need num */
/* sort upward till need num, or sort downward tille (totalnum-neednum) */
/* start, end: return value of range where [0..neednum-1] tokens exist */
static void
sort_token_no_order(int neednum, int *start, int *end)
{
  int totalnum = tnum[tn];
  int restnum;

  restnum = totalnum - neednum;

  if (neednum >= totalnum) {
    /* no need to sort */
    *start = 0;
    *end = totalnum - 1;
  } else if (neednum < restnum)  {
    sort_token_upward(neednum,totalnum);
    *start = totalnum - neednum;
    *end = totalnum - 1;
  } else {
    sort_token_downward(restnum,totalnum);
    *start = 0;
    *end = neednum - 1;
  }
}
    

#ifdef SP_BREAK_CURRENT_FRAME
/* -------------------------------------------------------------------- */
/*                     ショートポーズ・セグメンテーション               */
/*                     short pause segmentation                         */
/* -------------------------------------------------------------------- */
/* ====================================================================== */
/* sp segmentation */
/*
  |---************-----*********************-------*******--|
[input segments]
  |-------------------|
                  |-------------------------------|
		                            |---------------|

		     
  |-------------------|t-2
                       |t-1 ... token processed (= lastlen)
		        |t  ... outprob computed
		       
*/

static boolean in_sparea;	/* TRUE if we are within a pause area */
static int sparea_start;	/* start frame of pause area */
static int tmp_sparea_start;
#ifdef SP_BREAK_RESUME_WORD_BEGIN
static WORD_ID tmp_sp_break_last_word;
#else
static WORD_ID last_tre_word;
#endif
static boolean first_sparea;	/* TRUE if this is the first pause segment in a input stream */
static int sp_duration;		/* number of sp frame to be durated */

/* ショートポーズの連続を見て, 入力の切目を検出する */
/* 第１パスが１フレーム進むごとに呼ばれる */
/* watch duration of short-pause words and detect end of input segment */
/* called for every frame in 1st pass */
/* return value: TRUE if segment end here, FALSE if continue */
static boolean
detect_end_of_segment(BACKTRELLIS *backtrellis, int time, WORD_INFO *winfo)
{
  TRELLIS_ATOM *tre;
  LOGPROB maxscore = LOG_ZERO;
  TRELLIS_ATOM *tremax = NULL;
  int count = 0;
  char *p;
  boolean detected = FALSE;

  /* look for the best trellis word on the given time frame */
  for(tre = backtrellis->list; tre != NULL && tre->endtime == time; tre = tre->next) {
    if (maxscore < tre->backscore) {
      maxscore = tre->backscore;
      tremax = tre;
    }
    count++;
  }
  if (tremax == NULL) {	/* no word end: possible in the very beggining of input*/
    detected = TRUE;		/* assume it's in the short-pause duration */
  } else if (count > 0) {	/* many words found --- check if maximum is sp */
    p = winfo->wseq[tremax->wid][0]->name;
    if (is_sil(p)) {
      detected = TRUE;
    }
  }
  
  /* sp区間持続チェック */
  /* check sp segment duration */
  if (in_sparea && detected) {	/* we are already in sp segment and sp continues */
    sp_duration++;		/* increment count */
#ifdef SP_BREAK_RESUME_WORD_BEGIN
    /* resume word at the "beggining" of sp segment */
    /* if this segment has triggered by (tremax == NULL) (in case the first
       several frame of input), the sp word (to be used as resuming
       word in the next segment) is not yet set. it will be detected here */
    if (tmp_sp_break_last_word == WORD_INVALID) {
      if (tremax != NULL) tmp_sp_break_last_word = tremax->wid;
    }
#else
    /* resume word at the "end" of sp segment */
    /* simply update the best sp word */
    if (tremax != NULL) last_tre_word = tremax->wid;
#endif
  }

  /* sp区間開始チェック */
  /* check if sp segment begins at this frame */
  else if (!in_sparea && detected) {
    /* 一時的に開始フレームとしてマーク */
    /* mark this frame as "temporal" begging of short-pause segment */
    tmp_sparea_start = time;
#ifdef SP_BREAK_RESUME_WORD_BEGIN
    /* sp 区間開始時点の最尤単語を保存 */
    /* store the best word in this frame as resuming word */
    tmp_sp_break_last_word = tremax ? tremax->wid : WORD_INVALID;
#endif
    in_sparea = TRUE;		/* yes, we are in sp segment */
    sp_duration = 1;		/* initialize duration count */
#ifdef SP_BREAK_DEBUG
    printf("sp start %d\n", time);
#endif /* SP_BREAK_DEBUG */
  }
  
  /* sp 区間終了チェック */
  /* check if sp segment ends at this frame */
  else if (in_sparea && !detected) {
    /* (time-1) is end frame of pause segment */
    in_sparea = FALSE;		/* we are not in sp segment */
#ifdef SP_BREAK_DEBUG
    printf("sp end %d\n", time);
#endif /* SP_BREAK_DEBUG */
    /* sp 区間長チェック */
    /* check length of the duration*/
    if (sp_duration < sp_frame_duration) {
      /* 短すぎる: 第１パスを中断せず続行 */
      /* too short segment: not break, continue 1st pass */
#ifdef SP_BREAK_DEBUG
      printf("too short (%d<%d), ignored\n", sp_duration, sp_frame_duration);
#endif /* SP_BREAK_DEBUG */
    } else if (first_sparea) {
      /* 最初のsp区間は silB にあたるので,第１パスを中断せず続行 */
      /* do not break at first sp segment: they are silB */
      first_sparea = FALSE;
#ifdef SP_BREAK_DEBUG
      printf("first silence, ignored\n");
#endif /* SP_BREAK_DEBUG */
    } else {
      /* 区間終了確定, 第１パスを中断して第2パスへ */
      /* break 1st pass */
#ifdef SP_BREAK_DEBUG
      printf(">> segment [%d..%d]\n", sparea_start, time-1);
#else
      j_printerr("|");
#endif /* SP_BREAK_DEBUG */
      /* store begging frame of the segment */
      sparea_start = tmp_sparea_start;
#ifdef SP_BREAK_RESUME_WORD_BEGIN
      /* resume word = most likely sp word on beginning frame of the segment */
      sp_break_last_word = tmp_sp_break_last_word;
#else
      /* resume word = most likely sp word on end frame of the segment */
      sp_break_last_word = last_tre_word;
#endif

      /*** segment: [sparea_start - time-1] ***/
      return(TRUE);
    }
  }
    
#ifdef SP_BREAK_EVAL
  printf("[%d %d %d]\n", time, count, (detected) ? 50 : 0);
#endif
  return (FALSE);
}

#endif /* SP_BREAK_CURRENT_FRAME */



/* -------------------------------------------------------------------- */
/*             第１パス(フレーム同期ビームサーチ) メイン                */
/*           main routines of 1st pass (frame-synchronous beam search)  */
/* -------------------------------------------------------------------- */

/* 初期化 */
/* initialization */
static void
init_nodescore(HTK_Param *param, WCHMM_INFO *wchmm)
{
  TOKENID newid;
  TOKEN2 *new;
  WORD_ID beginword;
  int node;
  int i, j;

  /* 初期仮説用単語履歴 */
  /* setup initial word context */
#ifdef SP_BREAK_CURRENT_FRAME
  /* initial word context = last non-sp word of previous 2nd pass at last segment*/
  if (sp_break_last_nword == wchmm->winfo->tail_silwid) {
    /* if end with silE, initialize as normal start of sentence */
    bos.wid = WORD_INVALID;
  } else {
    bos.wid = sp_break_last_nword;
  }
#else
  bos.wid = WORD_INVALID;	/* no context */
#endif
  bos.begintime = bos.endtime = -1;

  /* ノード・トークンを初期化 */
  /* clear tree lexicon nodes and tokens */
  for(node=0;node<totalnodenum;node++) {
    token[node] = TOKENID_UNDEFINED;
  }
  tnum[0] = tnum[1]  = 0;
  
#ifdef PASS1_IWCD
  /* 出力確率計算キャッシュを初期化 */
  /* initialize outprob cache */
  outprob_style_cache_init(wchmm);
#endif

  /* 初期仮説の作成: 初期単語の決定と初期トークンの生成 */
  /* initial word hypothesis */
#ifdef USE_NGRAM
  
#ifdef SP_BREAK_CURRENT_FRAME
  if (sp_break_last_word != WORD_INVALID) { /* last segment exist */
    /* 開始単語＝前のセグメント計算時の最後の最尤単語 */
    /* 文終了単語(silE,句点(IPAモデル))なら，silB で開始 */
    /* initial word = best last word hypothesis on the last segment */
    /* if silE or sp, begin with silB */
    if (sp_break_last_word == wchmm->winfo->tail_silwid ||
	sp_break_last_word == 4) { /* sp: super ad-hoc, need fix... */
      beginword = wchmm->winfo->head_silwid;
      bos.wid = WORD_INVALID;	/* reset initial word context */
    } else {
      beginword = sp_break_last_word;
    }
  } else {
    /* initial word fixed to silB */
    beginword = wchmm->winfo->head_silwid;
  }
#else
  /* initial word fixed to silB */
  beginword = wchmm->winfo->head_silwid;
#endif
#ifdef SP_BREAK_DEBUG
  printf("startword=[%s], last_nword=[%s]\n",
	 (beginword == WORD_INVALID) ? "WORD_INVALID" : wchmm->winfo->wname[beginword],
	 (bos.wid == WORD_INVALID) ? "WORD_INVALID" : wchmm->winfo->wname[bos.wid]);
#endif
  /* create the first token at the first node of initial word */
  node = wchmm->wordbegin[beginword];
  newid = create_token();
  new = &(tlist[tn][newid]);
  if (wchmm->state[node].scid != -1
#ifdef UNIGRAM_FACTORING
      || wchmm->state[node].fscore != LOG_ZERO
#endif
      ) {
    new->last_lscore = max_successor_prob(wchmm, bos.wid, node);
  } else {
    new->last_lscore = 0.0;
  }
#ifdef FIX_PENALTY
  new->last_lscore = new->last_lscore * lm_weight;
#else
  new->last_lscore = new->last_lscore * lm_weight + lm_penalty;
#endif
  new->last_tre = &bos;
  new->last_cword = bos.wid;
  new->score = new->last_lscore;
  node_assign_token(node, newid);
      
#else  /* USE_DFA */
  
  /* 初期仮説: 文法上文頭に接続しうる単語集合 */
  /* initial words: all words that can be begin of sentence grammatically */
  /* アクティブな文法に属する単語のみ許す */
  /* only words in active grammars are allowed to be an initial words */
  {
    MULTIGRAM *m;
    int t,tb,te;
    WORD_ID iw;

    for(m = gramlist; m; m = m->next) {
      if (m->active) {
	tb = m->cate_begin;
	te = tb + m->dfa->term_num;
	for(t=tb;t<te;t++) {
	  if (dfa_cp_begin(dfa, t) == TRUE) {
	    for (iw=0;iw<dfa->term.wnum[t];iw++) {
	      i = dfa->term.tw[t][iw];
	      /* create the first token at the first node of the beggning word */
	      node = wchmm->wordbegin[i];
	      /* words in the same category share the same root node, so
		 skip if already exist */
	      if (node_exist_token(tn, node, bos.wid) != TOKENID_UNDEFINED) continue;
	      newid = create_token();
	      new = &(tlist[tn][newid]);
	      new->last_tre = &bos;
	      new->score = 0.0;
	      node_assign_token(node, newid);
	    }
	  }
	}
      }
    }
  }
/* 
 *   for (i=0;i<wchmm->winfo->num;i++) {
 *     if (dfa->cp_begin[wchmm->winfo->wton[i]] == TRUE) {
 *	 node = wchmm->offset[i][0];
 *	 if (node_exist_token(tn, node, bos.wid) != TOKENID_UNDEFINED) continue;
 *	 newid = create_token();
 *	 new = &(tlist[tn][newid]);
 *	 new->last_tre = &bos;
 *	 new->score = outprob_style(wchmm, node, bos.wid, 0, param);
 *	 node_assign_token(node, newid);
 *     }
 *   }
 */
  
#endif /* USE_DFA */
}

/******************************************************/
/* フレーム同期ビーム探索の実行 --- 最初のフレーム用  */
/* frame synchronous beam search --- first frame only */
/******************************************************/
void
get_back_trellis_init(HTK_Param *param,	/* parameter vectors of speech data (only t=0 used in this function) */
		      WCHMM_INFO *wchmm, /* lexicon tree */
		      BACKTRELLIS *backtrellis	/* backward trellis (return value) */
		      )
{
  
  /* Viterbi演算用ワークエリアのスイッチャー tl,tn の初期値設定 */
  /* tn: このフレーム用ID   tl: １フレーム前のID */
  /* initialize switch tl, tn for Viterbi computation */
  /* tn: this frame  tl: last frame */
  tn = 0;
  tl = 1;

  /* 結果の単語トレリスを格納するバックトレリス構造体を初期化 */
  /* initialize backtrellis structure to store resulting word trellis */
  bt_prepare(backtrellis);

  /* ワークエリアを確保 */
  /* malloc work area */
  malloc_nodes(wchmm->n);
  
  /* 初期スコアを nodescore[tn] にセット */
  /* set initial score to nodescore[tn] */
  init_nodescore(param, wchmm);
  sort_token_no_order(trellis_beam_width, &n_start, &n_end);

  /* テキスト出力を初期化 */
  /* initialize message output */
  result_pass1_begin();
  /* 漸次出力を行なう場合のインターバルを計算 */
  /* set interval frame for progout */
  progout_interval_frame = (float)progout_interval / ((float)param->header.wshift / 10000.0);

  /* 進行状況表示を初期化 */
  /* progress bar setting */
  if (!realtime_flag && verbose_flag && (!progout_flag) && isatty(1)) {
    idc_on = TRUE;
  } else { 
    idc_on = FALSE;
  }
  
#ifdef SP_BREAK_CURRENT_FRAME
  /* ショートポーズセグメンテーション用パラメータの初期化 */
  /* initialize parameter for short pause segmentation */
  in_sparea = TRUE;		/* assume beginning is silence */
  sparea_start = tmp_sparea_start = 0; /* set start frame to 0 */
#ifdef SP_BREAK_RESUME_WORD_BEGIN
  tmp_sp_break_last_word = WORD_INVALID;
#endif
  sp_break_last_word = WORD_INVALID;
  /* 最初のセグメント: 次の非ポーズフレームで第2パスへ移行しない */
  /* the first end of pause segment should be always silB, so
     skip the first segment */
  first_sparea = TRUE;
  sp_break_2_begin_word = WORD_INVALID;
#endif
}

/* local work area for get_back_trellis_proceed() */
static LOGPROB maxscore, minscore;
static LOGPROB tmpprob;
static TRELLIS_ATOM *tre;
static int node, stid, isoid;
static A_CELL *ac;
static LOGPROB *iwparray;
static int next_node2;
#ifdef UNIGRAM_FACTORING
/* for wordend processing with 1-gram factoring */
static LOGPROB wordend_best_score; /* best score of wordend nodes */
static int wordend_best_node;	/* node id of the best wordend nodes */
static TRELLIS_ATOM *wordend_best_tre; /* trellis word corresponds to above */
static WORD_ID wordend_best_last_cword;	/* last_cword of above */
#endif

/*****************************************************/
/* frame synchronous beam search --- proceed 1 frame */
/* フレーム同期ビーム探索の実行 --- 1フレーム進める  */
/*****************************************************/
boolean				/* return value: FALSE if segmented */
get_back_trellis_proceed(
     int t,			/* frame t*/
     HTK_Param *param,		/* parameter vector of speech data (use only t) */
     WCHMM_INFO *wchmm,
     BACKTRELLIS *backtrellis,	/* backward trellis (return value) */
     boolean final		/* if this is final frame, set this to TRUE */
     )
{
  LOGPROB tmpsum;
  TOKEN2  *tk, *tknext;
  TOKENID  tknextid;
#ifdef USE_NGRAM
  LOGPROB ngram_score_cache;
#endif
  int j, next_node, sword;

  /*********************/
  /* 1. 初期化         */
  /*    initialization */
  /*********************/

  /* tl と tn を入れ替えて作業領域を切り替え */
  /* tl (= 直前の tn) は直前フレームの結果を持つ */
  /* swap tl and tn to switch work buffer */
  /* tl (= last tn) holds result of the previous frame */
  tl = tn;
  if (tn == 0) tn = 1; else tn = 0;

  /* 進行状況表示を更新 */
  /* update progress bar */
  if (idc_on) {
#ifdef SP_BREAK_CURRENT_FRAME
    if (in_sparea) j_printerr("."); else j_printerr("-");
#else  /* normal */
    j_printerr(".");
#endif /* SP_BREAK_CURRENT_FRAME */
  }

  /* 最大/最小スコア値を初期化 */
  /* initialize max/min score */
  maxscore = LOG_ZERO;
  minscore = 0;

#ifdef UNIGRAM_FACTORING
  /* 1-gram factoring では単語先頭での言語確率が一定で直前単語に依存しない
     ため，単語間 Viterbi において選ばれる直前単語は,次単語によらず共通である．
     よって単語終端からfactoring値のある単語先頭への遷移は１つにまとめられる．
     ただし，木から独立した単語については, 単語先頭で履歴に依存した2-gramが
     与えられるため, 最尤の単語間 Viterbi パスは次単語ごとに異なる．
     よってそれらについてはまとめずに別に計算する */
  /* In 1-gram factoring, the language score on the word-head node is constant
     and independent of the previous word.  So, the same word hypothesis will
     be selected as the best previous word at the inter-word Viterbi
     processing.  So, in inter-word processing, we can (1) select only
     the best word-end hypothesis, and then (2) process viterbi from the node
     to each word-head node.  On the other hand, the isolated words,
     i.e. words not sharing any node with other word, has unique word-head
     node and the true 2-gram language score is determined at the top node.
     In such case the best word hypothesis prior to each node will differ
     according to the language scores.  So we have to deal such words separately. */
  /* initialize max value to delect best word-end hypothesis */
  wordend_best_score = LOG_ZERO;
#endif

#ifdef DEBUG
  /* debug */
  /* node_check_token(tl); */
#endif

  /* トークンバッファを初期化: 直前フレームで使われた部分だけクリアすればよい */
  /* initialize token buffer: for speedup, only ones used in the last call will be cleared */
  clear_tokens(tl);

  /****************************/
  /* 2. 単語内Viterbi計算     */
  /*    word-internal Viterbi */
  /****************************/
  /* 直前フレームからこのフレームへの Viterbi 計算を行なう */
  /* tindex[tl][n_start..n_end] に直前フレーム上位ノードのIDが格納されている */
  /* do one viterbi computation from last frame to this frame */
  /* tindex[tl][n_start..n_end] holds IDs of survived nodes in last frame */
  for (j=n_start;j<=n_end;j++) {

    /* tk: 対象トークン  node: そのトークンを持つ木構造化辞書ノードID */
    /* tk: token data  node: lexicon tree node ID that holds the 'tk' */
    tk = &(tlist[tl][tindex[tl][j]]);
    node = tk->node;
    if (tk->score <= LOG_ZERO) continue; /* invalid node */

    for (ac = wchmm->state[node].ac; ac; ac = ac->next) {
      next_node = ac->arc;

      /******************************************************************/
      /* 2.1   遷移先へのスコア計算(遷移確率＋言語スコア)               */
      /*       compute score of destination node (transition prob + LM) */
      /******************************************************************/
      tmpsum = tk->score + ac->a;
#ifdef USE_NGRAM
      ngram_score_cache = LOG_ZERO;
#endif

#ifndef CATEGORY_TREE
      /* 言語スコア factoring:
         arcが自己遷移でない単語内の遷移で，かつ遷移先にsuccessorリスト
	 があれば，lexicon tree の分岐部分の遷移である */
      /* LM factoring:
	 If arc is not a self transition and destination node has successor
	 list, this is branching transition
       */
      if (next_node != node) {
	if (wchmm->state[next_node].scid != -1
#ifdef UNIGRAM_FACTORING
	    /* 1-gram factoring 使用時は, 複数で共有される枝では
	       wchmm->state[node].sc は NULL となり，それを共有している
	       単語集合の1-gramの最大値が wchmm->state[node].fscore に格納
	       されている．末端の枝(複数単語で共有されない)では，その
	       １単語を sc として持つので正確な2-gramを計算 */
	    /* When uni-gram factoring,
	       Shared branches: has maximum uni-gram probability for sub-tree
	                        in wchmm->state[node].fscore, and
				wchmm->state[node].sc set to NULL.
	       Leaf branches (with only one successor word): has
	       the word ID in wchmm->state[node].sc.  So precise 2-gram
	       is computed in this point */
	    || wchmm->state[next_node].fscore != LOG_ZERO
#endif
	    ){
#ifdef USE_NGRAM
	  /* N-gram確率からfactoring 値を計算 */
	  /* compute new factoring value from N-gram probabilities */
#ifdef FIX_PENALTY
	  if (tk->last_cword == WORD_INVALID) {
	    ngram_score_cache = max_successor_prob(wchmm, tk->last_cword, next_node) * lm_weight;
	  } else {
	    ngram_score_cache = max_successor_prob(wchmm, tk->last_cword, next_node) * lm_weight + lm_penalty;
	  }
#else
	  ngram_score_cache = max_successor_prob(wchmm, tk->last_cword, next_node) * lm_weight + lm_penalty;
#endif
	  /* スコアの更新: tk->last_lscore に単語内での最後のfactoring値 */
	  /* update score: tk->last_lscore holds the last factoring value in this word */
	  tmpsum -= tk->last_lscore;
	  tmpsum += ngram_score_cache;
	  
#else  /* USE_DFA --- not CATEGORY_TREE */
	  
          /* 決定的factoring: 直前単語に対して,sub-tree内にカテゴリ対制約上
	     接続しうる単語が１つもなければ, この遷移は不可 */
	  /* deterministic factoring in grammar mode:
	     transition disabled if there are totally no sub-tree word that can
	     grammatically (in category-pair constraint) connect
	     to the previous word.
	   */
          if (!can_succeed(wchmm, tk->last_tre->wid, next_node)) {
            tmpsum = LOG_ZERO;
          }
	  
#endif /* USE_NGRAM */
	}
      }
#endif /* CATEGORY_TREE */
      /* factoring not needed when DFA mode and uses category-tree */

      /****************************************/
      /* 2.2   遷移先ノードへトークン伝搬     */
      /*       pass token to destination node */
      /****************************************/

      if ((tknextid = node_exist_token(tn, next_node, tk->last_tre->wid)) != TOKENID_UNDEFINED) {
	/* 遷移先ノードには既に他ノードから伝搬済み: スコアが高いほうを残す */
	/* the destination node already has a token: compare score */
	tknext = &(tlist[tn][tknextid]);
	if (tknext->score < tmpsum) {
	  /* その遷移先ノードが持つトークンの内容を上書きする(新規トークンは作らない) */
	  /* overwrite the content of existing destination token: not create a new token */
	  tknext->last_tre = tk->last_tre;
#ifdef USE_NGRAM
	  tknext->last_cword = tk->last_cword;
	  tknext->last_lscore = (ngram_score_cache != LOG_ZERO) ? ngram_score_cache : tk->last_lscore;
#endif /* USE_NGRAM */
	  tknext->score = tmpsum;
	}
      } else {
	/* 遷移先ノードは未伝搬: 新規トークンを作って割り付ける */
	/* token unassigned: create new token and assign */
	if (tmpsum > LOG_ZERO) { /* valid token */
	  tknextid = create_token();
	  tknext = &(tlist[tn][tknextid]);
	  tk = &(tlist[tl][tindex[tl][j]]);
	  tknext->last_tre = tk->last_tre;
#ifdef USE_NGRAM
	  tknext->last_cword = tk->last_cword;
	  tknext->last_lscore = (ngram_score_cache != LOG_ZERO) ? ngram_score_cache : tk->last_lscore;
#endif /* USE_NGRAM */
	  tknext->score = tmpsum;
	  node_assign_token(next_node, tknextid);
	}
      }
    }
  }
  
  /*******************************************************/
  /* 3. スコアでトークンをソートしビーム幅分の上位を決定 */
  /*    sort tokens by score up to beam width            */
  /*******************************************************/
  /* ヒープソートを用いてこの段のノード集合から上位(bwidth)個を得ておく */
  /* (上位内の順列は必要ない) */
  sort_token_no_order(trellis_beam_width, &n_start, &n_end);

  /*************************/
  /* 4. 単語間Viterbi計算  */
  /*    cross-word viterbi */
  /*************************/

  for(j=n_start; j<=n_end; j++) {
    tk = &(tlist[tn][tindex[tn][j]]);
    node = tk->node;
    
    /* 単語終端のNULLノードだったならば */
    /* if it is the end null state of a word, */
    if (wchmm->stend[node] != WORD_INVALID) {
      sword = wchmm->stend[node];

      /**************************/
      /* 4.1. トレリス単語保存  */
      /*      save trellis word */
      /**************************/

      /* この遷移元の単語終端ノードは「直前フレームで」生き残ったノード．
	 (「このフレーム」でないことに注意！！)
	 よってここで, 時間(t-1) を単語終端とするトレリス上の単語仮説
	 (TRELLIS_ATOM)として，単語トレリス構造体に保存する．*/
      /* This source node (= word end node) has been survived in the
	 "last" frame (notice: not "this" frame!!).  So this word end
	 is saved here to the word trellis structure (BACKTRELLIS) as a
	 trellis word (TRELLIS_ATOM) with end frame (t-1). */
      tre = (TRELLIS_ATOM *)mymalloc(sizeof(TRELLIS_ATOM));
      tre->wid = sword;		/* word ID */
      tre->backscore = tk->score; /* log score (AM + LM) */
      tre->begintime = tk->last_tre->endtime + 1; /* word beginning frame */
      tre->endtime   = t-1;	/* word end frame */
      tre->last_tre  = tk->last_tre; /* previous word context */
#ifdef USE_NGRAM
      tre->lscore    = tk->last_lscore;	/* log score (LM only) */
#endif
      bt_store(backtrellis, tre); /* save to backtrellis */
      
      /******************************/
      /* 4.2. 単語間遷移            */
      /*      cross-word transition */
      /******************************/
      /* 最終フレームであればここまで：遷移はさせない */
      if (final) continue;

      /* 先ほどのトレリス単語treが，ここからの遷移先の単語先頭ノード以降の
	 直前単語情報としても参照される */
      /* The trellis atom 'tre' will be refered as the word context
	 information for next word-beginning nodes */

#ifdef UNIGRAM_FACTORING
      /* ここで処理されるのは isolated words のみ，
	 shared nodes はまとめてこのループの外で計算する */
      /* Only the isolated words will be processed here.
	 The shared nodes with constant factoring values will be computed
	 after this loop */
#endif

#ifdef USE_NGRAM
      /* 遷移元単語が末尾単語の終端なら，どこへも遷移させない */
      /* do not allow transition if the source word is end-of-sentence word */
      if (sword == wchmm->winfo->tail_silwid) continue;

#ifdef UNIGRAM_FACTORING
      /* 最大尤度を持つ単語終端ノードを記録しておく */
      /* find the max wordend node */
      if (wordend_best_score < tk->score) {
	wordend_best_score = tk->score;
	wordend_best_node = node;
	wordend_best_tre = tre;
	wordend_best_last_cword = tk->last_cword;
      }
#endif
      
      /* N-gramにおいては常に全単語の接続を考慮する必要があるため，
	 ここで単語間の言語確率値をすべて計算しておく．
	 キャッシュは max_successor_prob_iw() 内で考慮．*/
      /* As all words are possible to connect in N-gram, we first compute
	 all the inter-word LM probability here.
	 Cache is onsidered in max_successor_prob_iw(). */
      if (wchmm->winfo->is_transparent[sword]) {
	iwparray = max_successor_prob_iw(wchmm, tk->last_cword);
      } else {
	iwparray = max_successor_prob_iw(wchmm, sword);
      }
#endif

      /* すべての単語始端ノードに対して以下を実行 */
      /* for all beginning-of-word nodes, */
      /* wchmm.startnode[0..stid-1] ... 単語始端ノードリスト */
      /* wchmm.startnode[0..stid-1] ... list of word start node (shared) */
      for (stid = wchmm->startnum - 1; stid >= 0; stid--) {
	next_node = wchmm->startnode[stid];
#ifdef USE_NGRAM
	/* connection to the head silence word is not allowed */
	if (wchmm->wordbegin[wchmm->winfo->head_silwid] == next_node) continue;
#endif

	/*****************************************/
	/* 4.2.1. 単語間言語制約を適用           */
	/*        apply cross-word LM constraint */
	/*****************************************/
	
#ifdef USE_NGRAM
	/* N-gram確率を計算 */
	/* compute N-gram probability */
#ifdef UNIGRAM_FACTORING
	/* 1-gram factoring における単語間言語確率キャッシュの効率化:
	   1-gram factoring は単語履歴に依存しないので，
	   ここで参照する factoring 値の多くは
	   wchmm->state[node].fscore に既に格納され, 探索中も不変である．
	   よって計算が必要な単語(どの単語ともノードを共有しない単語)
	   についてのみ iwparray[] で計算・キャッシュする． */
	/* Efficient cross-word LM cache:
	   As 1-gram factoring values are independent of word context,
	   they remain unchanged while search.  So, in cross-word LM
	   computation, beginning-of-word states which share nodes with
	   others and has factoring value in wchmm does not need cache.
	   So only the unshared beginning-of-word states are computed and
	   cached here in iwparray[].
	 */
	/* wchmm,start2isolate[0..stid-1] ... ノードを共有しない単語は
	   その通しID, 共有する(キャッシュの必要のない)単語は -1 */
	/* wchmm->start2isolate[0..stid-1] ... isolate ID for
	   beginning-of-word state.  value: -1 for states that has
	   1-gram factoring value (share nodes with some other words),
	   and ID for unshared words
	 */
	isoid = wchmm->start2isolate[stid];
	/* 計算が必要でない単語先頭ノードはパスをまとめて後に計算するので
	   ここではスキップ */
	/* the shared nodes will be computed afterward, so just skip them
	   here */
	if (isoid == -1) continue;
	tmpprob = iwparray[isoid];
#else
	tmpprob = iwparray[stid];
#endif
#endif

#ifdef USE_NGRAM
	/* 遷移先の単語が先頭単語なら遷移させない．
	   これは wchmm.c で該当単語に stid を割り振らないことで対応  */
	/* do not allow transition if the destination word is
	   beginning-of-sentence word.  This limitation is realized by
	   not assigning 'stid' for the word in wchmm.c.
	*/
#endif
	
#ifdef CATEGORY_TREE
	/* 文法の場合, 制約は決定的: カテゴリ対制約上許されない場合は遷移させない */
	/* In grammar mode, LM constraint is deterministic:
	   do not allow transition if the category connection is not allowed
	   (with category tree, constraint can be determined on top node) */
	if (dfa_cp(dfa, wchmm->winfo->wton[sword], wchmm->winfo->wton[wchmm->start2wid[stid]]) == FALSE) continue;
#endif

	/*******************************************************************/
	/* 4.2.2. 遷移先の単語先頭へのスコア計算(遷移確率＋言語スコア)     */
	/*        compute score of destination node (transition prob + LM) */
	/*******************************************************************/
	tmpsum = tk->score;
#ifdef USE_NGRAM
	/* 言語スコアを追加 */
	/* add LM score */
	ngram_score_cache = tmpprob * lm_weight + lm_penalty;
	tmpsum += ngram_score_cache;
	if (wchmm->winfo->is_transparent[sword] && wchmm->winfo->is_transparent[tk->last_cword]) {
	  
	  tmpsum += lm_penalty_trans;
	}
#else  /* USE_DFA */
	/* grammar: 単語挿入ペナルティを追加 */
	/* grammar: add insertion penalty */
	tmpsum += penalty1;

	/* grammar: deterministic factoring (in case category-tree not enabled) */
#ifdef CATEGORY_TREE
	/* no need to factoring */
#else
	if (!can_succeed(wchmm, sword, next_node)) {
	  tmpsum = LOG_ZERO;
	}
#endif /* CATEGORY_TREE */
#endif /* USE_NGRAM */
	
	/*********************************************************************/
	/* 4.2.3. 遷移先ノードへトークン伝搬(単語履歴情報は更新)             */
	/*        pass token to destination node (updating word-context info */
	/*********************************************************************/
	for(ac=wchmm->state[next_node].ac; ac; ac=ac->next) {
	  next_node2 = ac->arc;
	  
	  if ((tknextid = node_exist_token(tn, next_node2, tre->wid)) != TOKENID_UNDEFINED) {
	    /* 遷移先ノードには既に他ノードから伝搬済み: スコアが高いほうを残す */
	    /* the destination node already has a token: compare score */
	    tknext = &(tlist[tn][tknextid]);
	    if (tknext->score < tmpsum) {
	      /* その遷移先ノードが持つトークンの内容を上書きする(新規トークンは作らない) */
	      /* overwrite the content of existing destination token: not create a new token */
#ifdef USE_NGRAM
	      tknext->last_lscore = ngram_score_cache;
	      if (wchmm->winfo->is_transparent[sword]) {
		tknext->last_cword = tk->last_cword;
	      } else {
		tknext->last_cword = sword;
	      }
#endif
	      tknext->score = tmpsum + ac->a;
	      /* 遷移先に，先ほど保存したswordのトレリス単語へのポインタを
		 「直前単語履歴情報」として伝搬 */
	      /* pass destination the pointer to the saved trellis atom
		 corresponds to 'sword' as "previous word-context info". */
	      tknext->last_tre = tre;
	    }
	  } else {
	    /* 遷移先ノードは未伝搬: 新規トークンを作って割り付ける */
	    /* token unassigned: create new token and assign */
	    if (tmpsum > LOG_ZERO) { /* valid token */
	      tknextid = create_token();
	      tknext = &(tlist[tn][tknextid]);
	      tk = &(tlist[tn][tindex[tn][j]]);
#ifdef USE_NGRAM
	      tknext->last_lscore = ngram_score_cache;
	      if (wchmm->winfo->is_transparent[sword]) {
		tknext->last_cword = tk->last_cword;
	      } else {
		tknext->last_cword = sword;
	      }
#endif
	      tknext->score = tmpsum + ac->a;
	      tknext->last_tre = tre;
	      node_assign_token(next_node2, tknextid);
	    }
	  }
	}
      }
    }
    
  } /* end of main viterbi loop */
#ifdef UNIGRAM_FACTORING
  /***********************************************************/
  /* 5.  単語終端からfactoring付き単語先頭への遷移 ***********/
  /*    transition from wordend to shared (factorized) nodes */
  /***********************************************************/
  if (wordend_best_score > LOG_ZERO) {
    node = wordend_best_node;
    sword = wchmm->stend[node];
    for (stid = wchmm->startnum - 1; stid >= 0; stid--) {
      next_node = wchmm->startnode[stid];
      /* skip nodes already calculated in the above main loop */
      if (wchmm->start2isolate[stid] != -1) continue; 
      tmpprob = wchmm->state[next_node].fscore;
      ngram_score_cache = tmpprob * lm_weight + lm_penalty;
      tmpsum = wordend_best_score + ngram_score_cache;
      if (wchmm->winfo->is_transparent[sword] && wchmm->winfo->is_transparent[wordend_best_last_cword]) {
	tmpsum += lm_penalty_trans;
      }
      for(ac=wchmm->state[next_node].ac; ac; ac=ac->next) {
	next_node2 = ac->arc;
	if ((tknextid = node_exist_token(tn, next_node2, sword)) != TOKENID_UNDEFINED) {
	  tknext = &(tlist[tn][tknextid]);
	  if (tknext->score < tmpsum) {
	    tknext->last_lscore = ngram_score_cache;
	    if (wchmm->winfo->is_transparent[sword]) {
	      tknext->last_cword = wordend_best_last_cword;
	    } else {
	      tknext->last_cword = sword;
	    }
	    tknext->score = tmpsum + ac->a;
	    tknext->last_tre = wordend_best_tre;
	  }
	} else {
	  if (tmpsum > LOG_ZERO) { /* valid token */
	    tknextid = create_token();
	    tknext = &(tlist[tn][tknextid]);
	    tknext->last_lscore = ngram_score_cache;
	    if (wchmm->winfo->is_transparent[sword]) {
	      tknext->last_cword = wordend_best_last_cword;
	    } else {
	      tknext->last_cword = sword;
	    }
	    tknext->score = tmpsum + ac->a;
	    tknext->last_tre = wordend_best_tre;
	    node_assign_token(next_node2, tknextid);
	  }
	}
      }
    }
  }
#endif /* UNIGRAM_FACTORING */

  /***************************************/
  /* 6. 状態の出力確率計算               */
  /*    compute state output probability */
  /***************************************/

  /* 次段の有効ノードについて出力確率を計算してスコアに加える */
  /* compute outprob for new valid (token assigned) nodes and add to score */
  /* 今扱っているのが入力の最終フレームの場合出力確率は計算しない */
  /* don't calculate the last frame (transition only) */
  if (! final) {
    for (j=0;j<tnum[tn];j++) {
      tk = &(tlist[tn][tindex[tn][j]]);
      if (wchmm->state[tk->node].out.state != NULL) {
	tk->score += outprob_style(wchmm, tk->node, tk->last_tre->wid, t, param);
      }
    }
  }

  /*******************************************************/
  /* 7. スコアでトークンをソートしビーム幅分の上位を決定 */
  /*    sort tokens by score up to beam width            */
  /*******************************************************/
  /* tlist[tl]を次段のためにリセット */
  clear_tlist(tl);
    
  /* ヒープソートを用いてこの段のノード集合から上位(bwidth)個を得ておく */
  /* (上位内の順列は必要ない) */
  sort_token_no_order(trellis_beam_width, &n_start, &n_end);

  /***************/
  /* 8. 終了処理 */
  /*    finalize */
  /***************/
    
  if (progout_flag) {
    /* 漸次出力: 現フレームのベストパスを一定時間おきに上書き出力 */
    /* progressive result output: output current best path in certain time interval */
    if ((t % progout_interval_frame) == 0) {
      bt_current_max(backtrellis, t-1, wchmm->winfo);
    }
  }
  /* j_printf("%d: %d\n",t,tnum[tn]); */
  /* for debug: output current max word */
  if (debug2_flag) {
    bt_current_max_word(backtrellis, t-1, wchmm->winfo);
  }

#ifdef SP_BREAK_CURRENT_FRAME
  /* ショートポーズセグメンテーション: 直前フレームでセグメントが切れたかどうかチェック */
  if (detect_end_of_segment(backtrellis, t-1, wchmm->winfo)) {
    /* セグメント終了検知: 第１パスここで中断 */
    return FALSE;		/* segment: [sparea_start..t-2] */
  }
#endif

  /* ビーム内ノード数が 0 になってしまったら，強制終了 */
  if (tnum[tn] == 0) {
    j_printerr("Error: %dth frame: no nodes left in beam! model mismatch or wrong input?\n", t);
    return(FALSE);
  }

  return(TRUE);
    
}

/*************************************************/
/* frame synchronous beam search --- last frame  */
/* フレーム同期ビーム探索の実行 --- 最終フレーム */
/*************************************************/
void
get_back_trellis_end(
     HTK_Param *param,		/* parameters */
     WCHMM_INFO *wchmm,		/* lexicon tree */
     BACKTRELLIS *backtrellis	/* backward trellis (return value) */
     )
{
  int t, node, j;
  TOKEN2 *tk;

  /* 最後にビーム内に残った単語終端トークンを処理する */
  /* 単語末ノードへの遷移のみ計算 */
  /* process the last wordend tokens */
  /* only arcs to word-end node is calculated */
  get_back_trellis_proceed(param->samplenum, param, wchmm, backtrellis, TRUE);

#ifdef SP_BREAK_CURRENT_FRAME
  /*if (detect_end_of_segment(backtrellis, param->samplenum-1, winfo)) {
    return;
    }*/
#endif
}

/*************************/
/* 探索終了 --- 終了処理 */
/* end of search         */
/*************************/
LOGPROB
finalize_1st_pass(BACKTRELLIS *backtrellis, WORD_INFO *winfo, int len)
{
  LOGPROB lastscore;
  
  /* 単語トレリス(backtrellis) を整理: トレリス単語の再配置とソート */
  /* re-arrange backtrellis: index them by frame, and sort by word ID */
  backtrellis->framelen = len;
  bt_relocate_rw(backtrellis);
  bt_sort_rw(backtrellis);
  
  /* 結果を表示する (best 仮説)*/
  /* output 1st pass result (best hypothesis) */
  if (verbose_flag && (!progout_flag)) j_printerr("\n");
  lastscore = print_1pass_result(backtrellis, len, winfo);

#ifdef USE_NGRAM
  /* free succesor cache */
  /* 次の認識でwchmm->winfoともに無変更の場合 free する必要なし */
  /* no need to free if wchmm and wchmm are not changed in the next recognition */
  /* max_successor_cache_free(); */
#endif
  /* free nodes */
  free_nodes();

  /* return the best score */
  return(lastscore);
}

#ifdef SP_BREAK_CURRENT_FRAME
/*******************************************************************/
/* 第１パスセグメント終了処理 (ショートポーズセグメンテーション用) */
/* end of 1st pass for a segment (for short pause segmentation)    */
/*******************************************************************/
void
finalize_segment(BACKTRELLIS *backtrellis, HTK_Param *param, int len)
{
  int t, v;

  /* トレリス始終端における最尤単語を第2パスの始終端単語として格納 */
  /* fix initial/last word hypothesis of the next 2nd pass to the best
     word hypothesis at the first/last frame in backtrellis*/
  set_terminal_words(backtrellis);

  /* パラメータを, 今第１パスが終了したセグメント区間と残りの区間に分割する．
     ただし接続部のsp区間部分(sparea_start..len-1)は「のりしろ」として両方に
     コピーする */
  /* Divide input parameter into two: the last segment and the rest.
     The short-pause area (sparea_start..len-1) is considered as "tab",
     copied in both parameters
   */
  /* param[sparea_start..framelen] -> rest_param
     param[0..len-1] -> param
     [sparea_start...len-1] overlapped
  */
  if (len != param->samplenum) {
    /* copy rest parameters for next process */
#ifdef SP_BREAK_DEBUG
    printf("[%d..%d]->[%d..%d] %d\n", sparea_start, param->samplenum-1,
	   0, param->samplenum-sparea_start-1, len);
#endif
    rest_param = new_param();
    memcpy(&(rest_param->header), &(param->header), sizeof(HTK_Param_Header));
    rest_param->samplenum = param->samplenum - sparea_start;
    rest_param->header.samplenum = rest_param->samplenum;
    rest_param->veclen = param->veclen;
    rest_param->parvec = (VECT **)mymalloc(sizeof(VECT *) * rest_param->samplenum);
    /* copy 1: overlap area (copy) */
    for(t=sparea_start;t<len;t++) {
      rest_param->parvec[t-sparea_start] = (VECT *)mymalloc(sizeof(VECT) * rest_param->veclen);
      memcpy(rest_param->parvec[t-sparea_start], param->parvec[t], sizeof(VECT) * rest_param->veclen);
    }
    /* copy 2: rest area (move) */
    for(t=len;t<param->samplenum;t++) {
      rest_param->parvec[t-sparea_start] = param->parvec[t];
    }

    /* shrink original param to [0..len-1] */
    /* just shrink the parvec */
    param->samplenum = len;
    param->parvec = (VECT **)myrealloc(param->parvec, sizeof(VECT *) * param->samplenum);
    sp_break_last_nword_allow_override = TRUE;
    
  } else {
    
    /* last segment is on end of input: no rest parameter */
    rest_param = NULL;
    /* reset last_word info */
    sp_break_last_word = WORD_INVALID;
    sp_break_last_nword = WORD_INVALID;
    sp_break_last_nword_allow_override = FALSE;
  }
}
#endif /* SP_BREAK_CURRENT_FRAME */
  
/********************************************************************/
/* 第１パスを実行するメイン関数                                     */
/* 入力をパイプライン処理する場合は realtime_1stpass.c を参照のこと */
/* main function to execute 1st pass                                */
/* the pipeline processing is not here: see realtime_1stpass.c      */
/********************************************************************/
void
get_back_trellis(
     HTK_Param *param,		/* whole parameter */
     WCHMM_INFO *wchmm,		/* lexicon tree */
     BACKTRELLIS *backtrellis,	/* backward trellis to hold result */
     LOGPROB *backmax		/* backmax (retuen value) */
     )
{
  int t;

  /* 初期化及び t=0 を計算 */
  /* initialize and compute frame = 0 */
  get_back_trellis_init(param, wchmm, backtrellis);

  /* メインループ */
  /* main loop */
  for (t = 0; t < param->samplenum; t++) {
    if (get_back_trellis_proceed(t,param, wchmm, backtrellis, FALSE) == FALSE
	|| catch_intr_flag
	|| (module_mode && module_wants_terminate())) {
      /* 探索中断: 処理された入力は 0 から t-2 まで */
      /* search terminated: processed input = [0..t-2] */
      /* この時点で第1パスを終了する */
      /* end the 1st pass at this point */
      *backmax = finalize_1st_pass(backtrellis, wchmm->winfo, t-1);
#ifdef SP_BREAK_CURRENT_FRAME
      /* ショートポーズセグメンテーションの場合,
	 入力パラメータ分割などの最終処理も行なう */
      /* When short-pause segmentation enabled */
      finalize_segment(backtrellis, param, t-1);
#endif
      /* terminate 1st pass here */
      return;
    }
  }
  /* 最終フレームを計算 */
  /* compute the last frame of input */
  get_back_trellis_end(param, wchmm, backtrellis);
  /* 第1パス終了処理 */
  /* end of 1st pass */
  *backmax = finalize_1st_pass(backtrellis, wchmm->winfo, param->samplenum);
#ifdef SP_BREAK_CURRENT_FRAME
  /* ショートポーズセグメンテーションの場合,
     入力パラメータ分割などの最終処理も行なう */
  /* When short-pause segmentation enabled */
  finalize_segment(backtrellis, param, param->samplenum);
#endif
}

/* end of file */
