/* Copyright (c) 1991-2002 Doshita Lab. Speech Group, Kyoto University */
/* Copyright (c) 2000-2002 Speech and Acoustics Processing Lab., NAIST */
/*   All rights reserved   */

/* factoring_sub.c --- functions to build successor tree and compute factoring value */

/* $Id: factoring_sub.c,v 1.8 2002/09/11 22:02:33 ri Exp $ */

/* for Julius (N-gram) : 2-gram factoring, 1-gram factoring implemented
   for Julian (grammar): deterministic factoring implemented
                         (when CATEGORY_TREE not defined)
			 
   default: 1-gram factoring for Julius, category-tree lexicon (deterministic
   factoring not used) for Julian
*/

#include <julius.h>

#ifndef CATEGORY_TREE		/* category tree does not need factoring */

/*----------------------------------------------------------------------*/
/* successor list:

   木構造化辞書の各ノードに割り付けられる，そのノード共有単語のリスト．
   枝分かれした最初のノードのみが持つ．
   Lists of words sharing the node, assigned to each lexicon tree node.
   Only the nodes just after the branch have this.
   

          2-o-o - o-o-o - o-o-o          word "A" 
         /
    1-o-o
	 \       4-o-o                   word "B"
	  \     /   
	   3-o-o - 5-o-o - 7-o-o         word "C"
	        \        \ 
		 \        8-o-o          word "D"
		  6-o-o                  word "E"
		   

    node  | successor list (wchmm->sclist[wchmm->state[node].scid])
    =======================
      1   | A B C D E
      2   | A
      3   |   B C D E
      4   |   B
      5   |     C D
      6   |         E
      7   |     C
      8   |       D

   注目: 上記の単語 "A" の場合,言語スコアは単語終端ではなくノード 2 の位置で
   既に確定する．
   Notice: language score for word "A" is determined in node 2 (not word end).

*/

/* 第１パスにおけるfactoring の実行: 実際には beam.c で行なわれる

   2-gram factoring: 
   「次ノードに successor list が存在すれば, その successor list の単語の
     2-gram の最大値を求めて, 伝搬してきているfactoring 値を更新する」
     →successor list 1つの単語確定ノードでは自動的に正しい2-gramが割り当たる

   1-gram factoring:
   「次ノードに successor list が存在すれば, その successor list の単語の
     1-gram の最大値を求めて，伝搬してきているfactoring 値を更新する」
     →単語確定ノードでは2-gram を計算

     高速化と効率化: 1-gram の最大値はコンテキスト非依存: 起動時に全部計算して
     wchmm->state[node].fscore に格納しておき，その successor list は free する

   DFA文法使用時の決定的 factoring (CATEGORY_TREE が undefined の場合):
   「次ノードに successor list が存在すれば, その successor list 内の
     単語のカテゴリを調べ, 一つでも直前単語とカテゴリ対制約上接続可能な
     単語があれば，遷移を許す．一つもなければ遷移させない」
*/

/* factoring execution in Viterbi: mainly operated in beam.c

   2-gram factoring:
   "If successor list exist in the next node, compute the maximum 2-gram
    likelihood of the successor list and update the factoring value."
    -> precise 2-gram will be automatically assigned at the node with single
       successor word.

   1-gram factoring:
   "If successor list exist in the next node, get the maximum 1-gram
    likelihood of the successor list and update the factoring value.
    If the next node has only one successor word, update the factoring value
    with true 2-gram."
    For efficiency, 1-gram factoring value (independent of the hypothesis) for
    each successor list is computed before search.  They are stored in
    wchmm->state[node].fscore.

   DFA deterministic factoring (in case Julian and CATEGORY_TREE undefined):
   "If successor list exist in the next node, allow transition only if there
   is any word that can connect to the previous word hypothesis."
   
*/
		  
/*----------------------------------------------------------------------*/
/* successor list を構築する．
   単語の ID は N-gram での番号(あるいはカテゴリ番号), 辞書の単語IDでない
   ことに注意．単語はIDで昇順に保存する */
/* build whole successor lists.
   IDs in the lists are those of N-gram entry, not word ID in dictionary.
   Words in lists should be ordered by ID. */

/* add word 'w' to successor list at 'node' (w = dictionary ID) */
static void
add_successor(WCHMM_INFO *wchmm, S_CELL **slist, int node, WORD_ID w)
{
  S_CELL *sctmp, *sc;

  /* malloc new */
  sctmp=(S_CELL *) mymalloc(sizeof(S_CELL));
  if (sctmp == NULL) {
    j_error("malloc fault at add_succesor(%d,%d)\n",node,w);
  }
  /* assign word ID */
  sctmp->word = wchmm->winfo->wton[w];
  /* add to list (keeping order) */
  sc = slist[node];
  if (sc == NULL || sctmp->word < sc->word) {
    sctmp->next = sc;
    slist[node] = sctmp;
  } else {
    for(;sc;sc=sc->next) {
      if (sc->next == NULL || sctmp->word < (sc->next)->word) {
	if (sctmp->word == sc->word) break; /* avoid duplication */
	sctmp->next = sc->next;
	sc->next = sctmp;
	break;
      }
    }
  }
}

/* check if successor lists in 'node1' and 'node2' are the same */
static boolean
match_successor(S_CELL **slist, int node1, int node2)
{
  S_CELL *sc1,*sc2;

  /* assume successor is sorted by ID */
  sc1 = slist[node1];
  sc2 = slist[node2];
  for (;;) {
    if (sc1 == NULL || sc2 == NULL) {
      if (sc1 == NULL && sc2 == NULL) {
	return TRUE;
      } else {
	return FALSE;
      }
    } else if (sc1->word != sc2->word) {
      return FALSE;
    }
    sc1 = sc1->next;
    sc2 = sc2->next;
  }
}

/* free successor list at 'node' */
static void
free_successor(S_CELL **slist, int node)
{
  S_CELL *sc;
  S_CELL *sctmp;

  sc = slist[node];
  while (sc != NULL) {
    sctmp = sc;
    sc = sc->next;
    free(sctmp);
  }
  slist[node] = NULL;
}

/* main function to build whole successor list to lexicon tree 'wchmm' */
void
make_successor_list(WCHMM_INFO *wchmm)
{
  S_CELL **sc_list;
  int node;
  WORD_ID w;
  int i;
  int freed_num = 0;
  boolean *freemark;

  VERMES("  make successor lists for factoring...");

  /* 1. initialize */
  sc_list = (S_CELL **)mymalloc(sizeof(S_CELL *) * wchmm->n);
  freemark = (boolean *)mymalloc(sizeof(boolean) * wchmm->n);
  for (node=0;node<wchmm->n;node++) {
    sc_list[node] = NULL;
    freemark[node] = FALSE;
  }

  /* 2. make initial successor list: assign at all possible nodes */
  for (w=0;w<wchmm->winfo->num;w++) {
    /* at each start node of phonemes */
    for (i=0;i<wchmm->winfo->wlen[w];i++) {
      add_successor(wchmm, sc_list, wchmm->offset[w][i], w);
    }
    /* at word end */
    add_successor(wchmm, sc_list, wchmm->wordend[w], w);
  }
  
  /* 3. erase unnecessary successor list */
  /* sucessor list same as the previous node is not needed, so */
  /* parse lexicon tree from every leaf to find the same succesor list */
  for (w=0;w<wchmm->winfo->num;w++) {
    node = wchmm->wordend[w];	/* begin from the word end node */
    i = wchmm->winfo->wlen[w]-1;
    while (i >= 0) {		/* for each phoneme start node */
      if (node == wchmm->offset[w][i]) {
	/*	printf("same:w=%d,phoneloc=%d,node=%d,%d\n", w, i, wchmm->offset[w][i], node);
	{
	  S_CELL *sc;
	  for(sc=wchmm->state[node].sc;sc;sc=sc->next) {
	    printf("%d[%s],", sc->word, ngram->wname[sc->word]);
	  }
	  printf("\n");
	  }*/
	/* word with only 1 state: skip */
	i--;
	continue;
      }
      if (match_successor(sc_list, node, wchmm->offset[w][i])) {
	freemark[node] = TRUE;	/* mark the node */
      }
/* 
 *	 if (freemark[wchmm->offset[w][i]] != FALSE) {
 *	   break;
 *	 }
 */
      node = wchmm->offset[w][i];
      i--;
    }
  }
  /* really free */
  for (node=0;node<wchmm->n;node++) {
    if (freemark[node] == TRUE) {
      freed_num++;
      free_successor(sc_list, node);
    }
  }

  if (debug2_flag) {
    j_printerr("%d freed...", freed_num);
  }

  free(freemark);

  wchmm->sclist = sc_list;

  VERMES("done\n");
}

#ifndef CATEGORY_TREE
/* make index to valid factoring node, and make mapping from node ID
   for factoring cache */
void
make_sc_index(WCHMM_INFO *wchmm)
{
  int n, node;
  S_CELL **new_sclist;

  /* count total valid factoring node and set to wchmm->scnum */
  n = 0;
  for (node=0;node<wchmm->n;node++) {
    if (wchmm->sclist[node] != NULL) n++;
  }
  wchmm->scnum = n;
  /* do garbage collection for the sclist, and */
  /* make mapping from wchmm->state[] --(scid)-->wchmm->sclist */
  n = 0;
  new_sclist = (S_CELL **)mymalloc(sizeof(S_CELL *) * wchmm->scnum);
  for (node=0;node<wchmm->n;node++) {
    if (wchmm->sclist[node] != NULL) {
      wchmm->state[node].scid = n;
      new_sclist[n] = wchmm->sclist[node];
      n++;
    } else {
      wchmm->state[node].scid = -1;
    }
  }
  free(wchmm->sclist);
  wchmm->sclist = new_sclist;

  /* duplicate scid for HMMs with more than one arc from initial state */
  {
    WORD_ID w;
    int k,j;
    HMM_Logical *ltmp;
    int ltmp_state_num;
    int ato;
    LOGPROB prob;
    int node, scid;
    
    for(w=0;w<wchmm->winfo->num;w++) {
      for(k=0;k<wchmm->winfo->wlen[w];k++) {
	node = wchmm->offset[w][k];
	scid = wchmm->state[node].scid;
	if (scid == -1) continue;
	ltmp = wchmm->winfo->wseq[w][k];
	ltmp_state_num = hmm_logical_state_num(ltmp);
	if ((hmm_logical_trans(ltmp))->a[0][ltmp_state_num-1] != LOG_ZERO) {
	  j = k + 1;
	  if (j == wchmm->winfo->wlen[w]) {
	    if (wchmm->state[wchmm->wordend[w]].scid == -1) {
	      printf("word %d: factoring node copied for skip phone\n", w);
	      wchmm->state[wchmm->wordend[w]].scid = scid;
	    }
	  } else {
	    if (wchmm->state[wchmm->offset[w][j]].scid == -1) {
	      printf("word %d: factoring node copied for skip phone\n", w);
	      wchmm->state[wchmm->offset[w][j]].scid = scid;
	    }
	  }
	}
	for(ato=1;ato<ltmp_state_num;ato++) {
	  prob = (hmm_logical_trans(ltmp))->a[0][ato];
	  if (prob != LOG_ZERO) {
	    wchmm->state[node+ato-1].scid = scid;
	  }
	}
      }
    }
  }
  /* move scid and fscore on the head state to the head grammar state */
  {
    int i, node;
    A_CELL *ac;
    for(i=0;i<wchmm->startnum;i++) {
      node = wchmm->startnode[i];
      if (wchmm->state[node].out.state != NULL) {
	j_error("Error: outprob exist in word-head node??\n");
      }
      for(ac=wchmm->state[node].ac;ac;ac=ac->next) {
	if (wchmm->state[ac->arc].scid != -1) {
	  if (wchmm->state[node].scid != -1 && wchmm->state[node].scid != wchmm->state[ac->arc].scid) {
	    j_error("Error: different successor list within word-head phone?\n");
	  }
	  wchmm->state[node].scid = wchmm->state[ac->arc].scid;
	  wchmm->state[ac->arc].scid = -1;
	}
#ifdef USE_NGRAM
#ifdef UNIGRAM_FACTORING
	if (wchmm->state[ac->arc].fscore != LOG_ZERO) {
	  if (wchmm->state[node].fscore != LOG_ZERO && wchmm->state[node].fscore != wchmm->state[ac->arc].fscore) {
	    j_error("Error: different fscore within word-head phone?\n");
	  }
	  wchmm->state[node].fscore = wchmm->state[ac->arc].fscore;
	  wchmm->state[ac->arc].fscore = LOG_ZERO;
	}
#endif /* UNIGRAM_FACTORING */
#endif /* USE_NGRAM */
      }
    }
  }
}

#endif /* CATEGORY_TREE */

/* -------------------------------------------------------------------- */
/* factoring computation */

#ifdef USE_NGRAM

/* word-internal factoring value cache:
   a branch node (that has successor list) will keep the factoring value
   on search, and re-use it if incoming token in the next frame has the
   same word context.
 */
static LOGPROB *probcache;	/* cached value: indexed by scid */
static WORD_ID *lastwcache;	/* cached lastword: indexed by scid*/
/* cross-word factoring value cache:
   when computing cross-word transition, (1)factoring values on all word
   start nodes should be computed for each word end, and thus (2)each start
   node has more than one transition within a frame.  So factoring value
   is better cached by the word end (last word) than by nodes.
 */
static LOGPROB **iw_sc_cache;
static int iw_cache_num;
#ifdef HASH_CACHE_IW
static WORD_ID *iw_lw_cache;
#endif
/* once initialized on startup, the factoring value caches will not be
   cleared */

/* initialize factoring cache (once on startup) */
void
max_successor_cache_init(WCHMM_INFO *wchmm)
{
  int i;

  /* for word-internal */
  probcache = (LOGPROB *) mymalloc(sizeof(LOGPROB) * wchmm->scnum);
  lastwcache = (WORD_ID *) mymalloc(sizeof(WORD_ID) * wchmm->scnum);
  for (i=0;i<wchmm->scnum;i++) {
    lastwcache[i] = WORD_INVALID;
  }
  /* for cross-word */
#ifdef HASH_CACHE_IW
# ifdef CLASS_NGRAM
  iw_cache_num = true_max_num * iw_cache_rate / 100;
# else
  iw_cache_num = wchmm->ngram->max_word_num * iw_cache_rate / 100;
# endif
  if (iw_cache_num < 10) iw_cache_num = 10;
#else
# ifdef CLASS_NGRAM
  iw_cache_num = true_max_num;
# else
  iw_cache_num = wchmm->ngram->max_word_num;
# endif
#endif /* HASH_CACHE_IW */
  iw_sc_cache = (LOGPROB **)mymalloc(sizeof(LOGPROB *) * iw_cache_num);
  for (i=0;i<iw_cache_num;i++) {
    iw_sc_cache[i] = NULL;
  }
#ifdef HASH_CACHE_IW
  iw_lw_cache = (WORD_ID *)mymalloc(sizeof(WORD_ID) * iw_cache_num);
  for (i=0;i<iw_cache_num;i++) {
    iw_lw_cache[i] = WORD_INVALID;
  }
#endif
}

/* free cross-word factoring cache */
static void
max_successor_prob_iw_free()
{
  int i;
  for (i=0;i<iw_cache_num;i++) {
    if (iw_sc_cache[i] != NULL) free(iw_sc_cache[i]);
    iw_sc_cache[i] = NULL;
  }
}

/* free word-internal factoring cache */
void
max_successor_cache_free()
{
  free(probcache);
  free(lastwcache);
  max_successor_prob_iw_free();
  free(iw_sc_cache);
#ifdef HASH_CACHE_IW
  free(iw_lw_cache);
#endif
}

#ifdef UNIGRAM_FACTORING

/* 1-gram factoring は直前単語に影響されないので，
   先頭ノードが(unigramの最大値を与える) factoring ノードであるなら
   そこを単語間 LM キャッシュする必要はない
   よって，factoring ノードでない先頭ノードだけキャッシュする */
/* In 1-gram factoring, the factoring values are constant in branch (not leaf)
   nodes.  So, in cross-word factoring, the word start node that is branch
   (shared, not leaf) nodes are not need to be cached. */
void
make_iwcache_index(WCHMM_INFO *wchmm)
{
  int i, node, num;

  wchmm->start2isolate = (int *)mymalloc(sizeof(int) * wchmm->startnum);
  num = 0;
  for(i=0;i<wchmm->startnum;i++) {
    node = wchmm->startnode[i];
    if (wchmm->state[node].fscore == LOG_ZERO) {	/* not a factoring node (isolated node, has no 1-gram factoring value) */
      wchmm->start2isolate[i] = num;
      num++;
    } else {			/* factoring node (shared) */
      wchmm->start2isolate[i] = -1;
    }
  }
  wchmm->isolatenum = num;
}

/* 1-gram factoring値（固定）を計算して格納する．
   探索前に一度読んでおけばよい．
   計算対象は 1以上のsuccessor word を持つ＝末端でないノード．
   計算後はそこのscは必要なくなるのでフリーする */
/* compute constant 1-gram factoring value for each factoring node.
   should be called once on startup.
   factoring node: has more than one successor word.
   after computation, the successor list will be freed.
 */
void
calc_all_unigram_factoring_values(WCHMM_INFO *wchmm)
{
  S_CELL *sc, *sctmp;
  LOGPROB tmpprob, maxprob;
  int n;

  for (n=0;n<wchmm->n;n++) {
    wchmm->state[n].fscore = LOG_ZERO; /* initial "undefined" value */
    sc = wchmm->sclist[n];
    if (sc != NULL) {		/* leave undefined if no successor list */
      if (sc->next == NULL) {
	/* only one successor = not factoring node (=leaf node) */
      } else {
	/* compute maximum 1-gram probability in successor words */
	/* and set to fscore */
	maxprob = LOG_ZERO;
	for (sctmp = sc; sctmp; sctmp = sctmp->next) {
#ifdef CLASS_NGRAM
	  tmpprob = class_uni_prob(wchmm->ngram, sctmp->word);
#else
	  tmpprob = uni_prob(wchmm->ngram, sctmp->word);
#endif
	  if (maxprob < tmpprob) maxprob = tmpprob;
	}
	wchmm->state[n].fscore = maxprob;
	free_successor(wchmm->sclist, n);
      }
    }
  }
}

#else  /* ~UNIGRAM_FACTORING */

/* compute 2-gram factoring value for the node and return the value */
static LOGPROB
calc_successor_prob(WCHMM_INFO *wchmm, WORD_ID last_nword, int node)
{
  S_CELL *sc;
  LOGPROB tmpprob, maxprob;

  maxprob = LOG_ZERO;
  for (sc = wchmm->sclist[wchmm->state[node].scid]; sc; sc = sc->next) {
#ifdef CLASS_NGRAM
    tmpprob = class_bi_prob_lr(wchmm->ngram, last_nword, sc->word);
#else
    tmpprob = bi_prob_lr(wchmm->ngram, last_nword, sc->word);
#endif
    if (maxprob < tmpprob) maxprob = tmpprob;
  }
  return(maxprob);
}

#endif  /* ~UNIGRAM_FACTORING */

/* ノードに対応するfactoring値を返す(単語内) */
/* For word-internal: return factoring value on node "node" with previous
   word "lastword", consulting cache. */
LOGPROB
max_successor_prob(WCHMM_INFO *wchmm, WORD_ID lastword, int node)
{
  LOGPROB maxprob;
  WORD_ID last_nword;
  int scid;

  if (lastword != WORD_INVALID) { /* return nothing if no previous word */
    last_nword = wchmm->winfo->wton[lastword];
#ifdef UNIGRAM_FACTORING
    if (wchmm->state[node].fscore != LOG_ZERO) {
      /* return 1-gram factoring value already calced */
      return(wchmm->state[node].fscore);
    } else {
      scid = wchmm->state[node].scid;
      /* return precise 2-gram score */
      if (last_nword != lastwcache[scid]) {
	/* calc and cache */
#ifdef CLASS_NGRAM
	maxprob = class_bi_prob_lr(wchmm->ngram, last_nword, (wchmm->sclist[wchmm->state[node].scid])->word);
#else
	maxprob = bi_prob_lr(wchmm->ngram, last_nword, (wchmm->sclist[wchmm->state[node].scid])->word);
#endif
	lastwcache[scid] = last_nword;
	probcache[scid] = maxprob;
	return(maxprob);
      } else {
	/* return cached */
	return (probcache[scid]);
      }
    }
#else  /* UNIGRAM_FACTORING */
    /* 2-gram */
    scid = wchmm->state[node].scid;
    if (last_nword != lastwcache[scid]) {
      maxprob = calc_successor_prob(wchmm, last_nword, node);
      /* store to cache */
      lastwcache[scid] = last_nword;
      probcache[scid] = maxprob;
      return(maxprob);
    } else {
      return (probcache[scid]);
    }
#endif /* UNIGRAM_FACTORING */
  } else {
    return(0.0);
#if 0
    maxprob = LOG_ZERO;
    for (sc=wchmm->sclist[wchmm->state[node].scid];sc;sc=sc->next) {
      tmpprob = uni_prob(wchmm->ngram, sc->word);
      if (maxprob < tmpprob) maxprob = tmpprob;
    }
    return(maxprob);
#endif
  }

}

/* ノードに対応するfactoring値を返す(単語間)
   factoringが必要なすべての単語先頭の計算を行なってから，その配列を返す */
/* For cross-word: return an array of factoring values for all word-start node
   when previous word context is 'lastword', consulting cache.
 */
#ifdef HASH_CACHE_IW
#define hashid(A) A % iw_cache_limit
#endif
LOGPROB *
max_successor_prob_iw(WCHMM_INFO *wchmm, WORD_ID lastword)
{
  int i, j, x, node;
  int last_nword;

  last_nword = wchmm->winfo->wton[lastword];
#ifdef HASH_CACHE_IW
  x = hashid(last_nword);
  if (iw_lw_cache[x] == last_nword) { /* cache hit */
    return(iw_sc_cache[x]);
  }
#else  /* full cache */
  if (iw_sc_cache[last_nword] != NULL) { /* cache hit */
    return(iw_sc_cache[last_nword]);
  }
  x = last_nword;
  /* cache mis-hit, calc probs and cache them as new */
#endif
  /* allocate cache memory */
  if (iw_sc_cache[x] == NULL) {
#ifdef UNIGRAM_FACTORING
    iw_sc_cache[x] = (LOGPROB *)mymalloc(sizeof(LOGPROB)*wchmm->isolatenum);
#else
    iw_sc_cache[x] = (LOGPROB *)mymalloc(sizeof(LOGPROB)*wchmm->startnum);
#endif
    if (iw_sc_cache[x] == NULL) { /* malloc failed */
      /* clear existing cache, and retry */
      max_successor_prob_iw_free();
      j_printf("inter-word LM cache (%dMB) rehashed\n",
	       (iw_cache_num * 
#ifdef UNIGRAM_FACTORING
		wchmm->isolatenum
#else
		wchmm->startnum
#endif
		) / 1000 * sizeof(LOGPROB) / 1000);
#ifdef UNIGRAM_FACTORING
      iw_sc_cache[x] = (LOGPROB *)mymalloc(sizeof(LOGPROB)*wchmm->isolatenum);
#else
      iw_sc_cache[x] = (LOGPROB *)mymalloc(sizeof(LOGPROB)*wchmm->startnum);
#endif
      if (iw_sc_cache[x] == NULL) { /* malloc failed again? */
	j_error("max_successor_prob_iw: cannot malloc\n");
      }
    }
  }

  /* calc prob for all startid */
#ifdef UNIGRAM_FACTORING
  for (j=0;j<wchmm->startnum;j++) {
    i = wchmm->start2isolate[j];
    if (i == -1) continue;
    node = wchmm->startnode[j];
    if (wchmm->state[node].fscore != LOG_ZERO) {
      /* should not be happen!!! below is just for debugging */
      j_error("No!!\n");
    } else {
#ifdef CLASS_NGRAM
      iw_sc_cache[x][i] = class_bi_prob_lr(ngram, last_nword, (wchmm->sclist[wchmm->state[node].scid])->word);
#else
      iw_sc_cache[x][i] = bi_prob_lr(ngram, last_nword, (wchmm->sclist[wchmm->state[node].scid])->word);
#endif
    }
  }
#else  /* ~UNIGRAM_FACTORING */
  for (i=0;i<wchmm->startnum;i++) {
    node = wchmm->startnode[i];
    iw_sc_cache[x][i] = calc_successor_prob(wchmm, last_nword, node);
  }
#endif
#ifdef HASH_CACHE_IW
  iw_lw_cache[x] = last_nword;
#endif

  return(iw_sc_cache[x]);
}


#else /* USE_DFA --- カテゴリ木なら不必要 */

/* 決定的 factoring: ノード node に直前単語履歴 lastword を持つトークンが
   カテゴリ対制約上接続できるかどうかを TRUE/FALSEで返す．*/
/* deterministic factoring: return whether a token with last word ID
   "lastword" can go into the branch node "node" by category-pair constraint.
*/
boolean
can_succeed(WCHMM_INFO *wchmm, WORD_ID lastword, int node)
{
  int lc;
  S_CELL *sc;

  /* return TRUE if at least one subtree word can connect */

  if (lastword == WORD_INVALID) { /* case at beginning-of-word */
    for (sc=wchmm->sclist[wchmm->state[node].scid];sc;sc=sc->next) {
      if (dfa_cp_begin(wchmm->dfa, sc->word) == TRUE) return(TRUE);
    }
    return(FALSE);
  } else {
    lc = wchmm->winfo->wton[lastword];
    for (sc=wchmm->sclist[wchmm->state[node].scid];sc;sc=sc->next) {
      if (dfa_cp(wchmm->dfa, lc, sc->word) == TRUE) return(TRUE);
    }
    return(FALSE);
  }
}

#endif /* USE_DFA */


#endif /* CATEGORY_TREE */








#ifdef CLASS_NGRAM

/*

  VERY EXPERIMENTAL, STILL UNDER CONSTRUCTION:
  PLEASE DO NOT USE FUNCTIONS BELOW.
  
  [Dynamic class N-gram handling]
*/
  
/*
  When apply:
  
   if wton[w] >= ngram->max_word_num; then
     wton[w] - ngram->max_word_num == ID of class N-gram 
   else
     wton[w] == ID of word N-gram

 */

static LOGPROB log_beta_w, log_beta_c; /* LM prob distibution coef. */
static WORD_ID *wtoc;		/* N-gram word ID -> class ID */
static LOGPROB *c_probsum;		/* sum of uniprob assigned by the class */
static LOGPROB *wtoc_prob;	/* interpolate prob for each word */
#ifndef CLASS_UNKNOWN_ONLY
static LOGPROB class_weight_log, class_weight_inv_log;
#endif

#define EP 2.302585093
/* try to use class N-gram for those whose N-gram entry does not
 exist in normal word N-gram */
void
make_class_ref(NGRAM_INFO *cinfo, WORD_INFO *winfo)
{
  WORD_ID w, c;
  char *cname, *buf, *p, *q;
  int i;
  int hitcount;


  true_max_num = ngram->max_word_num;
    
  buf = (char *)mymalloc(4096);

  wtoc = (WORD_ID *)mybmalloc(sizeof(WORD_ID) * ngram->max_word_num);
  
  /* regist class of each words for use as context */
  hitcount = 0;
  for(w=0;w<winfo->num;w++) {
    /* (1)同じ単語 */
    c = ngram_lookup_word(cinfo, winfo->wname[w]);
    if (c == WORD_INVALID) {
      /* (2)表記+単語ID (読み抜き) */
      if (strlen(winfo->wname[w]) >= 4096) {
	j_error("ERROR: make_class_ref: wname len exceed 4096!!\n");
      }
      strcpy(buf,winfo->wname[w]);
      p = strchr(buf, '+');
      q = strrchr(buf, '+');
      if (p && q && p != q) {
	memmove(p+1,q+1,buf+strlen(winfo->wname[w])-q);
	c = ngram_lookup_word(cinfo, buf);
	}
    }
    if (c == WORD_INVALID) {
      /* (3) 単語ID */
      strcpy(buf,winfo->wname[w]);
      q = strrchr(buf, '+');
      if (q) {
	c = ngram_lookup_word(cinfo, q+1);
      }
    }
    if (c == WORD_INVALID) {
      /* (4) 表記 */
      p = strchr(buf, '+');
      if (p) {
	*p = '\0';
	c = ngram_lookup_word(cinfo, buf);
      }
    }
    if (c == WORD_INVALID) {
      /* ギブアップ */
      /*j_printerr("Warning: class N-gram entry for \"%s\" not found, use UNK class\n", winfo->wname[w]);*/
      c = cinfo->unk_id;
    } else {
      hitcount++;
    }
#if 0
    printf("%d (ngram=%d): %s [%s] belongs to [%s]\n", w, winfo->wton[w], ngram->wname[winfo->wton[w]], winfo->wname[w], cinfo->wname[c]);
#endif
    wtoc[winfo->wton[w]] = c;
  }
  printf("%d / %d hit\n", hitcount, winfo->num);

  /* set wtoc_prob */
  c_probsum = (LOGPROB *)mybmalloc(sizeof(LOGPROB)*cinfo->max_word_num);
  for(i=0;i<cinfo->max_word_num;i++) c_probsum[i] = 0.0;
  for(w=0;w<winfo->num;w++) {
    c = wtoc[winfo->wton[w]];
    if (c >= cinfo->max_word_num) {
      j_error("ERROR!!!\n");
    }
    c_probsum[c] += exp(uni_prob(ngram, winfo->wton[w])*EP);
  }
  for(i=0;i<cinfo->max_word_num;i++) {
    if (c_probsum[i] > 0.0) c_probsum[i] = log10(c_probsum[i]);
  }
  wtoc_prob = (LOGPROB *)mybmalloc(sizeof(LOGPROB) * ngram->max_word_num);
  for(w=0;w<winfo->num;w++) {
    c = wtoc[winfo->wton[w]];
    wtoc_prob[winfo->wton[w]] = uni_prob(ngram, winfo->wton[w]) - c_probsum[c];
  }
#if 0
  for(w=0;w<winfo->num;w++) {
    printf("%f %s-%s(%s)\n", wtoc_prob[winfo->wton[w]],cinfo->wname[wtoc[winfo->wton[w]]],ngram->wname[winfo->wton[w]], winfo->wname[w]);
  }
#endif

  free(buf);

#ifndef CLASS_UNKNOWN_ONLY
  if (class_weight > 0.0) {
    class_weight_log = log10(class_weight);
  } else {
    class_weight_log = LOG_ZERO;
  }
  if (1.0 - class_weight > 0.0) {
    class_weight_inv_log = log10(1.0 - class_weight);
  } else {
    class_weight_inv_log = LOG_ZERO;
  }
#endif
}

LOGPROB
class_uni_prob(NGRAM_INFO *ndata, WORD_ID w)
{
#ifdef CLASS_UNKNOWN_ONLY
  LOGPROB f;
#ifdef CLASS_UNKNOWN_ONLY_SCALED
  LOGPROB scale;
  scale = - log(1.0 - exp(uni_prob(ndata,ndata->unk_id)*EP) + exp(uni_prob(class_ngram, wtoc[ndata->unk_id])*EP))/EP ;
  if (w == ndata->unk_id) {
    f = uni_prob(class_ngram, wtoc[w]) + wtoc_prob[w] + scale;
  } else {
    f = uni_prob(ndata, w) + scale;
  }
  return(f);
#else
  if (w == ndata->unk_id) {
    f = uni_prob(class_ngram, wtoc[w]) + wtoc_prob[w];
  } else {
    f = uni_prob(ndata, w);
  }
  return(f);
#endif
#else
  /* linear interpolation for all (\beta = class_wright (option "-clw") */
  LOGPROB fw, fc;
  fc = uni_prob(class_ngram, wtoc[w]) + class_weight_log + wtoc_prob[w];
  fw = uni_prob(ndata, w) + class_weight_inv_log;
  return(log(exp(fc*EP)+exp(fw*EP))/EP);
#endif
}

LOGPROB
class_bi_prob_lr(NGRAM_INFO *ndata, WORD_ID w1, WORD_ID w2)
{
#ifdef CLASS_UNKNOWN_ONLY
  LOGPROB f;
#ifdef CLASS_UNKNOWN_ONLY_SCALED
  LOGPROB scale;
  scale = - log(1.0 - exp(bi_prob_lr(ndata, w1, ndata->unk_id)*EP) + exp(bi_prob_lr(class_ngram, wtoc[w1], wtoc[ndata->unk_id])*EP))/EP ;
  if (w2 == ndata->unk_id) {
    f = bi_prob_lr(class_ngram, wtoc[w1], wtoc[w2]) + wtoc_prob[w2] + scale;
  } else {
    f = bi_prob_lr(ndata, w1, w2) + scale;
  }
  return(f);
#else  
  if (w2 == ndata->unk_id) {
    f = bi_prob_lr(class_ngram, wtoc[w1], wtoc[w2]) + wtoc_prob[w2];
#if 0
    printf("[%s %s] - ", ngram->wname[w1], ngram->wname[w2]);
    printf("[%s %s %f]]\n",class_ngram->wname[wtoc[w1]], class_ngram->wname[wtoc[w2]], f);
#endif
  } else {
    f = bi_prob_lr(ndata, w1, w2);
  }
  return(f);
#endif
#else
  LOGPROB fw, fc;
  fc = bi_prob_lr(class_ngram, wtoc[w1], wtoc[w2]) + class_weight_log + wtoc_prob[w2];
  fw = bi_prob_lr(ndata, w1, w2) + class_weight_inv_log;
  return(log(exp(fc*EP)+exp(fw*EP))/EP);
#endif
}
LOGPROB
class_bi_prob_rl(NGRAM_INFO *ndata, WORD_ID w1, WORD_ID w2)
{
#ifdef CLASS_UNKNOWN_ONLY
  LOGPROB f;
#ifdef CLASS_UNKNOWN_ONLY_SCALED
  LOGPROB scale;
  scale = - log(1.0 - exp(bi_prob_rl(ndata, ndata->unk_id, w2)*EP) + exp(bi_prob_rl(class_ngram, wtoc[ndata->unk_id], wtoc[w2])*EP))/EP ;
  if (w1 == ndata->unk_id) {
    f = bi_prob_rl(class_ngram, wtoc[w1], wtoc[w2]) + wtoc_prob[w1] + scale;
  } else {
    f = bi_prob_rl(ndata, w1, w2) + scale;
  }
  return(f);
#else
  if (w1 == ndata->unk_id) {
    f = bi_prob_rl(class_ngram, wtoc[w1], wtoc[w2]) + wtoc_prob[w1];
  } else {
    f = bi_prob_rl(ndata, w1, w2);
  }
  return(f);
#endif
#else
  LOGPROB fw, fc;
  fc = bi_prob_rl(class_ngram, wtoc[w1], wtoc[w2]) + class_weight_log + wtoc_prob[w1];
  fw = bi_prob_rl(ndata, w1, w2) + class_weight_inv_log;
  return(log(exp(fc*EP)+exp(fw*EP))/EP);
#endif
}

LOGPROB
class_tri_prob_rl(NGRAM_INFO *ndata, WORD_ID w1, WORD_ID w2, WORD_ID w3)
{
#ifdef CLASS_UNKNOWN_ONLY
  LOGPROB f;
#ifdef CLASS_UNKNOWN_ONLY_SCALED
  LOGPROB scale;
  scale = - log(1.0 - exp(tri_prob_rl(ndata, ndata->unk_id, w2, w3)*EP) + exp(tri_prob_rl(class_ngram, wtoc[ndata->unk_id], wtoc[w2], wtoc[w3])*EP))/EP ;
  if (w1 == ndata->unk_id) {
    f = tri_prob_rl(class_ngram, wtoc[w1], wtoc[w2], wtoc[w3]) + wtoc_prob[w1] + scale;
  } else {
    f = tri_prob_rl(ndata, w1, w2, w3) + scale;
  }
  return(f);
#else
  if (w1 == ndata->unk_id) {
    f = tri_prob_rl(class_ngram, wtoc[w1], wtoc[w2], wtoc[w3]) + wtoc_prob[w1];
  } else {
    f = tri_prob_rl(ndata, w1, w2, w3);
  }
  return(f);
#endif
#else
  LOGPROB fw, fc;
  fc = tri_prob_rl(class_ngram, wtoc[w1], wtoc[w2], wtoc[w3]) + class_weight_log + wtoc_prob[w1];
  fw = tri_prob_rl(ndata, w1, w2, w3) + class_weight_inv_log;
  return(log(exp(fc*EP)+exp(fw*EP))/EP);
#endif
}
    
#endif
