/* Copyright (c) 1991-2002 Doshita Lab. Speech Group, Kyoto University */
/* Copyright (c) 2000-2002 Speech and Acoustics Processing Lab., NAIST */
/*   All rights reserved   */

/* wchmm.c --- build lexicon tree (word-conjunction HMM) */

/* $Id: wchmm.c,v 1.9 2002/09/11 22:02:33 ri Exp $ */

/* wchmm = word conjunction HMM = lexicon tree */

#include <julius.h>


static int bogus_arc = 0;	/* reduced arc (for debug) */
static int homophone_word_num;	/* number of homophone words in vocabulary */
static int dupcount = 0;	/* number of same phone words */

/* #define WCHMM_SIZE_CHECK */


/**************************************************************/
/*********** wchmm initialize *********************************/
/**************************************************************/

/* wchmm を malloc する */
WCHMM_INFO *
wchmm_new()
{
  WCHMM_INFO *w;
  w = (WCHMM_INFO *)mymalloc(sizeof(WCHMM_INFO));
#ifdef USE_NGRAM
  w->ngram = NULL;
#endif
#ifdef USE_DFA
  w->dfa = NULL;
#endif
  w->winfo = NULL;
  w->malloc_root = NULL;
  return w;
}

/* wchmm を初期化する */
/* initialize wchmm data */
static void
wchmm_init(WCHMM_INFO *wchmm)
{
  wchmm->maxwcn = MAXWCNSTEP;
  wchmm->state = (WCHMM_STATE *)mymalloc(sizeof(WCHMM_STATE)*wchmm->maxwcn);
  wchmm->stend = (WORD_ID *)mymalloc(sizeof(WORD_ID)*wchmm->maxwcn);
  wchmm->offset = (int **)mymalloc(sizeof(int *)*wchmm->winfo->num);
  wchmm->wordbegin = (int *)mymalloc(sizeof(int)*wchmm->winfo->num);
  wchmm->wordend = (int *)mymalloc(sizeof(int)*wchmm->winfo->num);
  wchmm->n = 0;

  wchmm->maxstartnum = STARTNODE_STEP;
  wchmm->startnode = (int *)mymalloc(sizeof(int)*STARTNODE_STEP);
  wchmm->startnum = 0;
#ifdef CATEGORY_TREE
  wchmm->start2wid = (WORD_ID *)mymalloc(sizeof(WORD_ID)*STARTNODE_STEP);
#endif

#ifdef UNIGRAM_FACTORING
  wchmm->start2isolate = NULL;
  wchmm->isolatenum = 0;
#endif
#ifndef CATEGORY_TREE
  wchmm->sclist = NULL;
  wchmm->scnum = 0;
#endif
}

/* 木の大きさに応じて領域を MAXWCNSTEP 分伸長 */
/* expand wchmm data area by MAXWCNSTEP */
static void
wchmm_extend(WCHMM_INFO *wchmm)
{
  wchmm->maxwcn += MAXWCNSTEP;
  wchmm->state = (WCHMM_STATE *)myrealloc(wchmm->state, sizeof(WCHMM_STATE)*wchmm->maxwcn);
  wchmm->stend = (WORD_ID *)myrealloc(wchmm->stend, sizeof(WORD_ID)*wchmm->maxwcn);
}

/* startnode を STARTNODE_STEP 分伸長 */
/* expand startnode[] by STARTNODE_STEP */
static void
wchmm_extend_startnode(WCHMM_INFO *wchmm)
{
  wchmm->maxstartnum += STARTNODE_STEP;
  wchmm->startnode = (int *)myrealloc(wchmm->startnode, sizeof(int) * wchmm->maxstartnum);
#ifdef CATEGORY_TREE
  wchmm->start2wid = (WORD_ID *)myrealloc(wchmm->startnode, sizeof(WORD_ID) * wchmm->maxstartnum);
#endif
}
  
/* wchmm を free する */
void
wchmm_free(WCHMM_INFO *w)
{
  int i;

  /* wchmm->state[i].ac malloced by mybmalloc2() */
  /* wchmm->offset[][] malloced by mybmalloc2() */
#ifdef PASS1_IWCD
  /* LRC_INFO, RC_INFO in wchmm->state[i].outsty malloced by mybmalloc2() */
#endif
  /* they all will be freed by a single mybfree2() call */
  mybfree2(&(w->malloc_root));
#ifndef CATEGORY_TREE
  if (w->sclist != NULL) {
    S_CELL *sc, *sctmp;
    for(i=0;i<w->scnum;i++) {
      sc = w->sclist[i];
      while(sc) {
	sctmp = sc->next;
	free(sc);
	sc = sctmp;
      }
    }
    free(w->sclist);
  }
#endif
#ifdef UNIGRAM_FACTORING
  if (w->start2isolate != NULL) free(w->start2isolate);
#endif
#ifdef CATEGORY_TREE
  free(w->start2wid);
#endif
  free(w->startnode);
  free(w->wordend);
  free(w->wordbegin);
  free(w->offset);
  free(w->stend);
  free(w->state);
  free(w);
}

/**************************************************************/
/*********** word sort functions ******************************/
/**************************************************************/

static WORD_INFO *local_winfo;	/* temporary for sort function callbacks */

/* 単語を音素のならびでソートするqsort関数 */
/* qsort function to sort words by their phoneme sequence */
static int
compare_wseq(WORD_ID *widx1, WORD_ID *widx2)
{
  int len1, len2, n;
  int p=0;
  
  len1 = local_winfo->wlen[*widx1];
  len2 = local_winfo->wlen[*widx2];

  n=0;
  /*  while (n < len1 && n < len2 && (p = (int)winfo->wseq[*widx1][n] - (int)winfo->wseq[*widx2][n]) == 0 ) n++;*/
  while (n < len1 && n < len2 && (p = strcmp((local_winfo->wseq[*widx1][n])->name, (local_winfo->wseq[*widx2][n])->name)) == 0 ) n++;
  if (n < len1) {
    if (n < len2) {
      /* differ */
      return(p);
    } else {
      /* 2 is part of 1 */
      return(1);
    }
  } else {
    if (n < len2) {
      /* 1 is part of 2 */
      return(-1);
    } else {
      /* same */
      return(0);
    }
  }
}

/* 単語IDの集合 windex[bgn..bgn+len-1] を単語の音素ならびでソートする */
/* sort word IDs in windex[bgn..bgn+len-1] by their phoneme sequence order */
static void
wchmm_sort_idx_by_wseq(WORD_INFO *winfo, WORD_ID *windex, WORD_ID bgn, WORD_ID len)
{
  local_winfo = winfo;
  qsort(&(windex[bgn]), len, sizeof(WORD_ID), (int (*)(const void *, const void *))compare_wseq);
}

#ifdef CATEGORY_TREE
/* 単語をカテゴリIDでソートするqsort関数 */
/* qsort function to sort words by their category ID */
static int
compare_category(WORD_ID *widx1, WORD_ID *widx2)
{
  int c1,c2;
  c1 = local_winfo->wton[*widx1];
  c2 = local_winfo->wton[*widx2];
  return(c1 - c2);
}

/* 単語ID集合 windex[0..len-1] をカテゴリIDでソートする */
/*  sort word IDs in windex[0..len-1] by their category ID */
static void
wchmm_sort_idx_by_category(WORD_INFO *winfo, WORD_ID *windex, WORD_ID len)
{
  local_winfo = winfo;
  qsort(windex, len, sizeof(WORD_ID), (int (*)(const void *, const void *))compare_category);
}
#endif /* CATEGORY_TREE */
  

/*******************************************************/
/************** link part of words  ********************/
/*******************************************************/

/* 2単語 i, j を比較し, 先頭から互いに共有可能な音素数を返す
   返り値: 共有音素数 */
/* compare 2 words 'i', 'j' from start phoneme, and return the number
   of sharable phonemes. */
static int
wchmm_check_match(WORD_INFO *winfo, int i, int j)
{
  int k,tmplen;

  for (tmplen=0,k=0;k<winfo->wlen[i];k++) {
    if (k > winfo->wlen[j]-1)
      break;
    if (! (strmatch(winfo->wseq[i][k]->name, winfo->wseq[j][k]->name)))
      break;
    tmplen++;
  }
  return(tmplen);
}

/* wchmm 上のノード node に ノード arc への対数確率 a の遷移を追加 */
/* add a transition from 'node' to 'arc' with logprob 'a' on wchmm */
static void
add_wacc(WCHMM_INFO *wchmm, int node, LOGPROB a, int arc)
{
  A_CELL *ac;
  ac       = (A_CELL *) mybmalloc2(sizeof(A_CELL), &(wchmm->malloc_root));
  ac->a    = a;
  ac->arc  = arc;
  ac->next = wchmm->state[node].ac;
  wchmm->state[node].ac   = ac;
}

/* ある単語のある位置の音素から外へ出る遷移のリストを得る */
/* make outgoing transition list for given phone position of a word */
static void
get_outtrans_list(WCHMM_INFO *wchmm, WORD_ID w, int pos, int *node, LOGPROB *a, int *num, int maxnum, boolean insert_sp)
{
  HMM_Logical *ltmp;
  int states;
  int k, ato;
  LOGPROB prob;
  int oldnum;

  if (pos < 0) {
    
    /* set the word-beginning node, and return */
    node[*num] = wchmm->wordbegin[w];
    a[*num] = 0.0;
    (*num)++;
    
  } else {

    ltmp = wchmm->winfo->wseq[w][pos];
    states = hmm_logical_state_num(ltmp);

    /* check initial->final state */
    if ((hmm_logical_trans(ltmp))->a[0][states-1] != LOG_ZERO) {
      /* recursive call for previous phone */
      oldnum = *num;
      get_outtrans_list(wchmm, w, pos-1, node, a, num, maxnum, FALSE); /* previous phone should not be an sp-inserted phone */
      /* add probability of the skip transition to all the previous ones */
      for(k=oldnum;k<*num;k++) {
	a[k] += (hmm_logical_trans(ltmp))->a[0][states-1];
      }
    }
    /* add to list the arcs from output state to final state */
    for (k = 1; k < states - 1; k++) {
      prob = (hmm_logical_trans(ltmp))->a[k][states-1];
      if (prob != LOG_ZERO) {
	if (*num >= maxnum) {
	  j_error("Maximum outtrans list num exceeded! (%d)\n", maxnum);
	}
	node[*num] = wchmm->offset[w][pos] + k - 1;
	a[*num] = prob;
	(*num)++;
      }
    }
    /* for -iwsp, add outgoing arc from the tail sp model
       only if need_sp == TRUE.
       need_sp should be TRUE only when the connecting [pos] phone is also an end phone of the to-be-added word (i.e. homophone word)
     */
    /*  */
    if (enable_iwsp && insert_sp) {
      /* consider sp */
      for (k = 1; k < hmm_logical_state_num(wchmm->hmminfo->sp) - 1; k++) {
	prob = hmm_logical_trans(wchmm->hmminfo->sp)->a[k][hmm_logical_state_num(wchmm->hmminfo->sp)-1];
	if (prob != LOG_ZERO) {
	  if (*num >= maxnum) {
	    j_error("Maximum outtrans list num exceeded! (%d)\n", maxnum);
	  }
	  node[*num] = wchmm->offset[w][pos] + (states - 2) + k - 1;
	  a[*num] = prob;
	  (*num)++;
	}
      }
    }
  }
  /*printf("   %d(%s)-%d:\"%s\", num=%d\n", w, wchmm->winfo->woutput[w], pos,
    (pos < 0) ? "BGN" : wchmm->winfo->wseq[w][pos]->name, *num);*/
  return;
}  
  

/**************************************************************/
/*************** add a word to wchmm lexicon tree *************/
/**************************************************************/
/* 単語 word (単語matchwordとmatchlen数の音素数マッチする)を wchmm に
   付け加える */
/* add a word 'word' that matches with 'matchword' for 'matchlen' length
   to wchmm */
static void
wchmm_add_word(
     WCHMM_INFO *wchmm,		/* wchmm info */
     int word,			/* word ID to add */
     int matchlen,		/* num of longest matched phone from start (0=add all) */
     int matchword)		/* longest matched word ID already added in wchmmtree */
{
  int   j,k,n;
  int   add_head, add_tail, add_to;
  int   word_len, matchword_len;
  int *out_from, *out_from_next;
  LOGPROB *out_a, *out_a_next;
  int out_num_prev, out_num_next;
  HMM_Logical *ltmp;
  int ltmp_state_num;
  int ato, kkk;
  LOGPROB prob;
  int ntmp;
  
/* 
 *   if (matchlen > 0) {
 *     printf("--\n");
 *     put_voca(wchmm->winfo, word);
 *     put_voca(wchmm->winfo, matchword);
 *     printf("matchlen=%d\n", matchlen);
 *   }
 */

  /* variable abbreviations */
  n = wchmm->n;
  word_len      = wchmm->winfo->wlen[word];
  matchword_len = wchmm->winfo->wlen[matchword];

  /* malloc phone offset area */
  if((wchmm->offset[word]=(int *)mybmalloc2(sizeof(int)*word_len, &(wchmm->malloc_root)))==NULL){ 
    j_error("malloc failed at wchmm_add_word()\n");
  }
  
  /* allocate unshared (new) part */
  add_head = matchlen;
  add_tail = word_len - 1;
  add_to   = matchlen - 1;

  /* make word-beginning node if needed */
  if (matchlen == 0) {
    /* create word-beginning node */
    wchmm->wordbegin[word] = n;
    wchmm->stend[n] = WORD_INVALID;
    wchmm->state[n].ac = NULL;
    wchmm->state[n].out.state = NULL;
    /* index the new word-beginning node as startnode (old ststart) */
    wchmm->startnode[wchmm->startnum] = n;
#ifdef CATEGORY_TREE
    wchmm->start2wid[wchmm->startnum] = word;
#endif
    /* expand data area if necessary */
    if (++wchmm->startnum >= wchmm->maxstartnum) wchmm_extend_startnode(wchmm);
    if (++n >= wchmm->maxwcn) wchmm_extend(wchmm);
  } else {
    wchmm->wordbegin[word] = wchmm->wordbegin[matchword];
  }

  /* now n is at beginning of output state */

  /* prepare work area for skip arcs */
  out_from = (int *)mymalloc(sizeof(int) * wchmm->winfo->maxwn);
  out_from_next = (int *)mymalloc(sizeof(int) * wchmm->winfo->maxwn);
  out_a = (LOGPROB *)mymalloc(sizeof(LOGPROB) * wchmm->winfo->maxwn);
  out_a_next = (LOGPROB *)mymalloc(sizeof(LOGPROB) * wchmm->winfo->maxwn);

  /* store the initial outgoing arcs to out_from[] and out_a[] */
  out_num_prev = 0;
  if (matchlen == 0) {
    /* set the word-beginning node */
    out_from[0] = wchmm->wordbegin[word];
    out_a[0] = 0.0;
    out_num_prev = 1;
  } else {
    /*printf("%d(%s)\n", word, wchmm->winfo->woutput[word]);*/
    /* on -iwsp, trailing sp is needed only when no phone will be created */
    get_outtrans_list(wchmm, matchword, add_to, out_from, out_a, &out_num_prev, wchmm->winfo->maxwn, (add_tail - add_head + 1 > 0) ? FALSE : TRUE);
    /*printf("NUM=%d\n", out_num_prev);*/
  }
  
  if (add_tail - add_head + 1 > 0) { /* there are new phones to be created */
    {
#ifdef PASS1_IWCD
      CD_Set *lcd = NULL;
#endif
      
      ntmp = n;
      for (j=add_head; j <= add_tail; j++) { /* for each new phones */
	ltmp = wchmm->winfo->wseq[word][j];
	ltmp_state_num = hmm_logical_state_num(ltmp);
#ifdef PASS1_IWCD
	if (ccd_flag) {
	  /* in the triphone lexicon tree, the last phone of a word has
	     left-context cdset */
	  if (wchmm->winfo->wlen[word] > 1 && j == wchmm->winfo->wlen[word] - 1) {
#ifdef CATEGORY_TREE
	    if (! old_iwcd_flag) {
	      lcd = lcdset_lookup_with_category(hmminfo, ltmp, wchmm->winfo->wton[word]);
	      if (lcd == NULL) {
		/* no category-aware cdset found.  This is case when no word
		   can follow this word grammatically.
		   so fallback to normal state */
		j_printerr("Warning: no lcdset found for [%s::%d], fallback to [%s]\n", ltmp->name, wchmm->winfo->wton[word], ltmp->name);
		lcd = lcdset_lookup_by_hmmname(hmminfo, ltmp->name);
	      }
	    } else {
	      lcd = lcdset_lookup_by_hmmname(hmminfo, ltmp->name);
	    }
#else
	    lcd = lcdset_lookup_by_hmmname(hmminfo, ltmp->name);
#endif
	    if (lcd == NULL) {
	      j_error("Error: no lcdset found for [%s]\n",ltmp->name);
	    }
	  }
	}
#endif /* PASS1_IWCD */
	for (k = 1; k < ltmp_state_num - 1; k++) { /* for each state in the phone */
	  /* set state output prob info */
#ifdef PASS1_IWCD
	  if (ccd_flag) {
	    /* output info of triphones needs special handling */
	    if (wchmm->winfo->wlen[word] == 1) { /* word with only 1 phone */
	      wchmm->state[ntmp].outstyle = AS_LRSET;
	      wchmm->state[ntmp].out.lrset = (LRC_INFO *)mybmalloc2(sizeof(LRC_INFO), &(wchmm->malloc_root));
	      (wchmm->state[ntmp].out.lrset)->hmm       = ltmp;
	      (wchmm->state[ntmp].out.lrset)->state_loc = k;
#ifdef CATEGORY_TREE
	      (wchmm->state[ntmp].out.lrset)->category  = wchmm->winfo->wton[word];
#endif
	    } else if (j == 0) {	/* head phone of a word */
	      wchmm->state[ntmp].outstyle = AS_RSET;
	      wchmm->state[ntmp].out.rset = (RC_INFO *)mybmalloc2(sizeof(RC_INFO), &(wchmm->malloc_root));
	      (wchmm->state[ntmp].out.rset)->hmm       = ltmp;
	      (wchmm->state[ntmp].out.rset)->state_loc = k;
	    } else if (j == wchmm->winfo->wlen[word] - 1) { /* last phone of a word */
	      wchmm->state[ntmp].outstyle = AS_LSET;
	      wchmm->state[ntmp].out.lset = &(lcd->stateset[k]);
	    } else {
	      wchmm->state[ntmp].outstyle = AS_STATE;
	      if (ltmp->is_pseudo) {
		j_printerr("Warning: word-internal phone should not be pseudo\n");
		put_voca(wchmm->winfo, word);
	      }
	      wchmm->state[ntmp].out.state = ltmp->body.defined->s[k];
	    }
	  } else {
	    /* monophone */
	    if (ltmp->is_pseudo) {
	      j_printerr("InternalError: CDSET phoneme exist in monophone?\n");
	      put_voca(wchmm->winfo, word);
	    }
	    wchmm->state[ntmp].outstyle = AS_STATE;
	    wchmm->state[ntmp].out.state = ltmp->body.defined->s[k];
	  }
#else  /* ~PASS1_IWCD */
	  if (ltmp->is_pseudo) {
	    j_printerr("InternalError: CDSET phone exist in monophone?\n");
	    put_voca(wchmm->winfo, word);
	  }
	  wchmm->state[ntmp].out = ltmp->body.defined->s[k];
#endif /* PASS1_IWCD */

	  /* initialize other info */
	  wchmm->state[ntmp].ac = NULL;
	  wchmm->stend[ntmp] = WORD_INVALID;
	  
	  ntmp++;
	  /* expand wchmm if neccesary */
	  if (ntmp >= wchmm->maxwcn) wchmm_extend(wchmm);
	}
      }

      /* make transition arc from HMM transition info */
      ntmp = n;
      for (j = add_head; j <= add_tail; j++) {
	ltmp = wchmm->winfo->wseq[word][j];
	ltmp_state_num = hmm_logical_state_num(ltmp);
	out_num_next = 0;
	/* arc from initial state ... need arc expansion from precious phone */
	for (ato = 1; ato < ltmp_state_num; ato++) {
	  prob = (hmm_logical_trans(ltmp))->a[0][ato];
	  if (prob != LOG_ZERO) {
	    /* expand arc from previous HMM */
	    if (ato == ltmp_state_num - 1) {
	      /* to final state ... just register states for next expansion */
	      for(kkk=0; kkk<out_num_prev; kkk++) {
		out_from_next[out_num_next] = out_from[kkk];
		out_a_next[out_num_next] = out_a[kkk] + prob;
		out_num_next++;
	      }
	    } else {
	      for(kkk=0; kkk<out_num_prev; kkk++) {
		add_wacc(wchmm, out_from[kkk], out_a[kkk] + prob, ntmp + ato - 1);
	      }
	    }
	  }
	}
	/* from outprob state */
	for(k = 1; k < ltmp_state_num - 1; k++) {
	  for (ato = 1; ato < ltmp_state_num; ato++) {
	    prob = (hmm_logical_trans(ltmp))->a[k][ato];
	    if (prob != LOG_ZERO) {
	      if (ato == ltmp_state_num - 1) {
		/* to final state ... register states for next expansion */
		out_from_next[out_num_next] = ntmp;
		out_a_next[out_num_next] = prob;
		out_num_next++;
	      } else {
		add_wacc(wchmm, ntmp, prob, ntmp + ato - k);
	      }
	    }
	  }
	  ntmp++;
        } /* end of state loop */
	/* swap out list for next phone */
	for(kkk=0;kkk<out_num_next;kkk++) {
	  out_from[kkk] = out_from_next[kkk];
	  out_a[kkk] = out_a_next[kkk];
	}
	out_num_prev = out_num_next;
      }	/* end of phone loop */
    }
  } /* new phone node creation loop for this word */

  /* if -iwsp, add noise model to the end of word at ntmp */
  if (enable_iwsp && add_tail - add_head + 1 > 0) { /* there are new phones to be created */
    int ntmp_bak;
    
    /* set short pause state info */
    ntmp_bak = ntmp;
    if (hmminfo->sp->is_pseudo) {
      for(k = 1;k < hmm_logical_state_num(hmminfo->sp) - 1; k++) {
	wchmm->state[ntmp].outstyle = AS_LSET;
	wchmm->state[ntmp].out.lset = &(hmminfo->sp->body.pseudo->stateset[k]);
	wchmm->state[ntmp].ac = NULL;
	wchmm->stend[ntmp] = WORD_INVALID;
	ntmp++;
	if (ntmp >= wchmm->maxwcn) wchmm_extend(wchmm);
      }
    } else {
      for(k = 1;k < hmm_logical_state_num(hmminfo->sp) - 1; k++) {
	wchmm->state[ntmp].outstyle = AS_STATE;
	wchmm->state[ntmp].out.state = hmminfo->sp->body.defined->s[k];
	wchmm->state[ntmp].ac = NULL;
	wchmm->stend[ntmp] = WORD_INVALID;
	ntmp++;
	if (ntmp >= wchmm->maxwcn) wchmm_extend(wchmm);
      }
    }
    ntmp = ntmp_bak;
    /* connect incoming arcs from previous phone */
    out_num_next = 0;
    for (ato = 1; ato < hmm_logical_state_num(hmminfo->sp); ato++) {
      prob = hmm_logical_trans(hmminfo->sp)->a[0][ato];
      if (prob != LOG_ZERO) {
	/* to control short pause insertion, transition probability toward
	 the word-end short pause will be given a penalty */
	prob += hmminfo->iwsp_penalty;
	if (ato == hmm_logical_state_num(hmminfo->sp) - 1) {
	  /* model has a model skip transition, just inherit them to next */
	  for(kkk=0; kkk<out_num_prev; kkk++) {
	    out_from_next[out_num_next] = out_from[kkk];
	    out_a_next[out_num_next] = out_a[kkk] + prob;
	    out_num_next++;
	  }
	} else {
	  /* connect incoming arcs from previous phone to this phone */
	  for(kkk=0; kkk<out_num_prev; kkk++) {
	    add_wacc(wchmm, out_from[kkk], out_a[kkk] + prob, ntmp + ato - 1);
	  }
	}
      }
    }
    /* if short pause model doesn't have a model skip transition, also add it */
    if (hmm_logical_trans(hmminfo->sp)->a[0][hmm_logical_state_num(hmminfo->sp)-1] == LOG_ZERO) {
      /* to make insertion sp model to have no effect on the original path,
	 the skip transition probability should be 0.0 (=100%) */
      prob = 0.0;
      for(kkk=0; kkk<out_num_prev; kkk++) {
	out_from_next[out_num_next] = out_from[kkk];
	out_a_next[out_num_next] = out_a[kkk] + prob;
	out_num_next++;
      }
    }
    /* connect arcs within model, and store new outgoing arcs for wordend node */
    for (k = 1; k < hmm_logical_state_num(hmminfo->sp) - 1; k++) {
      for (ato = 1; ato < hmm_logical_state_num(hmminfo->sp); ato++) {
	prob = hmm_logical_trans(hmminfo->sp)->a[k][ato];
	if (prob != LOG_ZERO) {
	  if (ato == hmm_logical_state_num(hmminfo->sp) - 1) {
	    out_from_next[out_num_next] = ntmp;
	    out_a_next[out_num_next] = prob;
	    out_num_next++;
	  } else {
	    add_wacc(wchmm, ntmp, prob, ntmp + ato - k);
	  }
	}
      }
      ntmp++;
    }
    /* swap work area for next */
    for(kkk=0;kkk<out_num_next;kkk++) {
      out_from[kkk] = out_from_next[kkk];
      out_a[kkk] = out_a_next[kkk];
    }
    out_num_prev = out_num_next;
  }
      
  /* make mapping: word <-> node on wchmm */
  for (j=0;j<word_len;j++) {
    if (j < add_head) {	/* shared part */
      wchmm->offset[word][j] = wchmm->offset[matchword][j];
    } else if (add_tail < j) { /* shared tail part (should not happen..) */
      wchmm->offset[word][j] = wchmm->offset[matchword][j+(matchword_len-word_len)];
    } else {			/* newly created part */
      wchmm->offset[word][j] = n;
      n += hmm_logical_state_num(wchmm->winfo->wseq[word][j]) - 2;
    }
  }
  if (enable_iwsp && add_tail - add_head + 1 > 0) {
    n += hmm_logical_state_num(hmminfo->sp) - 2;
    if (n != ntmp) j_error("Algorithm Error! cannot match\n");
  }
  /* make word-end node (always create for each new word) */
  wchmm->wordend[word] = n;	/* tail node of 'word' is 'n' */
  wchmm->stend[n] = word;	/* node 'k' is a tail node of 'word' */
  wchmm->state[n].ac = NULL;
  wchmm->state[n].out.state = NULL;
  /* connect the final outgoing arcs in out_from[] to the word end node */
  for(k = 0; k < out_num_prev; k++) {
    add_wacc(wchmm, out_from[k], out_a[k], n);
  }
  n++;
  if (n >= wchmm->maxwcn) wchmm_extend(wchmm);

  if (matchlen == 0) {
    /* check if the new word has whole word-skipping transition */
    /* (use out_from and out_num_prev temporary) */
    out_num_prev = 0;
    get_outtrans_list(wchmm, word, word_len-1, out_from, out_a, &out_num_prev, wchmm->winfo->maxwn, TRUE);
    for(k=0;k<out_num_prev;k++) {
      if (out_from[k] == wchmm->wordbegin[word]) {
	j_printerr("\n*** ERROR: WORD SKIPPING TRANSITION NOT ALLOWED ***\n");
	j_printerr("  Word id=%d (%s[%s]) has \"word skipping transition\".\n", word, wchmm->winfo->wname[word], wchmm->winfo->woutput[word]);
	j_printerr("  All HMMs in the word:\n    ");
	for(kkk=0;kkk<wchmm->winfo->wlen[word];kkk++) {
	  j_printerr("%s ", wchmm->winfo->wseq[word][kkk]->name);
	}
	j_printerr("\n  has transitions from initial state to final state.\n");
	j_error("  This type of word skipping is not supported.\n");
      }
    }
  }

  wchmm->n = n;
  free(out_from);
  free(out_from_next);
  free(out_a);
  free(out_a_next);
}

/***************************************************************/
/****** for separation of high-frequent words from tree  *******/
/***************************************************************/
#ifdef USE_NGRAM
#ifdef SEPARATE_BY_UNIGRAM

/* unigram確率でソートするための qsort 関数 */
/* qsort function to sort unigram values */
static int
compare_prob(LOGPROB *a, LOGPROB *b)
{
  if (*a < *b)  return (1);
  if (*a > *b)  return (-1);
  return(0);
}

/* 1-gramスコアの上位 N 番目の値を求める */
/* get the Nth-best unigram probability from all words */
static LOGPROB
get_nbest_uniprob(WORD_INFO *winfo, int n)
{
  LOGPROB *u_p;
  WORD_ID w;
  LOGPROB x;

  if (n < 1) n = 1;
  if (n > winfo->num) n = winfo->num;

  /* store all unigram probability to u_p[] */
  u_p = (LOGPROB *)mymalloc(sizeof(LOGPROB) * winfo->num);
  for(w=0;w<winfo->num;w++) u_p[w] =
#ifdef CLASS_NGRAM
			     class_uni_prob(ngram, winfo->wton[w]);
#else
                             uni_prob(ngram, winfo->wton[w]);
#endif
  /* sort them downward */
  qsort(u_p, winfo->num, sizeof(LOGPROB),
	(int (*)(const void *,const void *))compare_prob);

  /* return the Nth value */
  x = u_p[n-1];
  free(u_p);
  return(x);
}

#endif
#endif /* USE_NGRAM */


/**********************************************************/
/****** MAKE WCHMM (LEXICON TREE) --- main function *******/
/**********************************************************/
static int separated_word_count; /* words num actually separated */

#ifdef CATEGORY_TREE

/*
  注意: build_wchmm() は過去のバージョンとのチェック用に残してあるのみ．
  通常はより早い build_wchmm2() を使います．
  (build_wchmm() は Julian モードで "-oldtree" オプションで使用できます)
/*
  NOTE: build_wchmm() is old and only left for historical reason.
  please use build_wchmm2().  it's faster.
  (build_wchmm() is used when Julian mode and "-oldtree" specified)
*/

#define COUNT_STEP 500         /* count log period */

/*
  木構造化辞書作成(古い)
  作り方: 既に登録済の単語の中から音素列が最長マッチする単語をみつけて
  そこに付け加える．
*/
/* old construction function
   tree-organization algorithm:
      search already added words to search for the best resemble word
*/
void
build_wchmm(WCHMM_INFO *wchmm)
{
  int i,j;
  int matchword=0, sharelen=0, maxsharelen=0;
  int counter = COUNT_STEP;
#ifdef SEPARATE_BY_UNIGRAM
  LOGPROB separate_thres;
#endif

  /* lingustic infos must be set before build_wchmm() is called */
  /* check if necessary lingustic info is already assigned (for debug) */
  if (wchmm->winfo == NULL
#ifdef USE_NGRAM
      || wchmm->ngram == NULL
#endif
#ifdef USE_DFA
      || wchmm->dfa == NULL
#endif
      ) {
    j_error("InternalError: build_wchmm: lingustic info not available!!\n");
  }
  
#ifdef SEPARATE_BY_UNIGRAM
  /* 上位[separate_wnum]番目の1-gramスコアを求める */
  /* 1-gramスコアがこの値以上のものは木から分ける */
  separate_thres = get_nbest_uniprob(wchmm->winfo, separate_wnum);
#endif

#ifdef PASS1_IWCD
#ifdef CATEGORY_TREE
  if (ccd_flag && !old_iwcd_flag) {
    /* 全てのカテゴリID付き lcd_set を作成 */
    lcdset_register_with_category_all(hmminfo, winfo, dfa);
  }
#endif /* CATEGORY_TREE */
#endif /* PASS1_IWCD */
  

  /* wchmmを初期化 */
  wchmm_init(wchmm);

  /* 同音語カウンタリセット */
  homophone_word_num = 0;
  separated_word_count=0;

  j_printerr("Building HMM lexicon tree (left-to-right)...\n");
  for (i=0;i<wchmm->winfo->num;i++) {
    if (verbose_flag) {
      if (i >= counter) {
       j_printerr("\r %5d words proceeded (%6d nodes)",i,wchmm->n);
       counter += COUNT_STEP;
      }
    }
#ifdef USE_NGRAM
    if (i == wchmm->winfo->head_silwid || i == wchmm->winfo->tail_silwid) {
      /* 先頭/末尾の無音モデルは木構造化せず，
       * 先頭の無音単語の先頭への遷移，末尾単語の末尾からの遷移は作らない*/
      wchmm_add_word(wchmm, i,0,0); /* sharelen=0でそのまま */
      continue;
    }
#ifndef NO_SEPARATE_SHORT_WORD
    if (wchmm->winfo->wlen[i] <= SHORT_WORD_LEN) {
      /* 長さの短い単語を木構造化しない(ここでは1音節) */
      wchmm_add_word(wchmm, i,0,0); /* sharelen=0でそのまま */
      separated_word_count++;
      continue;
    }
#endif
#ifdef SEPARATE_BY_UNIGRAM
    if (
#ifdef CLASS_NGRAM
       class_uni_prob(wchmm->ngram, wchmm->winfo->wton[i])
#else
       uni_prob(wchmm->ngram, wchmm->winfo->wton[i])
#endif
       >= separate_thres && separated_word_count < separate_wnum) {
      /* 頻度の高い単語を木構造化しない */
      /* separate_thres は上位separate_wnum番目のスコア */
      wchmm_add_word(wchmm, i,0,0);
      separated_word_count++;
      continue;
    }
#endif
#endif /* USE_NGRAM */
    /* 最も長く音素を共有出来る単語を探す */
    maxsharelen=0;
    for (j=0;j<i;j++) {
#ifdef CATEGORY_TREE
      if (wchmm->winfo->wton[i] != wchmm->winfo->wton[j]) continue;
#endif
      sharelen = wchmm_check_match(wchmm->winfo, i, j);
      if (sharelen == wchmm->winfo->wlen[i] && sharelen == wchmm->winfo->wlen[j]) {
       /* word に同音語が存在する */
       homophone_word_num++;
       /* 必ず最大の長さであり，重複カウントを避けるためここで抜ける */
       maxsharelen = sharelen;
       matchword = j;
       break;
      }
      if (sharelen > maxsharelen) {
       matchword = j;
       maxsharelen = sharelen;
      }
    }
    wchmm_add_word(wchmm, i,maxsharelen,matchword);
  }
  j_printerr("\r %5d words ended     (%6d nodes)\n",i,wchmm->n);

#if 0
  /* 木構造を作らない */
  for (i=0;i<wchmm->winfo->num;i++) {
    if (verbose_flag) {
      if (i >= counter) {
       j_printerr("  %5d words proceeded (%6d nodes)\n",i,wchmm->n);
       counter += COUNT_STEP;
      }
    }
    wchmm_add_word(wchmm, i,0,0); /* sharelen=0でそのまま */
  }
  j_printerr("  %5d words ended     (%6d nodes)\n",i,wchmm->n);
#endif  
  
  /* wchmmの整合性をチェックする */
  check_wchmm(wchmm);

  /* factoring用に各状態に後続単語のリストを付加する */
#ifndef CATEGORY_TREE
  make_successor_list(wchmm);
#ifdef USE_NGRAM
#ifdef UNIGRAM_FACTORING
  /* 前もってfactoring値を計算 */
  /* 末端以外のscは必要ないのでフリーする */
  calc_all_unigram_factoring_values(wchmm);
  j_printerr("  1-gram factoring values has been pre-computed\n");
#endif /* UNIGRAM_FACTORING */
#endif /* USE_NGRAM */
  /* sc のリスト作成 */
  make_sc_index(wchmm);
#endif
#ifndef CATEGORY_TREE
#ifdef USE_NGRAM
#ifdef UNIGRAM_FACTORING
  /* 単語間LMキャッシュが必要なノードのリストを作る */
  make_iwcache_index(wchmm);
#endif /* UNIGRAM_FACTORING */
#endif /* USE_NGRAM */
#endif /* CATEGORY_TREE */
  j_printerr("done\n");

  /* 起動時 -check でチェックモードへ */
  if (wchmm_check_flag) {
    wchmm_check_interactive(wchmm);
  }
}

#endif /* CATEGORY_TREE */

/*
  木構造化辞書作成(新)
  作り方: (1)まず音素列でソートする．(2)ソート後は,各単語について
  その「直前の単語」が常に最も長くマッチする単語となる．
*/
/* new lexicon tree construction function */
/* tree-organization algorithm:
   1. sort all words by their phone sequence.
   2. Then,, for each word, the previous one is the best matched word.
*/
  
void
build_wchmm2(WCHMM_INFO *wchmm)
{
  int i,j, last_i;
  int count_step, counter;
  WORD_ID *windex;
#ifdef USE_NGRAM
#ifdef SEPARATE_BY_UNIGRAM
  LOGPROB separate_thres;
#endif
#endif

  /* lingustic infos must be set before build_wchmm() is called */
  /* check if necessary lingustic info is already assigned (for debug) */
  if (wchmm->winfo == NULL
#ifdef USE_NGRAM
      || wchmm->ngram == NULL
#endif
#ifdef USE_DFA
      || wchmm->dfa == NULL
#endif
      ) {
    j_error("InternalError: build_wchmm: lingustic info not available!!\n");
  }
  
  separated_word_count = 0;
  count_step = wchmm->winfo->num / 10;
  counter = count_step;
  
  j_printerr("Building HMM lexicon tree");
  
#ifdef USE_NGRAM
#ifdef SEPARATE_BY_UNIGRAM
  /* compute score threshold beforehand to separate words from tree */
  /* here we will separate best [separate_wnum] words from tree */
  separate_thres = get_nbest_uniprob(wchmm->winfo, separate_wnum);
#endif
#endif

#ifdef PASS1_IWCD
#ifdef CATEGORY_TREE
  if (ccd_flag && !old_iwcd_flag) {
    /* when Julian mode (category-tree) and triphone is used,
       make all category-indexed context-dependent phone set (cdset) here */
    /* these will be assigned on the last phone of each word on tree */
    lcdset_register_with_category_all(hmminfo, winfo, dfa);
  }
#endif /* CATEGORY_TREE */
#endif /* PASS1_IWCD */
  
  /* initialize wchmm */
  wchmm_init(wchmm);

  /* make sorted word index ordered by phone sequence */
  windex = (WORD_ID *)mymalloc(sizeof(WORD_ID) * wchmm->winfo->num);
  for(i=0;i<wchmm->winfo->num;i++) windex[i] = i;
#ifdef CATEGORY_TREE
  /* sort by category -> sort by word ID in each category */
  wchmm_sort_idx_by_category(wchmm->winfo, windex, wchmm->winfo->num);
  {
    int last_cate;
    last_i = 0;
    last_cate = wchmm->winfo->wton[windex[0]];
    for(i = 1;i<wchmm->winfo->num;i++) {
      if (wchmm->winfo->wton[windex[i]] != last_cate) {
	wchmm_sort_idx_by_wseq(wchmm->winfo, windex, last_i, i - last_i);
	last_cate = wchmm->winfo->wton[windex[i]];
	last_i = i;
      }
    }
    wchmm_sort_idx_by_wseq(wchmm->winfo, windex, last_i, wchmm->winfo->num - last_i);
  }
#else
  /* sort by word ID for whole vocabulary */
  wchmm_sort_idx_by_wseq(wchmm->winfo, windex, 0, wchmm->winfo->num);
#endif

/* 
 *   {
 *     int i,w;
 *     for(i=0;i<wchmm->winfo->num;i++) {
 *	 w = windex[i];
 *	 printf("%d: cate=%4d wid=%4d %s\n",i, wchmm->winfo->wton[w], w, wchmm->winfo->woutput[w]);
 *     }
 *   }
 */

  /* incrementaly add words to lexicon tree */
  /* now for each word, the previous word (last_i) is always the most matched one */
  last_i = WORD_INVALID;
  for (j=0;j<wchmm->winfo->num;j++) {
    i = windex[j];
    if (j >= counter) {
      /*j_printerr("\r %5d words proceeded (%6d nodes)",j, wchmm->n);*/
      j_printerr(".");
      counter += count_step;
    }
#ifdef USE_NGRAM
    /* start/end silence word should not be shared */
    if (i == wchmm->winfo->head_silwid || i == wchmm->winfo->tail_silwid) {
      wchmm_add_word(wchmm, i,0,0); /* add whole word as new (sharelen=0) */
      continue;
    }
#ifndef NO_SEPARATE_SHORT_WORD
    /* separate short words from tree */
    if (wchmm->winfo->wlen[i] <= SHORT_WORD_LEN) {
      wchmm_add_word(wchmm, i,0,0);
      separated_word_count++;
      continue;
    }
#endif
#ifdef SEPARATE_BY_UNIGRAM
    /* separate high-frequent words from tree (threshold = separate_thres) */
    if (
#ifdef CLASS_NGRAM
	class_uni_prob(wchmm->ngram, wchmm->winfo->wton[i])
#else
	uni_prob(wchmm->ngram, wchmm->winfo->wton[i])
#endif
	>= separate_thres && separated_word_count < separate_wnum) {
      wchmm_add_word(wchmm, i,0,0);
      separated_word_count++;
      continue;
    }
#endif
#endif /* USE_NGRAM */
    if (last_i == WORD_INVALID) { /* first word */
      wchmm_add_word(wchmm, i,0,0);
    } else {
      /* the previous word (last_i) is always the most matched one */
#ifdef CATEGORY_TREE
      if (wchmm->winfo->wton[i] != wchmm->winfo->wton[last_i]) {
	wchmm_add_word(wchmm, i,0,0);
      } else {
	wchmm_add_word(wchmm, i, wchmm_check_match(wchmm->winfo, i, last_i), last_i);
      }
#else
      wchmm_add_word(wchmm, i, wchmm_check_match(wchmm->winfo, i, last_i), last_i);
#endif
    }
    last_i = i;
    
  } /* end of add word loop */
  
  /*j_printerr("\r %5d words ended     (%6d nodes)\n",j,wchmm->n);*/

  /* free work area */
  free(windex);
  
  j_printerr("%d nodes\n", wchmm->n);
  
  /* check wchmm coherence (internal debug) */
  check_wchmm(wchmm);

  /* make successor list for all branch nodes for N-gram factoring */
#ifndef CATEGORY_TREE
  make_successor_list(wchmm);
#ifdef UNIGRAM_FACTORING
  /* for 1-gram factoring, we can compute the values before search */
  calc_all_unigram_factoring_values(wchmm);
  j_printerr("  1-gram factoring values has been pre-computed\n");
#endif /* UNIGRAM_FACTORING */
  /* make index to factoring node for factoring cache */
  make_sc_index(wchmm);
#endif
#ifndef CATEGORY_TREE
#ifdef UNIGRAM_FACTORING
  /* make list of start nodes that needs inter-word LM cache */
  make_iwcache_index(wchmm);
#endif /* UNIGRAM_FACTORING */
#endif /* CATEGORY_TREE */

  j_printerr("done\n");

  /* go into interactive check mode ("-check" on start) */
  if (wchmm_check_flag) {
    wchmm_check_interactive(wchmm);
  }

#ifdef WCHMM_SIZE_CHECK
  /* detailed check of lexicon tree size (maybe inaccurate!) */
  j_printf("--- estimated size of word lexicon ---\n");
  j_printf("wchmm: %d words, %d nodes\n", wchmm->winfo->num, wchmm->n);
  j_printf("%9d bytes: wchmm->state[node] (exclude ac, sc)\n", sizeof(WCHMM_STATE) * wchmm->n);
  j_printf("%9d bytes: wchmm->stend[node]\n", sizeof(WORD_ID) * wchmm->n);
  {
    int w,count;
    count = 0;
    for(w=0;w<wchmm->winfo->num;w++) {
      count += wchmm->winfo->wlen[w] * sizeof(int) + sizeof(int *);
    }
    j_printf("%9d bytes: wchmm->offset[w][]\n", count);
  }
  j_printf("%9d bytes: wchmm->wordbegin[w]\n", wchmm->winfo->num * sizeof(int));
  j_printf("%9d bytes: wchmm->wordend[w]\n", wchmm->winfo->num * sizeof(int));
  j_printf("%9d bytes: wchmm->startnode[]\n", wchmm->startnum * sizeof(int));
#ifdef CATEGORY_TREE
  j_printf("%9d bytes: wchmm->start2wid[]\n", wchmm->startnum * sizeof(WORD_ID));
#endif
#ifdef UNIGRAM_FACTORING
  j_printf("%9d bytes: wchmm->start2isolate[]\n", wchmm->isolatenum * sizeof(int));
#endif
#ifndef CATEGORY_TREE
  {
    int i, n;
    S_CELL *sc;
    n = sizeof(S_CELL *) * wchmm->scnum;
    for(i=0;i<wchmm->scnum;i++) {
      for(sc=wchmm->sclist[i];sc;sc=sc->next) {
	n += sizeof(S_CELL);
      }
    }
    j_printf("%9d bytes: wchmm->sclist[]\n", n);
  }
#endif
  j_printf("under state[]:\n");
  {
    A_CELL *ac;
    int count,n;
    count = 0;
    for(n=0;n<wchmm->n;n++) {
      for(ac=wchmm->state[n].ac;ac;ac=ac->next) {
	count += sizeof(A_CELL);
      }
    }
    j_printf("\t%9d: ac\n", count);
  }
#endif /* WCHMM_SIZE_CHECK */
}


/* wchmmの情報を標準出力に出力する */
/* output wchmm info to stdout */
void
print_wchmm_info(WCHMM_INFO *wchmm)
{
  int n,i;

  j_printf("Lexicon tree info:\n");
  j_printf("\t total node num = %6d\n", wchmm->n);
  j_printf("\t  root node num = %6d\n", wchmm->startnum);
#ifdef USE_NGRAM
#ifdef NO_SEPARATE_SHORT_WORD
#ifdef SEPARATE_BY_UNIGRAM
  j_printf(" (%d hi-freq. words are separated from tree)\n", separated_word_count);
#else
  j_printf(" (no words are separated from tree)\n");
#endif /* SEPARATE_BY_UNIGRAM */
#else
  j_printf(" (%d short words (<= %d phonemes) are separated from tree)\n", separated_word_count, SHORT_WORD_LEN);
#endif /* NO_SEPARATE_SHORT_WORD */
#endif /* USE_NGRAM */
  for(n=0,i=0;i<wchmm->n;i++) {
    if (wchmm->stend[i] != WORD_INVALID) n++;
  }
  j_printf("\t  leaf node num = %6d\n", n);
#ifndef CATEGORY_TREE
  j_printf("\t fact. node num = %6d\n", wchmm->scnum);
#endif /* CATEGORY_TREE */
}
