--- lingucomponent/source/spellcheck/myspell/hashmgr.hxx	2003-06-24 12:48:03.000000000 -0700
+++ lingucomponent/source/spellcheck/myspell/hashmgr.hxx	2005-07-27 15:26:18.214586336 -0700
@@ -5,18 +5,27 @@
 
 class HashMgr
 {
-  int             tablesize;
-  struct hentry * tableptr;
+  int                 tablesize;
+  char             ** tableptr;
+  mutable struct hentry compat;
+  int                   compat_word_length;
+  int   fd;
+  char *buffer;
+  size_t length;
+  size_t seek_newline(size_t from_offset) const;
+  int slurp( int fd, size_t bytes );
 
 public:
   HashMgr(const char * tpath);
   ~HashMgr();
 
+  struct hentry *setup_hentry(const char *ptr);
+
   struct hentry * lookup(const char *) const;
-  int hash(const char *) const;
-  struct hentry * walk_hashtable(int & col, struct hentry * hp) const;
+  struct hentry * walk_hashtable(int & col, struct hentry *p) const;
 
 private:
+  int hash(const char *) const;
   HashMgr( const HashMgr & ); // not implemented
   HashMgr &operator=( const HashMgr & ); // not implemented
   int load_tables(const char * tpath);
--- lingucomponent/source/spellcheck/myspell/hashmgr.cxx	2005-11-13 11:26:46.031250000 +0200
+++ lingucomponent/source/spellcheck/myspell/hashmgr.cxx	2005-11-13 11:32:11.062500000 +0200
@@ -6,11 +6,32 @@
 
 #include "hashmgr.hxx"
 
+#ifdef UNX
+#include <sys/mman.h>
+#include <unistd.h>
+#endif
+#ifdef WNT
+#include <io.h>
+typedef int ssize_t;
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <fcntl.h>
+#ifndef _O_BINARY
+#define _O_BINARY 0
+#endif
+#define IS_END(c) ((c) == '\r' || (c) == '\n')
+
+//Uncomment this to print debug msgs
+//#define HASH_DEBUG
+
 extern void mychomp(char * s);
 extern char * mystrdup(const char *);
 
 using namespace std;
 
+#define WORD_END_CONDITN(c) ((c) == 0 || (c) == '/')
 
 // build a hash table from a munched word list
 
@@ -18,6 +39,15 @@
 {
   tablesize = 0;
   tableptr = NULL;
+  compat.wlen = 0;
+  compat.alen = 0;
+  compat.word = NULL;
+  compat_word_length = 0;
+  compat.astr = NULL;
+  compat.next = NULL;
+  fd = -1;
+  length = 0;
+  buffer = NULL;
   int ec = load_tables(tpath);
   if (ec) {
     /* error condition - what should we do here */
@@ -33,27 +63,23 @@
 
 HashMgr::~HashMgr()
 {
-  if (tableptr) {
-    // now pass through hash table freeing up everything
-    // go through column by column of the table
-    for (int i=0; i < tablesize; i++) {
-      struct hentry * pt = &tableptr[i];
-      struct hentry * nt = NULL;
-      if (pt) {
-	if (pt->word) free(pt->word);
-        if (pt->astr) free(pt->astr);
-        pt = pt->next;
-      }
-      while(pt) {
-        nt = pt->next;
-	if (pt->word) free(pt->word);
-        if (pt->astr) free(pt->astr);
-        free(pt);
-	pt = nt;
-      }
-    }
-    free(tableptr);
+#ifdef UNX
+  if (fd != -1)
+  {
+	  if (buffer)
+		 munmap (buffer, length);
+	  close (fd);
   }
+  else
+#endif
+  {
+	  if (buffer)
+		free (buffer);
+  }
+  free (compat.word);
+
+  buffer = NULL;
+  fd = -1;
   tablesize = 0;
 }
 
@@ -63,145 +89,205 @@
 
 struct hentry * HashMgr::lookup(const char *word) const
 {
-    struct hentry * dp;
-    if (tableptr) {
-       dp = &tableptr[hash(word)];
-       if (dp->word == NULL) return NULL;
-       for (  ;  dp != NULL;  dp = dp->next) {
-          if (strcmp(word,dp->word) == 0) return dp;
-       }
-    }
-    return NULL;
+	const char *hash_str;
+	int hash_id = hash(word);
+	while ((hash_str = tableptr[hash_id%tablesize]) != NULL)
+	{
+		int i;
+		for (i = 0; (word[i] != '\0' && !IS_END(hash_str[i]) &&
+					 hash_str[i] != '/' && word[i] == hash_str[i]); i++)
+				;
+		if (word[i] == '\0' && (IS_END(hash_str[i]) || hash_str[i] == '/'))
+			return ((HashMgr*)this)->setup_hentry (hash_str);
+		hash_id++;
+	}
+#ifdef HASH_DEBUG
+	fprintf( stderr, "No instance of '%s' hashid 0x%x\n", word, hash(word) );
+#endif
+	return NULL;
 }
 
 
 
-// add a word to the hash table (private)
-
-int HashMgr::add_word(const char * word, int wl, const char * aff, int al)
+struct hentry *HashMgr::setup_hentry(const char *ptr)
 {
-    int i = hash(word);
-    struct hentry * dp = &tableptr[i];
-    struct hentry* hp;
-    if (dp->word == NULL) {
-       dp->wlen = wl;
-       dp->alen = al;
-       dp->word = mystrdup(word);
-       dp->astr = mystrdup(aff);
-       dp->next = NULL;
-       if ((wl) && (dp->word == NULL)) return 1;
-       if ((al) && (dp->astr == NULL)) return 1;
-    } else {
-       hp = (struct hentry *) malloc (sizeof(struct hentry));
-       if (hp == NULL) return 1;
-       hp->wlen = wl;
-       hp->alen = al;
-       hp->word = mystrdup(word);
-       hp->astr = mystrdup(aff);
-       hp->next = NULL;      
-       while (dp->next != NULL) dp=dp->next; 
-       dp->next = hp;
-       if ((wl) && (hp->word == NULL)) return 1;
-       if ((al) && (hp->astr == NULL)) return 1;
-    }
-    return 0;
-}     
+	int i;
+
+	for ( i = 0; !IS_END(ptr[i]); i++);
+
+	if ( i > compat_word_length )
+		compat.word = (char *)realloc (compat.word,
+									   (compat_word_length = i + 1) + 1);
+	compat.astr = NULL;
+    compat.wlen = i;
+	compat.alen = 0;
+
+
+	for (i = 0; !IS_END(ptr[i]); i++) {
+        compat.word[i] = ptr[i];
 
+	    if (ptr[i] == '/') {
+		    compat.wlen = i;
+		    compat.astr = compat.word + i + 1;
+            compat.word[i] = '\0';
+	    }
+	}
+    compat.word[i] = '\0';
 
+	if (compat.astr)
+       compat.alen = i - compat.wlen - 1;
+
+#ifdef HASH_DEBUG
+	fprintf( stderr, "Hit '%s' ('%s')\n", compat.word, compat.astr );
+#endif 
+
+	return &compat;
+}
 
 // walk the hash table entry by entry - null at end
 struct hentry * HashMgr::walk_hashtable(int &col, struct hentry * hp) const
 {
-  //reset to start
-  if ((col < 0) || (hp == NULL)) {
-    col = -1;
-    hp = NULL;
-  }
-
-  if (hp && hp->next != NULL) {
-    hp = hp->next;
-  } else {
-    col++;
-    hp = (col < tablesize) ? &tableptr[col] : NULL;
-    // search for next non-blank column entry
-    while (hp && (hp->word == NULL)) {
-        col ++;
-        hp = (col < tablesize) ? &tableptr[col] : NULL;
-    }
-    if (col < tablesize) return hp;
-    hp = NULL;
-    col = -1;
-  }
-  return hp;
+  if (col < 0)
+	  col = seek_newline (0);
+  size_t next_line = (size_t)seek_newline(col);
+  if (next_line >= length)
+	return NULL;
+  struct hentry *value = ((HashMgr *)this)->setup_hentry(buffer + col);
+  col = (int)next_line;
+  return value;
 }
 
 
 
 // load a munched word list and build a hash table on the fly
-
-int HashMgr::load_tables(const char * tpath)
+int
+HashMgr::slurp( int fd, size_t bytes )
 {
-  int wl, al;
-  char * ap;
-
-  // raw dictionary - munched file
-  FILE * rawdict = fopen(tpath, "r");
-  if (rawdict == NULL) return 1;
-
-  // first read the first line of file to get hash table size */
-  char ts[MAXDELEN];
-  if (! fgets(ts, MAXDELEN-1,rawdict)) return 2;
-  mychomp(ts);
-  tablesize = atoi(ts);
-  if (!tablesize) return 4; 
-  tablesize = tablesize + 5;
-  if ((tablesize %2) == 0) tablesize++;
-
-  // allocate the hash table
-  tableptr = (struct hentry *) calloc(tablesize, sizeof(struct hentry));
-  if (! tableptr) return 3;
-
-  // loop through all words on much list and add to hash
-  // table and create word and affix strings
-
-  while (fgets(ts,MAXDELEN-1,rawdict)) {
-    mychomp(ts);
-    // split each line into word and affix char strings
-    ap = strchr(ts,'/');
-    if (ap) {
-      *ap = '\0';
-      ap++;
-      al = strlen(ap);
-    } else {
-      al = 0;
-      ap = NULL;
-    }
+	size_t offset = 0;
+	ssize_t count;
+	do
+	{
+		count = read (fd, buffer + offset, bytes);
+		if (count < 0)
+		{
+			if (errno == EINTR)
+				continue;
+			else
+				return 1;
+		}
+		bytes -= count;
+		offset += count;
+	}
+	while (bytes > 0);
+    return 0;
+}
 
-    wl = strlen(ts);
+size_t
+HashMgr::seek_newline( size_t i ) const
+{
+  while (i < length && !IS_END(buffer[i])) i++;
+  while (i < length && IS_END(buffer[i])) i++;
+  return i;
+}
 
-    // add the word and its index
-    if (add_word(ts,wl,ap,al)) 
-      return 5;;
+int HashMgr::load_tables(const char * tpath)
+{
 
+#ifdef HASH_DEBUG
+  fprintf( stderr, "Load tables from '%s'\n", tpath );
+#endif
+  fd = open (tpath, O_RDONLY|_O_BINARY);
+  if (fd < 0)
+	  return 1;
+  struct stat info;
+  if (fstat(fd, &info) < 0)
+	  return 1;
+  length = info.st_size;
+#ifdef UNX
+  if (MAP_FAILED == (buffer = (char *)mmap (NULL, info.st_size, PROT_READ, MAP_SHARED, fd, 0)))
+	  ;
+  else
+#endif 
+  {
+	  buffer = new char[ length ];
+	  int failed = slurp (fd, length);
+	  close (fd);
+	  fd = -1;
+	  if (failed)
+	  {
+		delete buffer;
+		return 1;
+	  }
   }
+  if (!buffer)
+	  return 1;
+  size_t second_line = seek_newline (0);
+  char *str = new char [ second_line + 1 ];
+  memcpy (str, buffer, second_line);
+  str[second_line] = '\0';
+  tablesize = strtol(str, NULL, 0);
+  delete[] str;
+  if (tablesize <= 0)
+	  return 4;
+  long sparse = tablesize * 2;
+
+  // Bin any small prime factors
+  while (!(sparse % 2) || !(sparse % 3) || !(sparse % 5 ) ||
+		 !(sparse % 7) || !(sparse % 11) || !(sparse % 13 ))
+	  sparse++;
+
+  /* fprintf( stderr, "Switch table size to %d from %d\n", 
+		   tablesize, sparse); */
+  tablesize = sparse;
+
+  // Hash - normal fixed size sparse hash, compare until we hit a NULL
+  tableptr = (char **)calloc (sizeof(char *), tablesize );
+
+#ifdef HASH_DEBUG
+  static int total_cols = 0;
+  static int total_lookups = 0;
+#endif
+  size_t i = 0;
+  // CHECKME: '\n' on Win32 ? - cf. Win32 binary downloads I guess.
+  for (i = second_line; i < length;)
+  {
+	  int hash_id;
+	  size_t next_line = seek_newline (i);
+	  if (next_line >= length)
+		  break;
+
+	  hash_id = hash(buffer + i);
+	  int collisions = 0;
+	  while (tableptr[hash_id%tablesize]) {
+			  collisions++;
+			  hash_id++;
+	  }
+#ifdef HASH_DEBUG
+	  total_lookups++;
+	  total_cols += collisions;
+#endif
+	  tableptr[hash_id%tablesize] = buffer + i;
+	  i = next_line;
+  }  
+#ifdef HASH_DEBUG
+  fprintf (stderr, "%d collisions on %d lookups\n", total_cols, total_lookups);
+#endif
 
-  fclose(rawdict);
   return 0;
 }
 
-
-// the hash function is a simple load and rotate
-// algorithm borrowed
-
 int HashMgr::hash(const char * word) const
 {
-    long  hv = 0;
-    for (int i=0; i < 4  &&  *word != 0; i++)
-	hv = (hv << 8) | (*word++);
-    while (*word != 0) {
-      ROTATE(hv,ROTATE_LEN);
-      hv ^= (*word++);
-    }
-    return (unsigned long) hv % tablesize;
+	unsigned long hv = word[0];
+	for (int i = 1; !WORD_END_CONDITN(word[i]) && !IS_END(word[i]); i++)
+		hv = (hv << 5) - hv + word[i];
+#ifdef HASH_DEBUG
+	fprintf(stderr, "string '");
+	for (int i = 0; !WORD_END_CONDITN(word[ i ] && !IS_END(word[i])); i++)
+		fprintf( stderr, "%c", word[i] );
+		fprintf(stderr, "' 0x%x \n", (int)hv % tablesize);
+#endif
+
+	return hv % tablesize;
 }
 
--- lingucomponent/source/spellcheck/myspell/suggestmgr.hxx	2004-02-04 04:58:34.000000000 -0800
+++ lingucomponent/source/spellcheck/myspell/suggestmgr.hxx	2005-07-21 17:26:26.000000000 -0700
@@ -42,6 +42,7 @@
    int twowords(char **, const char *, int);
    int ngram(int n, char * s1, const char * s2, int uselen);
    void bubblesort( char ** rwd, int * rsc, int n);
+   int expand_rootword_affix(struct hentry *hp, char *word, int wrdlen, int thresh, int glistscore[], char *guesslist[], int *lp);
 };
 
 #endif
--- lingucomponent/source/spellcheck/myspell/suggestmgr.cxx	2004-07-13 09:00:25.000000000 -0700
+++ lingucomponent/source/spellcheck/myspell/suggestmgr.cxx	2005-07-27 15:29:58.584085080 -0700
@@ -366,40 +366,13 @@
   int i, j;
   int sc;
   int lp;
+  int lpMG;
 
   if (! pHMgr) return 0;
 
-  // exhaustively search through all root words
-  // keeping track of the MAX_ROOTS most similar root words
-  struct hentry * roots[MAX_ROOTS];
-  int scores[MAX_ROOTS];
-  for (i = 0; i < MAX_ROOTS; i++) {
-    roots[i] = NULL;
-    scores[i] = -100 * i;
-  }
-  lp = MAX_ROOTS - 1;
 
   int n = strlen(word);
 
-  struct hentry* hp = NULL;
-  int col = -1;
-  while ((hp = pHMgr->walk_hashtable(col, hp)))
-  {
-    sc = ngram(3, word, hp->word, NGRAM_LONGER_WORSE);
-    if (sc > scores[lp])
-    {
-      scores[lp] = sc;
-      roots[lp] = hp;
-      int lval = sc;
-      for (j=0; j < MAX_ROOTS; j++)
-	if (scores[j] < lval)
-	{
-	  lp = j;
-          lval = scores[j];
-	}
-    }  
-  }
-
   // find minimum threshhold for a passable suggestion
   // mangle original word three differnt ways
   // and score them to generate a minimum acceptable score
@@ -412,9 +388,14 @@
   thresh = thresh / 3;
   thresh--;
 
-  // now expand affixes on each of these root words and
-  // and use length adjusted ngram scores to select
-  // possible suggestions
+
+  struct hentry * roots[MAX_ROOTS];
+  int scores[MAX_ROOTS];
+  for (i = 0; i < MAX_ROOTS; i++) {
+    roots[i] = NULL;
+    scores[i] = -100 * i;
+  }
+
   char * guess[MAX_GUESS];
   int gscore[MAX_GUESS];
   for(i=0;i<MAX_GUESS;i++) {
@@ -422,46 +403,28 @@
      gscore[i] = -100 * i;
   }
 
-  lp = MAX_GUESS - 1;
-
-  struct guessword * glst;
-  glst = (struct guessword *) calloc(MAX_WORDS,sizeof(struct guessword));
-  if (! glst) return 0;
+  // exhaustively search through all root words
+  // keeping track of the MAX_ROOTS most similar root words
+  lp = MAX_ROOTS - 1;
 
-  for (i = 0; i < MAX_ROOTS; i++) {
+  lpMG = MAX_GUESS - 1;
 
-      if (roots[i]) {
-        struct hentry * rp = roots[i];
-	int nw = pAMgr->expand_rootword(glst, MAX_WORDS, rp->word, rp->wlen,
-                                        rp->astr, rp->alen);
-        for (int k = 0; k < nw; k++) {
-           sc = ngram(n, word, glst[k].word, NGRAM_ANY_MISMATCH);
-	   if (sc > thresh)
-	   {
-		if (sc > gscore[lp])
-		{
-			if (guess[lp]) free(guess[lp]);
-			gscore[lp] = sc;
-			guess[lp] = glst[k].word;
-			glst[k].word = NULL;
-			int lval = sc;
-			for (j=0; j < MAX_GUESS; j++)
-			{
-				if (gscore[j] < lval)
-				{
-					lp = j;
-					lval = gscore[j];
-				}
-			}
-		}
-	   }
-	   free (glst[k].word);
-	   glst[k].word = NULL;
-	   glst[k].allow = 0;
-	}
-      }
+  struct hentry* hp = NULL;
+  int col = -1;
+  while ((hp = pHMgr->walk_hashtable(col, hp))) {
+    sc = ngram(3, word, hp->word, NGRAM_LONGER_WORSE);
+    if (sc > scores[lp]) {
+      scores[lp] = sc;
+      // Get the list of guesswords for this rootword
+      if (-1 == expand_rootword_affix(hp, word, n, thresh, gscore, guess, &lpMG)) return 0;
+      int lval = sc;
+      for (j=0; j < MAX_ROOTS; j++)
+	    if (scores[j] < lval) {
+	        lp = j;
+            lval = scores[j];
+	    }
+    }  
   }
-  if (glst) free(glst);
 
   // now we are done generating guesses
   // sort in order of decreasing score and copy over
@@ -485,6 +448,50 @@
 }
 
 
+int SuggestMgr::expand_rootword_affix(struct hentry *hp, char *word, int wrdlen, int thresh, int glistscore[], char *guesslist[], int *lp)
+{
+  int sc;
+  int lval;
+
+  struct guessword * glst;
+  glst = (struct guessword *) calloc(MAX_WORDS,sizeof(struct guessword));
+  if (! glst) return -1;
+
+  // now expand affixes on each of these root words and
+  // and use length adjusted ngram scores to select
+  // possible suggestions
+  if (hp) {
+	int nw = pAMgr->expand_rootword(glst, MAX_WORDS, hp->word, hp->wlen,
+                                        hp->astr, hp->alen);
+	for (int k = 0; k < nw; k++) 
+	{
+		sc = ngram(wrdlen, word, glst[k].word, NGRAM_ANY_MISMATCH);
+		if (sc > thresh && sc > glistscore[*lp])
+		{
+			if (guesslist[*lp]) free(guesslist[*lp]);
+			glistscore[*lp] = sc;
+			guesslist[*lp] = glst[k].word;
+			glst[k].word = NULL;
+			lval = sc;
+			for (int j=0; j < MAX_GUESS; j++)
+			{
+				if (glistscore[j] < lval)
+				{
+					*lp = j;
+					lval = glistscore[j];
+				}
+			}
+		}
+		free (glst[k].word);
+		glst[k].word = NULL;
+		glst[k].allow = 0;
+	}
+  }
+  if (glst) free(glst);
+  
+  return 0;
+}
+
 
 
 // see if a candidate suggestion is spelled correctly
