ccmain/fixspace.cpp File Reference

#include "mfcpch.h"
#include <ctype.h>
#include "reject.h"
#include "statistc.h"
#include "genblob.h"
#include "control.h"
#include "fixspace.h"
#include "tessvars.h"
#include "tessbox.h"
#include "secname.h"

Go to the source code of this file.

Defines

• #define MAXSPACING   128
• #define PERFECT_WERDS   999

Functions

• float blob_noise_score (PBLOB *blob)

  Compute dimensions of largest blob ?
  

• void break_noisiest_blob_word (WERD_RES_LIST &words)

  Find the word with the blob which looks like the worst noise.
  

• BOOL8 digit_or_numeric_punct (WERD_RES *word, char ch)

  Check if ch in word is a digit or digit-compatible punctuation.
  

• void dump_words (WERD_RES_LIST &perm, INT16 score, INT16 mode, BOOL8 improved)

  Dump out the words recognized & stats.
  

• INT16 eval_word_spacing (WERD_RES_LIST &word_res_list)

  Basic measure is number of characters in contextually confirmed words = the word is done.
  

• void fix_fuzzy_space_list (WERD_RES_LIST &best_perm, ROW *row)

  Explore spaces in list.
  

• void fix_fuzzy_spaces (volatile ETEXT_DESC *monitor, INT32 word_count, PAGE_RES *page_res)

  Walk over the page, finding sequences of words joined by fuzzy spaces.
  

• void fix_noisy_space_list (WERD_RES_LIST &best_perm, ROW *row)

  Go down and fix spaces, starting with the worst/biggest.
  

• void fix_sp_fp_word (WERD_RES_IT &word_res_it, ROW *row)

  Test the current word to see if it can be split by deleting noise blobs.
  

• void fixspace_dbg (WERD_RES *word)

  Dump out info for fixing spaces functions.
  

• BOOL8 fixspace_thinks_word_done (WERD_RES *word)

  Use all the standard pass 2 conditions for mode 5 in set_done() BUT DONT REJECT IF THE WERD IS AMBIGUOUS - FOR SPACING WE DONT CARE WHETHER WE HAVE of/at on/an etc.
  

• INT16 fp_eval_word_spacing (WERD_RES_LIST &word_res_list)

  Evaluation function for fixed pitch word lists.
  

• void initialise_search (WERD_RES_LIST &src_list, WERD_RES_LIST &new_list)

  ?
  

• void match_current_words (WERD_RES_LIST &words, ROW *row)

  classifies some words (those not part of combo & not outword) using pass2?
  

• void transform_to_next_perm (WERD_RES_LIST &words)

  Checks current word list to find smallest word gap size.
  

• BOOL8 uniformly_spaced (WERD_RES *word)

  Test if spacing of word is uniform/sensible.
  

• INT16 worst_noise_blob (WERD_RES *word_res, float *worst_noise_score)

  Examine blobs and return reference to the worst one,.
  

---

Define Documentation

#define MAXSPACING   128

Max expected spacing in pix

Definition at line 66 of file fixspace.cpp.

Referenced by block_space_stat(), block_spacing_stats(), isolated_row_stats(), row_space_stat(), row_spacing_stats(), and uniformly_spaced().

#define PERFECT_WERDS   999

Definition at line 64 of file fixspace.cpp.

Referenced by eval_word_spacing(), fix_fuzzy_space_list(), and fix_noisy_space_list().

---

Function Documentation

float blob_noise_score

(

PBLOB *

blob

)

Compute dimensions of largest blob ?

Parameters:

blob

Blob

Note:

Global: fixsp_noise_score_fixing, bln_baseline_offset

Returns:

score of blob's noise

Definition at line 963 of file fixspace.cpp.

References BOX::bottom(), PBLOB::bounding_box(), BOX::height(), PBLOB::out_list(), outline_it, BOX::top(), and BOX::width().

Referenced by fp_eval_word_spacing(), and worst_noise_blob().

                                    { 
  OUTLINE_IT outline_it;
  BOX box;                       //BB of outline
  INT16 outline_count = 0;
  INT16 max_dimension;
  INT16 largest_outline_dimension = 0;

  outline_it.set_to_list (blob->out_list ());
  for (outline_it.mark_cycle_pt ();
  !outline_it.cycled_list (); outline_it.forward ()) {
    outline_count++;
    box = outline_it.data ()->bounding_box ();
    if (box.height () > box.width ())
      max_dimension = box.height ();
    else
      max_dimension = box.width ();

    if (largest_outline_dimension < max_dimension)
      largest_outline_dimension = max_dimension;
  }

  if (fixsp_noise_score_fixing) {
    if (outline_count > 5)
      largest_outline_dimension *= 2; //penalise LOTS of blobs

    box = blob->bounding_box ();

    if ((box.bottom () > bln_baseline_offset * 4) ||
      (box.top () < bln_baseline_offset / 2))
      largest_outline_dimension /= 2; //Lax blob is if high or low
  }
  return largest_outline_dimension;
}

void break_noisiest_blob_word

(

WERD_RES_LIST &

words

)

Find the word with the blob which looks like the worst noise.

Parameters:

words

Words

Returns:

none, modifies words

Break the word into two, deleting the noise blob.

Definition at line 800 of file fixspace.cpp.

References WERD_RES::best_choice, WERD::cblob_list(), WERD_RES::done, FALSE, NULL, WERD_RES::outword, WERD_RES::raw_choice, WERD::rej_cblob_list(), WERD::set_blanks(), WERD::set_flag(), W_BOL, W_EOL, WERD_RES::word, and worst_noise_blob().

Referenced by fix_noisy_space_list().

                                                    { 
  WERD_RES_IT word_it(&words); 
  WERD_RES_IT worst_word_it;
  float worst_noise_score = 9999;
  int worst_blob_index = -1;     //noisiest blb of noisiest wd
  int blob_index;                //of wds noisiest blb
  float noise_score;             //of wds noisiest blb
  WERD_RES *word_res;
  C_BLOB_IT blob_it;
  C_BLOB_IT rej_cblob_it;
  C_BLOB_LIST new_blob_list;
  C_BLOB_IT new_blob_it;
  C_BLOB_IT new_rej_cblob_it;
  WERD *new_word;
  INT16 start_of_noise_blob;
  INT16 i;

  for (word_it.mark_cycle_pt (); !word_it.cycled_list (); word_it.forward ()) {
    blob_index = worst_noise_blob (word_it.data (), &noise_score);
    if ((blob_index > -1) && (worst_noise_score > noise_score)) {
      worst_noise_score = noise_score;
      worst_blob_index = blob_index;
      worst_word_it = word_it;
    }
  }
  if (worst_blob_index < 0) {
    words.clear ();              //signal termination
    return;
  }

  /* Now split the worst_word_it */
  word_res = worst_word_it.data ();

  /* Move blobs before noise blob to a new bloblist */
  new_blob_it.set_to_list (&new_blob_list);
  blob_it.set_to_list (word_res->word->cblob_list ());
  for (i = 0; i < worst_blob_index; i++, blob_it.forward ()) {
    new_blob_it.add_after_then_move (blob_it.extract ());
  }
  start_of_noise_blob = blob_it.data ()->bounding_box ().left ();
  delete blob_it.extract ();     //throw out noise blb

  new_word = new WERD (&new_blob_list, word_res->word);
  new_word->set_flag (W_EOL, FALSE);
  word_res->word->set_flag (W_BOL, FALSE);
  word_res->word->set_blanks (1);//After break

  new_rej_cblob_it.set_to_list (new_word->rej_cblob_list ());
  rej_cblob_it.set_to_list (word_res->word->rej_cblob_list ());
  for (;
    (!rej_cblob_it.empty () &&
    (rej_cblob_it.data ()->bounding_box ().left () <
  start_of_noise_blob)); rej_cblob_it.forward ()) {
    new_rej_cblob_it.add_after_then_move (rej_cblob_it.extract ());
  }

  worst_word_it.add_before_then_move (new WERD_RES (new_word));

  word_res->done = FALSE;
  if (word_res->outword != NULL) {
    delete word_res->outword;
    delete word_res->best_choice;
    delete word_res->raw_choice;
    word_res->outword = NULL;
    word_res->best_choice = NULL;
    word_res->raw_choice = NULL;
  }
}

BOOL8 digit_or_numeric_punct	(	WERD_RES *	word,
		char	ch
	)

Check if ch in word is a digit or digit-compatible punctuation.

Parameters:

	word
	ch

Returns:

TRUE if matches

Definition at line 411 of file fixspace.cpp.

References WERD_RES::best_choice, and NUMBER_PERM.

Referenced by eval_word_spacing().

                                                      { 
  return (isdigit (ch) ||
    (fixsp_numeric_fix &&
    (word->best_choice->permuter () == NUMBER_PERM) &&
    STRING (numeric_punctuation).contains (ch)));
}

void dump_words	(	WERD_RES_LIST &	perm,
		INT16	score,
		INT16	mode,
		BOOL8	improved
	)

Dump out the words recognized & stats.

Parameters:

	perm	?
	score	score
	mode	mode
	improved	0 or 1

Note:

Global: debug_fix_space_level

Returns:

none

Definition at line 526 of file fixspace.cpp.

References STRING::string(), and tprintf().

Referenced by fix_fuzzy_space_list(), and fix_noisy_space_list().

                                                                              { 
  WERD_RES_IT word_res_it(&perm); 
  static STRING initial_str;

  if (debug_fix_space_level > 0) {
    if (mode == 1) {
      initial_str = "";
      for (word_res_it.mark_cycle_pt ();
      !word_res_it.cycled_list (); word_res_it.forward ()) {
        if (!word_res_it.data ()->part_of_combo) {
          initial_str += word_res_it.data ()->best_choice->string ();
          initial_str += ' ';
        }
      }
    }

    #ifndef SECURE_NAMES
    if (debug_fix_space_level > 1) {
      switch (mode) {
        case 1:
          tprintf ("EXTRACTED (%d): \"", score);
          break;
        case 2:
          tprintf ("TESTED (%d): \"", score);
          break;
        case 3:
          tprintf ("RETURNED (%d): \"", score);
          break;
      }

      for (word_res_it.mark_cycle_pt ();
      !word_res_it.cycled_list (); word_res_it.forward ()) {
        if (!word_res_it.data ()->part_of_combo)
          tprintf ("%s/%1d ",
            word_res_it.data ()->best_choice->string ().
            string (),
            (int) word_res_it.data ()->best_choice->permuter ());
      }
      tprintf ("\"\n");
    }
    else if (improved) {
      tprintf ("FIX SPACING \"%s\" => \"", initial_str.string ());
      for (word_res_it.mark_cycle_pt ();
      !word_res_it.cycled_list (); word_res_it.forward ()) {
        if (!word_res_it.data ()->part_of_combo)
          tprintf ("%s/%1d ",
            word_res_it.data ()->best_choice->string ().
            string (),
            (int) word_res_it.data ()->best_choice->permuter ());
      }
      tprintf ("\"\n");
    }
    #endif
  }
}

INT16 eval_word_spacing

(

WERD_RES_LIST &

word_res_list

)

Basic measure is number of characters in contextually confirmed words = the word is done.

Parameters:

word_res_list

Input word list

Note:

Global:

• conflict_set_I_l_1,
• tessedit_test_uniform_wd_spacing,
• fixsp_prefer_joined_1s,
• tessedit_prefer_joined_punct

Returns:

score

If all words are contextually confirmed the evaluation is deemed perfect.

Some fiddles are done to handle "1"s as these are VERY frequent causes of fuzzy spaces. The problem with the basic measure is that "561 63" would score the same as "56163", though given our knowledge that the space is fuzzy, and that there is a "1" next to the fuzzy space, we need to ensure that "56163" is prefered.

The solution is to NOT COUNT the score of any word which has a digit at one end and a "1Il" as the character the other side of the space.

Conversly, any character next to a "1" within a word is counted as a positive score. Thus "561 63" would score 4 (3 chars in a numeric word plus 1 side of the "1" joined). "56163" would score 7 - all chars in a numeric word + 2 sides of a "1" joined.

The joined 1 rule is applied to any word REGARDLESS of contextual confirmation. Thus "PS7a71 3/7a" scores 1 (neither word is contexutally confirmed. The only score is from the joined 1. "PS7a713/7a" scores 2.

Can we add the prev word score and potentially count this word? Yes IF it didnt end in a 1 when the first char of this word is a digit AND it didnt end in a digit when the first char of this word is a 1

Definition at line 285 of file fixspace.cpp.

References WERD_RES::best_choice, STRING::contains(), cprintf(), digit_or_numeric_punct(), FALSE, fixspace_thinks_word_done(), REJMAP::length(), NUMBER_PERM, PERFECT_WERDS, WERD_RES::reject_map, WERD_RES::tess_failed, TRUE, uniformly_spaced(), and word_count.

Referenced by fix_fuzzy_space_list(), and fp_eval_word_spacing().

                                                      { 
  WERD_RES_IT word_res_it(&word_res_list); 
  INT16 total_score = 0;
  INT16 word_count = 0;
  INT16 done_word_count = 0;
  INT16 word_len;
  INT16 i;
  WERD_RES *word;                //current word
  INT16 prev_word_score = 0;
  BOOL8 prev_word_done = FALSE;
  BOOL8 prev_char_1 = FALSE;     //prev ch a "1/I/l"?
  BOOL8 prev_char_digit = FALSE; //prev ch 2..9 or 0
  BOOL8 current_char_1 = FALSE;
  BOOL8 current_word_ok_so_far;
  STRING punct_chars = "!\"`',.:;";
  BOOL8 prev_char_punct = FALSE;
  BOOL8 current_char_punct = FALSE;
  BOOL8 word_done = FALSE;

#ifdef TEXT_VERBOSE
  // gets a 'z', see ccmain/tesseractmain.dox
  cprintf("z");
#endif
  do {
    word = word_res_it.data ();
    word_done = fixspace_thinks_word_done (word);
    word_count++;
    if (word->tess_failed) {
      total_score += prev_word_score;
      if (prev_word_done)
        done_word_count++;
      prev_word_score = 0;
      prev_char_1 = FALSE;
      prev_char_digit = FALSE;
      prev_word_done = FALSE;
    }
    else {
      word_len = word->reject_map.length ();
      current_word_ok_so_far = FALSE;
      if (!((prev_char_1 &&
        digit_or_numeric_punct (word,
        word->best_choice->string ()[0])) ||
        (prev_char_digit &&
        ((word_done &&
        (word->best_choice->string ()[0] == '1')) ||
        (!word_done &&
        STRING (conflict_set_I_l_1).contains (word->best_choice->
           string ()[0])))))) {
        total_score += prev_word_score;
        if (prev_word_done)
          done_word_count++;
        current_word_ok_so_far = word_done;
      }

      if ((current_word_ok_so_far) &&
        (!tessedit_test_uniform_wd_spacing ||
        ((word->best_choice->permuter ()==NUMBER_PERM)||uniformly_spaced (word)))) {
        prev_word_done = TRUE;
        prev_word_score = word_len;
      }
      else {
        prev_word_done = FALSE;
        prev_word_score = 0;
      }

      if (fixsp_prefer_joined_1s) {
        /* Add 1 to total score for every joined 1 regardless of context and rejtn */

        for (i = 0, prev_char_1 = FALSE; i < word_len; i++) {
          current_char_1 = word->best_choice->string ()[i] == '1';
          if (prev_char_1 || (current_char_1 && (i > 0)))
            total_score++;
          prev_char_1 = current_char_1;
        }
      }

      /* Add 1 to total score for every joined punctuation regardless of context
        and rejtn */
      if (tessedit_prefer_joined_punct) {
        for (i = 0, prev_char_punct = FALSE; i < word_len; i++) {
          current_char_punct =
            punct_chars.contains (word->best_choice->string ()[i]);
          if (prev_char_punct || (current_char_punct && (i > 0)))
            total_score++;
          prev_char_punct = current_char_punct;
        }
      }
      prev_char_digit = digit_or_numeric_punct (word,
        word->best_choice->
        string ()[word_len - 1]);
      prev_char_1 =
        ((word_done
        && (word->best_choice->string ()[word_len - 1] == '1'))
        || (!word_done
        && STRING (conflict_set_I_l_1).contains (word->best_choice->
        string ()[word_len -
        1])));
    }
    /* Find next word */
    do
    word_res_it.forward ();
    while (word_res_it.data ()->part_of_combo);
  }
  while (!word_res_it.at_first ());
  total_score += prev_word_score;
  if (prev_word_done)
    done_word_count++;
  if (done_word_count == word_count)
    return PERFECT_WERDS;
  else
    return total_score;
}

void fix_fuzzy_space_list	(	WERD_RES_LIST &	best_perm,
		ROW *	row
	)

Explore spaces in list.

Parameters:

	best_perm	for eval_word_spacing()
	row	Row

Returns:

none

Definition at line 178 of file fixspace.cpp.

References dump_words(), eval_word_spacing(), FALSE, initialise_search(), match_current_words(), PERFECT_WERDS, transform_to_next_perm(), and TRUE.

Referenced by fix_fuzzy_spaces().

                                    {
  INT16 best_score;
  WERD_RES_LIST current_perm;
  INT16 current_score;
  BOOL8 improved = FALSE;


  best_score = eval_word_spacing (best_perm); //default score

  dump_words (best_perm, best_score, 1, improved);

  if (best_score != PERFECT_WERDS)
    initialise_search(best_perm, current_perm); 

  while ((best_score != PERFECT_WERDS) && !current_perm.empty ()) {
    match_current_words(current_perm, row); 
    current_score = eval_word_spacing (current_perm);
    dump_words (current_perm, current_score, 2, improved);
    if (current_score > best_score) {
      best_perm.clear ();
      best_perm.deep_copy (&current_perm);
      best_score = current_score;
      improved = TRUE;
    }
    if (current_score < PERFECT_WERDS)
      transform_to_next_perm(current_perm); 
  }
  dump_words (best_perm, best_score, 3, improved);
}

void fix_fuzzy_spaces	(	volatile ETEXT_DESC *	monitor,
		INT32	word_count,
		PAGE_RES *	page_res
	)

Walk over the page, finding sequences of words joined by fuzzy spaces.

Parameters:

	monitor	progress monitor
	word_count	count of words in doc
	page_res

Returns:

none

Extract them as a sublist, process the sublist to find the optimal arrangement of spaces then replace the sublist in the ROW_RES.

Definition at line 79 of file fixspace.cpp.

References check_debug_pt(), WERD_RES::combination, fix_fuzzy_space_list(), fix_sp_fp_word(), WERD::gblob_list(), NULL, TRUE, W_FUZZY_NON, W_FUZZY_SP, and WERD_RES::word.

Referenced by recog_all_words().

                                          {
  BLOCK_RES_IT block_res_it;     //iterators
  ROW_RES_IT row_res_it;
  WERD_RES_IT word_res_it_from;
  WERD_RES_IT word_res_it_to;
  WERD_RES *word_res;
  WERD_RES_LIST fuzzy_space_words;
  INT16 new_length;
  BOOL8 prevent_null_wd_fixsp;   //DONT process blobless wds
  INT32 word_index;              //current word

  block_res_it.set_to_list (&page_res->block_res_list);
  word_index = 0;
  for (block_res_it.mark_cycle_pt ();
  !block_res_it.cycled_list (); block_res_it.forward ()) {
    row_res_it.set_to_list (&block_res_it.data ()->row_res_list);
    for (row_res_it.mark_cycle_pt ();
    !row_res_it.cycled_list (); row_res_it.forward ()) {
      word_res_it_from.set_to_list (&row_res_it.data ()->word_res_list);
      while (!word_res_it_from.at_last ()) {
        word_res = word_res_it_from.data ();
        while (!word_res_it_from.at_last () &&
          !(word_res->combination ||
          word_res_it_from.data_relative (1)->
          word->flag (W_FUZZY_NON) ||
          word_res_it_from.data_relative (1)->
        word->flag (W_FUZZY_SP))) {
          fix_sp_fp_word (word_res_it_from, row_res_it.data ()->row);
          word_res = word_res_it_from.forward ();
          word_index++;
          if (monitor != NULL) {
            monitor->ocr_alive = TRUE;
            monitor->progress = 90 + 5 * word_index / word_count;
          }
        }

        if (!word_res_it_from.at_last ()) {
          word_res_it_to = word_res_it_from;
          prevent_null_wd_fixsp =
            word_res->word->gblob_list ()->empty ();
          if (check_debug_pt (word_res, 60))
            debug_fix_space_level.set_value (10);
          word_res_it_to.forward ();
          word_index++;
          if (monitor != NULL) {
            monitor->ocr_alive = TRUE;
            monitor->progress = 90 + 5 * word_index / word_count;
          }
          while (!word_res_it_to.at_last () &&
            (word_res_it_to.data_relative (1)->
            word->flag (W_FUZZY_NON) ||
            word_res_it_to.data_relative (1)->
          word->flag (W_FUZZY_SP))) {
            if (check_debug_pt (word_res, 60))
              debug_fix_space_level.set_value (10);
            if (word_res->word->gblob_list ()->empty ())
              prevent_null_wd_fixsp = TRUE;
            word_res = word_res_it_to.forward ();
          }
          if (check_debug_pt (word_res, 60))
            debug_fix_space_level.set_value (10);
          if (word_res->word->gblob_list ()->empty ())
            prevent_null_wd_fixsp = TRUE;
          if (prevent_null_wd_fixsp)
            word_res_it_from = word_res_it_to;
          else {
            fuzzy_space_words.assign_to_sublist (&word_res_it_from,
              &word_res_it_to);
            fix_fuzzy_space_list (fuzzy_space_words,
              row_res_it.data ()->row);
            new_length = fuzzy_space_words.length ();
            word_res_it_from.add_list_before (&fuzzy_space_words);
            for (;
              (!word_res_it_from.at_last () &&
            (new_length > 0)); new_length--) {
              word_res_it_from.forward ();
            }
          }
          if (test_pt)
            debug_fix_space_level.set_value (0);
        }
        fix_sp_fp_word (word_res_it_from, row_res_it.data ()->row);
        //Last word in row
      }
    }
  }
}

void fix_noisy_space_list	(	WERD_RES_LIST &	best_perm,
		ROW *	row
	)

Go down and fix spaces, starting with the worst/biggest.

Parameters:

	best_perm	to get scores
	row	Row

Returns:

void, modifies best_perm

Definition at line 751 of file fixspace.cpp.

References break_noisiest_blob_word(), WERD_RES::combination, dump_words(), FALSE, fp_eval_word_spacing(), match_current_words(), PERFECT_WERDS, and TRUE.

Referenced by fix_sp_fp_word().

                                                              { 
  INT16 best_score;
  WERD_RES_IT best_perm_it(&best_perm); 
  WERD_RES_LIST current_perm;
  WERD_RES_IT current_perm_it(&current_perm); 
  WERD_RES *old_word_res;
  WERD_RES *new_word_res;
  INT16 current_score;
  BOOL8 improved = FALSE;

  best_score = fp_eval_word_spacing (best_perm); //default score

  dump_words (best_perm, best_score, 1, improved);

  new_word_res = new WERD_RES;
  old_word_res = best_perm_it.data ();
  old_word_res->combination = TRUE; //Kludge to force deep copy
  *new_word_res = *old_word_res; //deep copy
  old_word_res->combination = FALSE; //Undo kludge
  new_word_res->combination = FALSE; //Undo kludge
  current_perm_it.add_to_end (new_word_res);

  break_noisiest_blob_word(current_perm); 

  while ((best_score != PERFECT_WERDS) && !current_perm.empty ()) {
    match_current_words(current_perm, row); 
    current_score = fp_eval_word_spacing (current_perm);
    dump_words (current_perm, current_score, 2, improved);
    if (current_score > best_score) {
      best_perm.clear ();
      best_perm.deep_copy (&current_perm);
      best_score = current_score;
      improved = TRUE;
    }
    if (current_score < PERFECT_WERDS)
      break_noisiest_blob_word(current_perm); 
  }
  dump_words (best_perm, best_score, 3, improved);
}

void fix_sp_fp_word	(	WERD_RES_IT &	word_res_it,
		ROW *	row
	)

Test the current word to see if it can be split by deleting noise blobs.

Parameters:

	word_res_it	Word results?
	row

Note:

Global: fixsp_check_for_fp_noise_space debug_fix_space_level

Returns:

none

If so, do the buisiness. Return with the iterator pointing to the same place if the word is unchanged, or the last of the replacement words.

Definition at line 708 of file fixspace.cpp.

References WERD_RES::best_choice, WERD_RES::combination, FALSE, fix_noisy_space_list(), WERD::flag(), gblob_sort_list(), WERD_RES::part_of_combo, WERD::rej_cblob_list(), tprintf(), W_DONT_CHOP, W_REP_CHAR, WERD_RES::word, and worst_noise_blob().

Referenced by fix_fuzzy_spaces().

                                                        { 
  WERD_RES *word_res;
  WERD_RES_LIST sub_word_list;
  WERD_RES_IT sub_word_list_it(&sub_word_list); 
  INT16 blob_index;
  INT16 new_length;
  float junk;

  word_res = word_res_it.data ();
  if (!fixsp_check_for_fp_noise_space ||
    word_res->word->flag (W_REP_CHAR) ||
    word_res->combination ||
    word_res->part_of_combo || !word_res->word->flag (W_DONT_CHOP))
    return;

  blob_index = worst_noise_blob (word_res, &junk);
  if (blob_index < 0)
    return;

  #ifndef SECURE_NAMES
  if (debug_fix_space_level > 1) {
    tprintf ("FP fixspace working on \"%s\"\n",
      word_res->best_choice->string ().string ());
  }
  #endif
  gblob_sort_list ((PBLOB_LIST *) word_res->word->rej_cblob_list (), FALSE);
  sub_word_list_it.add_after_stay_put (word_res_it.extract ());
  fix_noisy_space_list(sub_word_list, row); 
  new_length = sub_word_list.length ();
  word_res_it.add_list_before (&sub_word_list);
  for (; (!word_res_it.at_last () && (new_length > 1)); new_length--) {
    word_res_it.forward ();
  }
}

void fixspace_dbg

(

WERD_RES *

word

)

Dump out info for fixing spaces functions.

Parameters:

word

Word in question

Note:

Global: debug_fp

Returns:

none

Definition at line 1006 of file fixspace.cpp.

References WERD_RES::best_choice, WERD::bounding_box(), debug_fp, WERD_RES::done, FALSE, REJMAP::full_print(), WERD::gblob_list(), WERD_RES::outword, BOX::print(), REJMAP::print(), WERD::rej_blob_list(), WERD_RES::reject_map, WERD_RES::tess_accepted, tprintf(), and WERD_RES::word.

                                  { 
  BOX box = word->word->bounding_box ();
  BOOL8 show_map_detail = FALSE;
  INT16 i;

  box.print ();
  #ifndef SECURE_NAMES
  tprintf (" \"%s\" ", word->best_choice->string ().string ());
  tprintf ("Blob count: %d (word); %d/%d (outword)\n",
    word->word->gblob_list ()->length (),
    word->outword->gblob_list ()->length (),
    word->outword->rej_blob_list ()->length ());
  word->reject_map.print (debug_fp);
  tprintf ("\n");
  if (show_map_detail) {
    tprintf ("\"%s\"\n", word->best_choice->string ().string ());
    for (i = 0; word->best_choice->string ()[i] != '\0'; i++) {
      tprintf ("**** \"%c\" ****\n", word->best_choice->string ()[i]);
      word->reject_map[i].full_print (debug_fp);
    }
  }

  tprintf ("Tess Accepted: %s\n", word->tess_accepted ? "TRUE" : "FALSE");
  tprintf ("Done flag: %s\n\n", word->done ? "TRUE" : "FALSE");
  #endif
}

BOOL8 fixspace_thinks_word_done

(

WERD_RES *

word

)

Use all the standard pass 2 conditions for mode 5 in set_done() BUT DONT REJECT IF THE WERD IS AMBIGUOUS - FOR SPACING WE DONT CARE WHETHER WE HAVE of/at on/an etc.

Parameters:

word

Word working on

Returns:

TRUE if might be word

Definition at line 675 of file fixspace.cpp.

References WERD_RES::best_choice, WERD_RES::done, FALSE, FREQ_DAWG_PERM, NULL, NUMBER_PERM, REJMAP::reject_count(), WERD_RES::reject_map, SYSTEM_DAWG_PERM, WERD_RES::tess_accepted, TRUE, and USER_DAWG_PERM.

Referenced by eval_word_spacing().

                                                { 
  if (word->done)
    return TRUE;

  if ((fixsp_done_mode > 0) &&
    (word->tess_accepted ||
    ((fixsp_done_mode == 2) &&
    (word->reject_map.reject_count () == 0)) ||
    (fixsp_done_mode == 3)) &&
    (strchr (word->best_choice->string ().string (), ' ') == NULL) &&
    ((word->best_choice->permuter () == SYSTEM_DAWG_PERM) ||
    (word->best_choice->permuter () == FREQ_DAWG_PERM) ||
    (word->best_choice->permuter () == USER_DAWG_PERM) ||
    (word->best_choice->permuter () == NUMBER_PERM)))
    return TRUE;
  else
    return FALSE;
}

INT16 fp_eval_word_spacing

(

WERD_RES_LIST &

word_res_list

)

Evaluation function for fixed pitch word lists.

Parameters:

word_res_list

list of word results

Note:

Global: bln_x_height, fixsp_small_outlines_size

Returns:

score

Basically, count the number of "nice" characters - those which are in tess acceptable words or in dict words and are not rejected. Penalise any potential noise chars

Definition at line 1045 of file fixspace.cpp.

References WERD_RES::best_choice, WERD::blob_list(), blob_noise_score(), WERD_RES::done, eval_word_spacing(), FREQ_DAWG_PERM, REJMAP::length(), WERD_RES::outword, WERD_RES::reject_map, safe_dict_word(), SYSTEM_DAWG_PERM, WERD_RES::tess_accepted, and USER_DAWG_PERM.

Referenced by fix_noisy_space_list().

                                                         { 
  WERD_RES_IT word_it(&word_res_list); 
  WERD_RES *word;
  PBLOB_IT blob_it;
  INT16 word_length;
  INT16 score = 0;
  INT16 i;
  const char *chs;
  float small_limit = bln_x_height * fixsp_small_outlines_size;

  if (!fixsp_fp_eval)
    return (eval_word_spacing (word_res_list));

  for (word_it.mark_cycle_pt (); !word_it.cycled_list (); word_it.forward ()) {
    word = word_it.data ();
    word_length = word->reject_map.length ();
    chs = word->best_choice->string ().string ();
    if ((word->done ||
      word->tess_accepted) ||
      (word->best_choice->permuter () == SYSTEM_DAWG_PERM) ||
      (word->best_choice->permuter () == FREQ_DAWG_PERM) ||
      (word->best_choice->permuter () == USER_DAWG_PERM) ||
    (safe_dict_word (chs) > 0)) {
      blob_it.set_to_list (word->outword->blob_list ());
      for (i = 0; i < word_length; i++, blob_it.forward ()) {
        if ((chs[i] == ' ') ||
          (blob_noise_score (blob_it.data ()) < small_limit))
          score -= 1;            //penalise possibly erroneous non-space

        else if (word->reject_map[i].accepted ())
          score++;
      }
    }
  }
  if (score < 0)
    score = 0;
  return score;
}

void initialise_search	(	WERD_RES_LIST &	src_list,
		WERD_RES_LIST &	new_list
	)

?

Parameters:

	src_list	source list
	new_list	new list

Returns:

none, new_list gets modified

Definition at line 218 of file fixspace.cpp.

References WERD_RES::combination, FALSE, and WERD_RES::part_of_combo.

Referenced by fix_fuzzy_space_list().

                                                                         { 
  WERD_RES_IT src_it(&src_list); 
  WERD_RES_IT new_it(&new_list); 
  WERD_RES *src_wd;
  WERD_RES *new_wd;

  for (src_it.mark_cycle_pt (); !src_it.cycled_list (); src_it.forward ()) {
    src_wd = src_it.data ();
    if (!src_wd->combination) {
      new_wd = new WERD_RES (*src_wd);
      new_wd->combination = FALSE;
      new_wd->part_of_combo = FALSE;
      new_it.add_after_then_move (new_wd);
    }
  }
}

void match_current_words	(	WERD_RES_LIST &	words,
		ROW *	row
	)

classifies some words (those not part of combo & not outword) using pass2?

Parameters:

	words	list of words
	row	Row

Note:

Global:

Returns:

Definition at line 244 of file fixspace.cpp.

References classify_word_pass2(), NULL, WERD_RES::outword, and WERD_RES::part_of_combo.

Referenced by fix_fuzzy_space_list(), and fix_noisy_space_list().

                                                         { 
  WERD_RES_IT word_it(&words); 
  WERD_RES *word;

  for (word_it.mark_cycle_pt (); !word_it.cycled_list (); word_it.forward ()) {
    word = word_it.data ();
    if ((!word->part_of_combo) && (word->outword == NULL))
      classify_word_pass2(word, row); 
  }
}

void transform_to_next_perm

(

WERD_RES_LIST &

words

)

Checks current word list to find smallest word gap size.

Parameters:

words

list of words

Returns:

none

Then walks the word list closing any gaps of this size by either inserted new combination words, or extending existing ones.

The routine COULD be limited to stop it building words longer than N blobs.

If there are no more gaps then it DELETES the entire list and returns the empty list to cause termination.

Definition at line 433 of file fixspace.cpp.

References WERD_RES::best_choice, WERD::bounding_box(), WERD_RES::combination, WERD_RES::copy_on(), WERD_RES::done, FALSE, WERD::flag(), WERD::join_on(), BOX::left(), MAX_INT16, NULL, WERD_RES::outword, WERD_RES::part_of_combo, WERD_RES::raw_choice, BOX::right(), WERD::set_flag(), TRUE, W_EOL, and WERD_RES::word.

Referenced by fix_fuzzy_space_list().

                                                  { 
  WERD_RES_IT word_it(&words); 
  WERD_RES_IT prev_word_it(&words); 
  WERD_RES *word;
  WERD_RES *prev_word;
  WERD_RES *combo;
  WERD *copy_word;
  INT16 prev_right = -1;
  BOX box;
  INT16 gap;
  INT16 min_gap = MAX_INT16;

  for (word_it.mark_cycle_pt (); !word_it.cycled_list (); word_it.forward ()) {
    word = word_it.data ();
    if (!word->part_of_combo) {
      box = word->word->bounding_box ();
      if (prev_right >= 0) {
        gap = box.left () - prev_right;
        if (gap < min_gap)
          min_gap = gap;
      }
      prev_right = box.right ();
    }
  }
  if (min_gap < MAX_INT16) {
    prev_right = -1;  //back to start
    word_it.set_to_list (&words);
    for (;  //cant use cycle pt due to inserted combos at start of list
    (prev_right < 0) || !word_it.at_first (); word_it.forward ()) {
      word = word_it.data ();
      if (!word->part_of_combo) {
        box = word->word->bounding_box ();
        if (prev_right >= 0) {
          gap = box.left () - prev_right;
          if (gap <= min_gap) {
            prev_word = prev_word_it.data ();
            if (prev_word->combination)
              combo = prev_word;
            else {
              /* Make a new combination and insert before the first word being joined */
              copy_word = new WERD;
              *copy_word = *(prev_word->word);
              //deep copy
              combo = new WERD_RES (copy_word);
              combo->combination = TRUE;
              prev_word->part_of_combo = TRUE;
              prev_word_it.add_before_then_move (combo);
            }
            combo->word->set_flag (W_EOL, word->word->flag (W_EOL));
            if (word->combination) {
              combo->word->join_on (word->word);
              //Move blbs to combo
              //old combo no longer needed
              delete word_it.extract ();
            }
            else {
              //Copy current wd to combo
              combo->copy_on (word);
              word->part_of_combo = TRUE;
            }
            combo->done = FALSE;
            if (combo->outword != NULL) {
              delete combo->outword;
              delete combo->best_choice;
              delete combo->raw_choice;
              combo->outword = NULL;
              combo->best_choice = NULL;
              combo->raw_choice = NULL;
            }
          }
          else
              //catch up
              prev_word_it = word_it;
        }
        prev_right = box.right ();
      }
    }
  }
  else
    words.clear ();              //signal termination
}

BOOL8 uniformly_spaced

(

WERD_RES *

word

)

Test if spacing of word is uniform/sensible.

Parameters:

word

Word of interest

Note:

Global: bln_x_height

Returns:

TRUE if uniform

Return true if one of the following are true:

• All inter-char gaps are the same width
• The largest gap is no larger than twice the mean/median of the others
• The largest gap is < 64/5 = 13 and all others are <= 0

Note:

REMEMBER - WE'RE NOW WORKING WITH A BLN WERD !!!

Definition at line 597 of file fixspace.cpp.

References STATS::add(), WERD_RES::best_choice, WERD::blob_list(), STRING::contains(), WERD_RES::denorm, STATS::get_total(), ROW::kern(), BOX::left(), MAX_INT16, MAXSPACING, STATS::mean(), STATS::median(), WERD_RES::outword, BOX::right(), DENORM::row(), ROW::space(), tprintf(), and ROW::x_height().

Referenced by eval_word_spacing().

                                       {
  PBLOB_IT blob_it;
  BOX box;
  INT16 prev_right = -MAX_INT16;
  INT16 gap;
  INT16 max_gap = -MAX_INT16;
  INT16 max_gap_count = 0;
  STATS gap_stats (0, MAXSPACING);
  BOOL8 result;
  const ROW *row = word->denorm.row ();
  float max_non_space;
  float normalised_max_nonspace;
  INT16 i = 0;
  STRING punct_chars = "\"`',.:;";

  blob_it.set_to_list (word->outword->blob_list ());

  for (blob_it.mark_cycle_pt (); !blob_it.cycled_list (); blob_it.forward ()) {
    box = blob_it.data ()->bounding_box ();
    if ((prev_right > -MAX_INT16) &&
      (!fixsp_ignore_punct ||
      (!punct_chars.contains (word->best_choice->string ()[i - 1]) &&
    !punct_chars.contains (word->best_choice->string ()[i])))) {
      gap = box.left () - prev_right;
      if (gap < max_gap)
        gap_stats.add (gap, 1);
      else if (gap == max_gap)
        max_gap_count++;
      else {
        if (max_gap_count > 0)
          gap_stats.add (max_gap, max_gap_count);
        max_gap = gap;
        max_gap_count = 1;
      }
    }
    prev_right = box.right ();
    i++;
  }

  max_non_space = (row->space () + 3 * row->kern ()) / 4;
  normalised_max_nonspace = max_non_space * bln_x_height / row->x_height ();

  result = ((gap_stats.get_total () == 0) ||
    (max_gap <= normalised_max_nonspace) ||
    ((gap_stats.get_total () > 2) &&
    (max_gap <= 2 * gap_stats.median ())) ||
    ((gap_stats.get_total () <= 2) &&
    (max_gap <= 2 * gap_stats.mean ())));
  #ifndef SECURE_NAMES
  if ((debug_fix_space_level > 1)) {
    if (result)
      tprintf
        ("ACCEPT SPACING FOR: \"%s\" norm_maxnon = %f max=%d maxcount=%d total=%d mean=%f median=%f\n",
        word->best_choice->string ().string (), normalised_max_nonspace,
        max_gap, max_gap_count, gap_stats.get_total (), gap_stats.mean (),
        gap_stats.median ());
    else
      tprintf
        ("REJECT SPACING FOR: \"%s\" norm_maxnon = %f max=%d maxcount=%d total=%d mean=%f median=%f\n",
        word->best_choice->string ().string (), normalised_max_nonspace,
        max_gap, max_gap_count, gap_stats.get_total (), gap_stats.mean (),
        gap_stats.median ());
  }
  #endif

  return result;
}

INT16 worst_noise_blob	(	WERD_RES *	word_res,
		float *	worst_noise_score
	)

Examine blobs and return reference to the worst one,.

Parameters:

	word_res	Word results
	worst_noise_score	Modified by this function

Note:

Global: bln_x_height, fixsp_small_outlines_size, fixsp_non_noise_limit, debug_fix_space_level

Returns:

number of worst blob

Definition at line 880 of file fixspace.cpp.

References ASSERT_HOST, WERD_RES::best_choice, blob_count, WERD::blob_list(), blob_noise_score(), WERD_RES::outword, WERD_RES::reject_map, tprintf(), and worst_noise_blob().

Referenced by break_noisiest_blob_word(), fix_sp_fp_word(), and worst_noise_blob().

                                                                     { 
  PBLOB_IT blob_it;
  INT16 blob_count;
  float noise_score[512];
  int i;
  int min_noise_blob;            //1st contender
  int max_noise_blob;            //last contender
  int non_noise_count;
  int worst_noise_blob;          //Worst blob
  float small_limit = bln_x_height * fixsp_small_outlines_size;
  float non_noise_limit = bln_x_height * 0.8;

  blob_it.set_to_list (word_res->outword->blob_list ());
  //normalised
  blob_count = blob_it.length ();
  ASSERT_HOST (blob_count <= 512);
  if (blob_count < 5)
    return -1;                   //too short to split
  /* Get the noise scores for all blobs */

  #ifndef SECURE_NAMES
  if (debug_fix_space_level > 5)
    tprintf ("FP fixspace Noise metrics for \"%s\": ",
      word_res->best_choice->string ().string ());
  #endif

  for (i = 0; i < blob_count; i++, blob_it.forward ()) {
    if (word_res->reject_map[i].accepted ())
      noise_score[i] = non_noise_limit;
    else
      noise_score[i] = blob_noise_score (blob_it.data ());

    if (debug_fix_space_level > 5)
      tprintf ("%1.1f ", noise_score[i]);
  }
  if (debug_fix_space_level > 5)
    tprintf ("\n");

  /* Now find the worst one which is far enough away from the end of the word */

  non_noise_count = 0;
  for (i = 0;
  (i < blob_count) && (non_noise_count < fixsp_non_noise_limit); i++) {
    if (noise_score[i] >= non_noise_limit)
      non_noise_count++;
  }
  if (non_noise_count < fixsp_non_noise_limit)
    return -1;
  min_noise_blob = i;

  non_noise_count = 0;
  for (i = blob_count - 1;
  (i >= 0) && (non_noise_count < fixsp_non_noise_limit); i--) {
    if (noise_score[i] >= non_noise_limit)
      non_noise_count++;
  }
  if (non_noise_count < fixsp_non_noise_limit)
    return -1;
  max_noise_blob = i;

  if (min_noise_blob > max_noise_blob)
    return -1;

  *worst_noise_score = small_limit;
  worst_noise_blob = -1;
  for (i = min_noise_blob; i <= max_noise_blob; i++) {
    if (noise_score[i] < *worst_noise_score) {
      worst_noise_blob = i;
      *worst_noise_score = noise_score[i];
    }
  }
  return worst_noise_blob;
}

---