# File raramorph.rb, line 102
  def self.analyze_token(token ,  output_buckwalter) #STring  , Boolean , REturn Boolean
     #TO DO SET UP THE PRINT STREAM
     token.force_encoding "UTF-8"
     @stream.puts "Processing token : " + "\t" + token
     #TODO : check accuracy
     #ignored \u0688 : ARABIC LETTER DDAL
     #ignored \u06A9 : ARABIC LETTER KEHEH
     #ignored \u0691 : ARABIC LETTER RREH
     #ignored \u06BA : ARABIC LETTER NOON GHUNNA
     #ignored \u06BE : ARABIC LETTER HEH DOACHASHMEE
     #ignored \u06C1 : ARABIC LETTER HEH GOAL
     #ignored \u0640 : ARABIC TATWEEL
     #ignored \u06D2 : ARABIC LETTER YEH BARREE
     unless(token.match(/([\u067E\u0686\u0698\u06AF\u0621-\u063A\u0641-\u0652])+/))
         token.strip!
         # tokenize it on white space
          sub_tokens = token.split(@@space_regex)
          sub_tokens.each{|sub_token|
            unless  sub_token.strip == ""  
              @not_arabic_tokens_counter+=1
              @output_stream != nil ? @stream.puts("Non-Arabic : " + sub_token) : puts("Non-Arabic : " + sub_token)
            end 
          }
          return false
     else
       has_solutions = false
       @not_arabic_tokens_counter+=1
       
       translitered = ArabicLatinTranslator.translate(token)
       @output_stream != nil ? @stream.puts("Transliteration : " + "\t" + translitered) : puts("Transliteration : " + "\t" + translitered)

      if @found.has_key?(translitered)        #Already processed : previously found
        @output_stream != nil  && @verbose ? @stream.puts("Token already processed.") : puts("Token already processed.")          
        #increase reference counter
        @found[translitered]+=1
        has_solutions = true
       elsif @not_found.has_key?(translitered) #Already processed : previously not found  
        @output_stream != nil  && @verbose ? @stream.puts("Token already processed without solution.") : puts("Token already processed without solution.")          
        @not_found[translitered]+=1         #increase reference counter
        has_solutions = false
       else
        @output_stream != nil  && @verbose ? @stream.puts("Token not yet processed.") : puts("Token not yet processed.")          

        if (feed_word_solutions(translitered)) #CHANGED  #word has solutions...
          #mark word as found
          raise "There is already a key for " + translitered + " in found" if @found.has_key?(translitered)
          @output_stream != nil  && @verbose ? @stream.puts("Token has direct solutions.") : puts("Token has direct solutions.")          
          #set reference counter to 1
          @found[translitered] = 1
          has_solutions = true
        else #word has no direct solution
           if(feed_alternative_spellings(translitered))
             alternatives_give_solutions = false
             
             alternatives = @@sol.get_alternative_spellings(translitered)
             alternatives.each{|alternative|
              alternatives_give_solutions =  (alternatives_give_solutions || feed_word_solutions(alternative))
             }
             if(alternatives_give_solutions)
               #consistency check
               raise "There is already a key for " + translitered + " in found" if @found.has_key?(translitered)
               @output_stream != nil  && @verbose ? @stream.puts("Token's alternative spellings have solutions.") : puts("Token's alternative spellings have solutions.")
               #mark word as found set reference counter to 1
               @found[translitered] = 1
               has_solutions = true
             else
               #consistency check
               raise "There is already a key for " + translitered + " in notFound" if @not_found.has_key?(translitered)
               @output_stream != nil  && @verbose ? @stream.puts("Token's alternative spellings have no solution.") : puts("Token's alternative spellings have no solution.")
               @not_found[translitered]=1 
               has_solutions = false  
           end
         else
           #there are no alternative
           raise "There is already a key for " + translitered + " in notFound" if @not_found.has_key?(translitered)
           @output_stream != nil  && @verbose ? @stream.puts("Token has no solution and no alternative spellings.") : puts("Token has no solution and no alternative spellings.")
           #mark word as not found and set reference counter to 1
           @not_found[translitered]=1 
           has_solutions = false  
         end
        end
      end
      
      
        #output solutions : TODO consider XML output
        if @output_stream != nil
          if @found.has_key?(translitered)
            if @@sol.has_solutions(translitered)
              @@sol.get_solutions(translitered).each{|solution| @stream.puts "#{output_buckwalter ? solution.to_s : solution.to_arabized_string}"}
            end
            if @@sol.has_alternative_spellings(translitered) 
              @output_stream != nil  && @verbose ? @stream.puts("No direct solution") : puts("No direct solution")   
              @@sol.get_alternative_spellings(translitered).each{|alternative| 
                 @output_stream != nil  && @verbose ? @stream.puts("Considering alternative spelling :" + "\t" + alternative) : puts("Considering alternative spelling :" + "\t" + alternative)   
                 if @@sol.has_solutions(alternative)
                   @@sol.get_solutions(alternative).each{|solution| @stream.puts "#{output_buckwalter ? solution.to_s : solution.to_arabized_string}"}
                 end
              }
            end            
          elsif @not_found.has_key?(translitered)
            @stream.puts "\nNo solution\n"
          else
            raise "#{translitered} is neither in found or notFound !" 
          end
       end
        return has_solutions
        
     end     
  end