/*
 * The contents of this file are subject to the Mozilla Public License Version 1.1
 * (the "License"); you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at <http://www.mozilla.org/MPL/>.
 * 
 * Software distributed under the License is distributed on an "AS IS" basis, WITHOUT
 * WARRANTY OF ANY KIND, either express or implied. See the License for the specific
 * language governing rights and limitations under the License.
 * 
 * The Original Code is the Venice Web Communities System.
 * 
 * The Initial Developer of the Original Code is Eric J. Bowersox <erbo@ricochet.com>,
 * for Silverwrist Design Studios.  Portions created by Eric J. Bowersox are
 * Copyright (C) 2001-2004 Eric J. Bowersox/Silverwrist Design Studios.  All Rights Reserved.
 * 
 * Contributor(s): 
 */
package com.silverwrist.venice.htmlcheck.impl;

import java.net.URL;
import java.util.*;
import org.apache.log4j.*;
import com.silverwrist.venice.htmlcheck.*;

class HTMLCheckerImpl implements HTMLChecker, HTMLCheckerBackend, RewriterServices
{
  /*--------------------------------------------------------------------------------
   * Wrapper for Rewriters to implement counting where necessary
   *--------------------------------------------------------------------------------
   */

  static class CountingRewriter implements Rewriter
  {
    /*====================================================================
     * Attributes
     *====================================================================
     */

    private Rewriter m_inner;
    private int m_count = 0;

    /*====================================================================
     * Constructor
     *====================================================================
     */

    public CountingRewriter(Rewriter inner)
    {
      m_inner = inner;

    } // end constructor
    
    /*====================================================================
     * Implementations from interface Rewriter
     *====================================================================
     */

    public String getName()
    {
      return m_inner.getName();

    } // end getName

    public MarkupData rewrite(String data, RewriterServices svc)
    {
      MarkupData rc = m_inner.rewrite(data,svc);
      if ((rc!=null) && !(rc.getRescan()))
	m_count++;
      return rc;

    } // end rewrite

    /*====================================================================
     * External operations
     *====================================================================
     */

    int getCount()
    {
      return m_count;

    } // end getCount

    void reset()
    {
      m_count = 0;

    } // end reset

  } // end class CountingRewriter

  /*--------------------------------------------------------------------------------
   * State machine constants
   *--------------------------------------------------------------------------------
   */

  private static final short ST_WHITESPACE = 0;
  private static final short ST_CHARS = 1;
  private static final short ST_LEFTANGLE = 2;
  private static final short ST_TAG = 3;
  private static final short ST_PAREN = 4;
  private static final short ST_TAGQUOTE = 5;
  private static final short ST_NEWLINE = 6;

  /*--------------------------------------------------------------------------------
   * Internal constants
   *--------------------------------------------------------------------------------
   */

  private static final int MARGIN_SLOP = 5;

  /*--------------------------------------------------------------------------------
   * Static data members
   *--------------------------------------------------------------------------------
   */

  private static Logger logger = Logger.getLogger(HTMLCheckerImpl.class);

  /*--------------------------------------------------------------------------------
   * Attributes
   *--------------------------------------------------------------------------------
   */

  private HTMLCheckerConfigImpl m_config;           // the configuration we're working from
  private boolean m_started = false;                // have we started parsing?
  private boolean m_finished = false;               // have we finished parsing?
  private short m_state = ST_WHITESPACE;            // current parser state
  private char m_quote_char;                        // quote character to match
  private int m_paren_level = 0;                    // parenthesization level
  private short m_columns = 0;                      // current number of columns
  private int m_lines = 0;                          // current number of lines
  private int m_nobreak_count = 0;                  // "no-break" count
  private boolean m_trigger_wbr = false;            // word break trigger
  private StringBuffer m_output_buffer = null;      // output buffer for characters
  private StringBuffer m_temp_buffer = null;        // temporary buffer used within one state
  private ArrayList m_tag_stack = null;             // stack of tags that have been opened
  private HashMap m_counters = new HashMap();       // the CountingRewriter instances
  private ArrayList m_string_rewriters = new ArrayList();   // string rewriter instances
  private ArrayList m_word_rewriters = new ArrayList();     // word rewriter instances
  private ArrayList m_tag_rewriters = new ArrayList();      // tag rewriter instances
  private ArrayList m_paren_rewriters = new ArrayList();    // paren rewriter instances
  private HashMap m_context_data = new HashMap();   // context variables
  private HashSet m_external_references = new HashSet();  // list of external references
  private HashSet m_internal_references = new HashSet();  // list of internal references

  /*--------------------------------------------------------------------------------
   * Constructor
   *--------------------------------------------------------------------------------
   */

  HTMLCheckerImpl(HTMLCheckerConfigImpl config)
  {
    this.m_config = config;
    copyRewriters(m_string_rewriters,config.getStringRewriters());
    copyRewriters(m_word_rewriters,config.getWordRewriters());
    copyRewriters(m_tag_rewriters,config.getTagRewriters());
    copyRewriters(m_paren_rewriters,config.getParenRewriters());

  } // end constructor

  /*--------------------------------------------------------------------------------
   * Internal functions
   *--------------------------------------------------------------------------------
   */

  /**
   * Returns <code>true</code> if this character belongs as part of a word, <code>false</code> if not.
   *
   * @param ch Character to be tested.
   * @return See above.
   */
  private static final boolean isWordChar(char ch)
  {
    return (Character.isUpperCase(ch) || Character.isLowerCase(ch) || (ch=='-') || (ch=='\''));

  } // end isWordChar

  /**
   * Returns the maximum length of a &quot;run&quot; of word characters or non-word characters in the
   * buffer, beginning at the specified start point, before a character of the opposite classification.
   *
   * @param buf The buffer to look through.
   * @param start The start position to look at.
   * @return See above.
   */
  private static final int getRunLength(StringBuffer buf, int start)
  {
    boolean word_char = isWordChar(buf.charAt(start));
    int l = 1;
    while ((start+l)<buf.length())
    { // see if there's a break from word characters to non-word characters, or vice versa
      if (isWordChar(buf.charAt(start+l))!=word_char)
	break;
      l++;

    } // end while

    return l;

  } // end getRunLength

  /**
   * Returns the maximum length of a &quot;run&quot; of word characters or non-word characters in the
   * buffer, counting from the start of the buffer, before a character of the opposite classification.
   *
   * @param buf The buffer to look through.
   * @return See above.
   */
  private static final int getRunLength(StringBuffer buf)
  {
    return getRunLength(buf,0);

  } // end getRunLength

  /**
   * Copies the <code>Rewriter</code> objects from an outside list to an internal list, wrapping
   * named rewriters in <code>CountingRewriter</code> objects as appropriate.
   *
   * @param dest Destination to copy rewriters to.
   * @param source List to copy rewriters from.
   */
  private final void copyRewriters(ArrayList dest, List source)
  {
    for (Iterator it=source.iterator(); it.hasNext(); )
    { // get each rewriter, and wrap it if it has a name
      Rewriter r = (Rewriter)(it.next());
      String name = r.getName();
      if (name!=null)
      { // wrap it in a CountingRewriter and hash it...
	CountingRewriter cr = new CountingRewriter(r);
	m_counters.put(name,cr);
	r = cr;

      } // end if

      dest.add(r);

    } // end for

    dest.trimToSize();

  } // end copyRewriters

  /**
   * Use the specified list of rewriters to attempt to rewrite the specified string data.
   * The first rewriter in the list that returns a valid <code>MarkupData</code> object takes
   * precedence.
   *
   * @param rewriters List of rewriters to try against the rewriter data.
   * @param data String data to attempt to rewrite.
   * @return A <code>MarkupData</code> object that contains the marked-up data to output, or
   *         <code>null</code> if no rewriter handled the data.
   */
  private final MarkupData attemptRewrite(List rewriters, String data)
  {
    MarkupData rc = null;
    for (Iterator it=rewriters.iterator(); (rc==null) && it.hasNext(); )
    { // look at each rewriter in turn and try seeing if it will mark this text up
      Rewriter r = (Rewriter)(it.next());
      rc = r.rewrite(data,this);

    } // end for

    return rc;

  } // end attemptRewrite

  /**
   * Initializes the internal state of the parser.
   */
  private final void initState()
  {
    m_output_buffer = new StringBuffer(1024);
    m_temp_buffer = new StringBuffer(64);
    m_tag_stack = new ArrayList();

  } // end initState

  /**
   * Erases any temporary data that is no longer needed after the parser finishes.
   */
  private final void killState()
  {
    m_temp_buffer = null;
    if (m_tag_stack!=null)
      m_tag_stack.clear();
    m_tag_stack = null;

  } // end killState

  /**
   * Emits one character to the output of the HTML checker, running it through a list of defined filters.
   *
   * @param ch Character to output.
   * @param filters List of filters to use to attempt to process the character.
   * @param count_cols <code>true</code> if the character output adds to the column counter,
   *                   <code>false</code> if not.
   */
  private final void emitChar(char ch, List filters, boolean count_cols)
  {
    boolean handled = false;
    if (filters.size()>0)
    { // look through all the output filters to see what we can do
      for (Iterator it=filters.iterator(); !handled && it.hasNext(); )
      { // look for an output filter that matches this character
	OutputFilter of = (OutputFilter)(it.next());
	handled = of.tryOutputCharacter(m_output_buffer,ch);
	
      } // end for

    } // end if

    if (!handled)  // output the character
      m_output_buffer.append(ch);
    if (count_cols && (m_config.getWordWrapLength()>0))
      m_columns++;  // adjust column indicator

  } // end emitChar

  /**
   * Emits a character string to the output of the HTML checker, running it through a list of defined filters.
   *
   * @param str String to output.
   * @param filters List of filters to use to attempt to process the string.
   * @param count_cols <code>true</code> if the characters output add to the column counter,
   *                   <code>false</code> if not.
   */
  private final void emitString(String str, List filters, boolean count_cols)
  {
    if (str==null)
      return;
    boolean real_count_cols = count_cols && (m_config.getWordWrapLength()>0);

    if (filters.size()==0)
    { // if there are no filters, just append the entire string directly
      m_output_buffer.append(str);
      if (real_count_cols)
	m_columns += (short)(str.length());
      return;

    } // end if

    String temp = str;
    while ((temp!=null) && (temp.length()>0))
    { // We output as much of the string as we possibly can at once.  For starters,
      // assume we're going to output the whole thing.
      int output_len = temp.length();

      // Now look at each of the output filters to see if we should try outputting
      // a lesser amount (i.e., does the string contain a "stopper" that one of the
      // output filters would like to mogrify?)
      OutputFilter stopper = null;
      for (Iterator it = filters.iterator(); it.hasNext() && (output_len>0); )
      { // look to find the length of characters that doesn't match this filter
	OutputFilter of = (OutputFilter)(it.next());
	int lnm = of.lengthNoMatch(temp);
	if ((lnm>=0) && (lnm<output_len))
	{ // we've found a new stopper - record the length and the filter
	  output_len = lnm;
	  stopper = of;

	} // end if

      } // end for (all filters)

      if (output_len>0)
      { // At least this many unaltered characters can be output, so copy them.
	m_output_buffer.append(temp.substring(0,output_len));
	if (real_count_cols)
	  m_columns += (short)output_len;

      } // end if

      if (stopper!=null)
      { // one of the output filters stopped us, so invoke it
	char tmpch = temp.charAt(output_len++);
	if (!(stopper.tryOutputCharacter(m_output_buffer,tmpch)))
	  m_output_buffer.append(tmpch);
	if (real_count_cols)
	  m_columns++;

      } // end if

      // Chop the string down the middle and go around again.
      if (output_len==temp.length())
	temp = null;
      else if (output_len>0)
	temp = temp.substring(output_len);

    } // end while (still data left to append)

  } // end emitString

  /**
   * Emits a line break to the output of the HTML checker, resetting the column counter and advancing
   * the line counter.
   */
  private final void emitLineBreak()
  {
    emitString("\r\n",m_config.getRawOutputFilters(),false);
    if (m_config.getWordWrapLength()>0)
      m_columns = 0;
    m_lines++;

  } // end emitLineBreak

  /**
   * If the current line of text contains more characters than will fit on the line, emit a line break.
   * Disabled if the checker does not word-wrap.
   */
  private final void emitPossibleLineBreak()
  {
    if ((m_config.getWordWrapLength()>0) && (m_nobreak_count<=0) && (m_columns>=m_config.getWordWrapLength()))
      emitLineBreak();

  } // end emitPossibleLineBreak

  /**
   * Ensure that the current line has a certain number of characters of space left on it; if it does not,
   * emit a line break.
   *
   * @param nchars Number of characters to reserve on the line.
   */
  private final void ensureSpaceOnLine(int nchars)
  {
    if ((m_config.getWordWrapLength()>0) && (m_nobreak_count<=0))
    { // line break might be required here
      int remain_space = (int)(m_config.getWordWrapLength() - m_columns);
      if (remain_space<nchars)
	emitLineBreak();
      
    } // end if

  } // end ensureSpaceOnLine

  private final void emitMarkupData(MarkupData md)
  {
    if (md.getRescan())
      return;
    ensureSpaceOnLine(md.getTextLength());
    List raw_filters = m_config.getRawOutputFilters();
    emitString(md.getBeginMarkup(),raw_filters,false);
    emitString(md.getText(),m_config.getOutputFilters(),true);
    emitString(md.getEndMarkup(),raw_filters,false);

  } // end emitMarkupData

  private final void emitMarkupData(MarkupData md, char prefix, char suffix)
  {
    if (md.getRescan())
      return;
    ensureSpaceOnLine(md.getTextLength() + (md.hasText() ? 2 : 0));
    List raw_filters = m_config.getRawOutputFilters();
    List cooked_filters = m_config.getOutputFilters();
    if (md.hasText())
      emitChar(prefix,cooked_filters,true);
    emitString(md.getBeginMarkup(),raw_filters,false);
    emitString(md.getText(),cooked_filters,true);
    emitString(md.getEndMarkup(),raw_filters,false);
    if (md.hasText())
      emitChar(suffix,cooked_filters,true);
    
  } // end emitMarkupData

  private final void doFlushWhitespace()
  {
    int output_len = m_temp_buffer.length();
    if (output_len<=0)
      return;
    boolean force_line_break = false;

    if ((m_config.getWordWrapLength()>0) && (m_nobreak_count<=0))
    { // adjust output if necessary for word wrapping
      int remain_space = (int)(m_config.getWordWrapLength() - m_columns);
      if (remain_space<output_len)
	output_len = remain_space;
      if (output_len<=0)
      { // this means that NONE of the whitespace would fit on this line...
	force_line_break = true;  // we need a line break to fill in!
	output_len = 0;

      } // end if

    } // end if

    if (force_line_break)
      emitLineBreak();
    if (output_len>0)
      emitString(m_temp_buffer.substring(0,output_len),m_config.getOutputFilters(),true);

    // clear out the buffer
    m_temp_buffer.setLength(0);

  } // end doFlushWhitespace

  private final void doFlushNewlines()
  {
    //logger.debug("FlushNewLines");
    int line_breaks = 0, crs = 0;
    for (int i=0; i<m_temp_buffer.length(); i++)
    { // convert the \r and \n string into a line break count
      char ch = m_temp_buffer.charAt(i);
      if (ch=='\r')
      { // count carriage returns
	//logger.debug("CR");
	crs++;

      } // end if
      else if (ch=='\n')
      { // count line breaks...
	//logger.debug("LF");
	crs = 0;
	line_breaks++;

      } // end else if

    } // end for

    if (crs>0)
      line_breaks++;
    //logger.debug("Total line breaks: " + line_breaks);

    if (m_config.getRewrapLines())
    { // rewrap lines forces a lot of adjustment in lines...
      if (line_breaks<2)
      { // convert a single line break to whitespace
	m_temp_buffer.setLength(0);
	m_temp_buffer.append(' ');
	m_state = ST_WHITESPACE;
	return;

      } // end if
      else
	line_breaks = 2;  // compress out multiple blank lines

    } // end if

    while ((line_breaks--)>0)
      emitLineBreak();  // emit the line breaks

    m_temp_buffer.setLength(0);  // clear out the buffer
    m_state = ST_WHITESPACE;

  } // end doFlushNewlines

  private final void emitFromStartOfTempBuffer(int nchars)
  {
    if (nchars<=0)
      return;  // can't emit less than 1 character!

    if ((m_config.getWordWrapLength()>0) && (m_nobreak_count<=0))
    { // we can output the line break anywhere in the subsequence...
      while (nchars>0)
      { // figure out how many characters we can output WITHOUT causing a line break
	int curlen = nchars;
	int remaining_space = (int)(m_config.getWordWrapLength() - m_columns);
	if (curlen>remaining_space)
	  curlen = remaining_space;

	// EJB 10/2/2001 - only output if we have something to output (and there is space for it)
	if (curlen>0)
	{ // output those characters
	  emitString(m_temp_buffer.substring(0,curlen),m_config.getOutputFilters(),true);
	  m_temp_buffer.delete(0,curlen);
	  nchars -= curlen;

	} // end if

	if (m_columns>=m_config.getWordWrapLength())  // EJB 10/2/2001 - might be greater
	  emitLineBreak();  // and line break us to get to the next line

      } // end while

    } // end if
    else
    { // just output the run of characters straight
      emitString(m_temp_buffer.substring(0,nchars),m_config.getOutputFilters(),true);
      m_temp_buffer.delete(0,nchars);
  
    } // end else

  } // end emitFromStartOfTempBuffer

  private final boolean doFlushString()
  {
    MarkupData md = attemptRewrite(m_string_rewriters,m_temp_buffer.toString());
    if (md!=null)
    { // we've got something marked up!  output it...
      emitMarkupData(md);
      m_temp_buffer.setLength(0);
      if (md.getRescan())
      { // reparse and flag
	parse(md.getAll());
	return true;

      } // end if

      return false;

    } // end if

    boolean first = true;
    while (m_temp_buffer.length()>0)
    { // find the length of the initial string of word or non-word characters
      int sublen = getRunLength(m_temp_buffer);

      if (isWordChar(m_temp_buffer.charAt(0)))
      { // we need to check the word...but first, we must eliminate leading hyphens and apostrophes
	int hyph_count = 0;
	final String hyph_apos = "-\'";
	while ((hyph_count<sublen) && (hyph_apos.indexOf(m_temp_buffer.charAt(hyph_count))>=0))
	  hyph_count++;
	emitFromStartOfTempBuffer(hyph_count);
	sublen -= hyph_count;

	// now determine how many hyphens/apostrophes there are at the end of the word...
	int word_len = sublen;
	hyph_count = 0;
	while ((word_len>0) && (hyph_apos.indexOf(m_temp_buffer.charAt(word_len-1))>=0))
	{ // decrement word length, increment hyphen/apostrophe count
	  hyph_count++;
	  word_len--;

	} // end while

	if (word_len>0)
	{ // extract the word from the start of the buffer
	  String word = m_temp_buffer.substring(0,word_len);
	  m_temp_buffer.delete(0,word_len);

	  // try to rewrite this word...
	  md = attemptRewrite(m_word_rewriters,word);
	  if (md!=null)
	  { // emit and/or re-parse
	    emitMarkupData(md);
	    if (md.getRescan())
	      parse(md.getAll());

	  } // end if
	  else
	  { // just output the word normally
	    ensureSpaceOnLine(word.length());
	    emitString(word,m_config.getOutputFilters(),true);

	  } // end else

	} // end if

	// now emit the rest of the hyphens/apostrophes
	emitFromStartOfTempBuffer(hyph_count);

      } // end if
      else
      { // just emit this many characters, line-breaking where required
	if ((sublen==m_temp_buffer.length()) && !first && (sublen<=MARGIN_SLOP))
        { // This is intended to handle a small run of non-word characters at the end of a string (i.e.
	  // followed by whitespace) that should stay on the same line with its preceding word, to
	  // eliminate "funnies" in punctuation formatting.
	  emitString(m_temp_buffer.toString(),m_config.getOutputFilters(),true);
	  m_temp_buffer.setLength(0);
	  break;

	} // end if

	// This is kind of the inverse of the above check; if we have a small run of non-word
	// characters at the START of a word (preceded by whitespace and followed by at least
	// one word character), then ensure that we can keep that word and its prefixing non-word
	// characters on the same line (again, avoiding "funnies" in formatting).
	if ((sublen<m_temp_buffer.length()) && first && (sublen<=MARGIN_SLOP))
	{ // get the length of the word and ensure they can be kept together
	  int fw_len = getRunLength(m_temp_buffer,sublen);
	  ensureSpaceOnLine(sublen+fw_len);

        } // end if

	emitFromStartOfTempBuffer(sublen);

      } // end else

      first = false;

    } // end while

    return false;

  } // end doFlushString

  private final boolean handleAsHTML()
  {
    if (logger.isDebugEnabled())
      logger.debug("handleAsHTML(): candidate buffer = [" + m_temp_buffer.toString() + "]");
    m_trigger_wbr = false;  // initialize

    // Figure out the place in the buffer where the command word starts.
    int start_cmd = 0;
    boolean closing_tag = false;
    if ((start_cmd<m_temp_buffer.length()) && (m_temp_buffer.charAt(start_cmd)=='/'))
    { // this is a closing tag - move the command start pointer...
      if (logger.isDebugEnabled())
	logger.debug("closing tag detected");
      start_cmd++;
      closing_tag = true;

    } // end if

    // Now figure out where it ends.
    int end_cmd = start_cmd;
    while ((end_cmd<m_temp_buffer.length()) && !(Character.isWhitespace(m_temp_buffer.charAt(end_cmd))))
      end_cmd++;

    if ((end_cmd==start_cmd) || ((end_cmd-start_cmd)>TagRepository.getMaxTagLength()))
    { // the command word is empty or is just too long to be an HTML tag
      if (logger.isDebugEnabled())
	logger.debug("HTML tag rejected due to length (" + String.valueOf(end_cmd-start_cmd) + ")");
      return false;

    } // end if

    // Look up the tag name to get a tag index from the repository.
    String poss_tag_name = m_temp_buffer.substring(start_cmd,end_cmd);
    int tag_index = TagRepository.tagNameToIndex(poss_tag_name);
    if (tag_index<0)
    { // not a known HTML tag
      if (logger.isDebugEnabled())
	logger.debug("<" + poss_tag_name + "> is not valid HTML");
      return false;

    } // end if

    // Look up the tag object that corresponds to the tag index.
    SimpleTag tagobj = TagRepository.tagIndexToObject(tag_index);
    if (closing_tag && !(tagobj.allowClose()))
    { // this is a closing tag, and this tag doesn't permit the "close" form
      if (logger.isDebugEnabled())
	logger.debug("</" + poss_tag_name + "> is not a permitted tag form (no close)");
      return false;

    } // end if

    // Get the HTML tag set index for this tag, and see if we allow that set.
    int tag_set_id = TagRepository.tagIndexToSet(tag_index);
    if (!(m_config.isTagSetAllowed(tag_set_id)))
    { // the tag is not allowed - either discard it or leave it in verbatim
      if (m_config.getDiscardHTMLTags() || m_config.getDiscardRejectedHTML())
      { // throw this tag the hell away!
	if (logger.isDebugEnabled())
	  logger.debug("<" + poss_tag_name + "> tag rejected and discarded");
	return true;

      } // end if
      else
      { // kick the tag out and let some other code deal with it
	if (logger.isDebugEnabled())
	  logger.debug("<" + poss_tag_name + "> is not allowed in this context");
	return false;

      } // end if

    } // end if (tag rejected by HTML rules)

    if (!(m_config.getDiscardHTMLTags()) && tagobj.balanceTags())
    { // this tag needs to be balanced - here is where we manipulate the stack
      boolean valid = false;

      if (closing_tag)
      { // hunt through the list to find the most recently-opened tag of this type
	int i = m_tag_stack.size();
	while ((--i)>=0)
	{ // look through the stack...
	  Integer foo = (Integer)(m_tag_stack.get(i));
	  if (foo.intValue()==tag_index)
	  { // found it - remove it from the tag stack
	    if (logger.isDebugEnabled())
	      logger.debug("balanced closure of <" + poss_tag_name + ">");
	    m_tag_stack.remove(i);
	    valid = true;
	    break;

	  } // end if

	} // end while

      } // end if
      else
      { // push a new opening tag!
	if (logger.isDebugEnabled())
	  logger.debug("open form of <" + poss_tag_name + ">");
	m_tag_stack.add(new Integer(tag_index));
	valid = true;

      } // end else

      if (!valid)
      { // not validated by the stack code
	if (logger.isDebugEnabled())
	  logger.debug("stack checking code rejects <" + poss_tag_name + ">");
	return false;

      } // end if

    } // end if
    // else tag doesn't need to be auto-balanced, or is being discarded
    
    // Give the tag object one last chance to dictate what we do with the tag.
    String real_tag_data = tagobj.rewriteTagContents(m_temp_buffer.toString(),closing_tag,this);
    if ((real_tag_data==null) || m_config.getDiscardHTMLTags())
    { // tag is being erased by rewriter, or is being discarded anyway
      if (logger.isDebugEnabled())
	logger.debug("rewriter erasure or discard of <" + poss_tag_name + ">");
      return true;

    } // end if

    if (logger.isDebugEnabled())
      logger.debug("real tag data = [" + real_tag_data + "]");

    // Emit the tag to the output.
    List filters = m_config.getRawOutputFilters();
    emitChar('<',filters,false);
    emitString(real_tag_data,filters,false);
    emitChar('>',filters,false);

    // Determine whether this tag causes a "logical line break."
    boolean logical_line_break = false;
    if (m_trigger_wbr && !closing_tag && (m_nobreak_count>0))
    { // a word break is a logical line break (but only if within <NOBR>...</NOBR>)
      if (logger.isDebugEnabled())
	logger.debug("<" + poss_tag_name + "> WORD BREAKS");
      logical_line_break = true;

    } // end if
    else
      logical_line_break = tagobj.causeLineBreak(closing_tag);
    if (logical_line_break)
    { // logical line break is triggered
      if (logger.isDebugEnabled())
	logger.debug("<" + poss_tag_name + "> triggers logical line break");
      m_columns = 0;

    } // end if

    return true;  // handled!

  } // end handleAsHTML

  /**
   * Returns <code>true</code> if the temporary buffer contains the start of an HTML comment.  (The
   * leading and trailing angle brackets are assumed.)
   *
   * @return See above.
   */
  private final boolean containsHTMLComment()
  {
    return ((m_temp_buffer.length()>=3) && (m_temp_buffer.substring(0,3).equals("!--")));

  } // end containsHTMLComment

  /**
   * Returns <code>true</code> if the temporary buffer contains a complete HTML comment.  (The leading
   * and trailing angle brackets are assumed.)
   *
   * @return See above.
   */
  private final boolean containsCompleteHTMLComment()
  {
    int l = m_temp_buffer.length();
    // note that a minimum HTML comment is <!---->, i.e. "<!--" followed by "-->" with no characters
    // in between...
    return ((l>=5) && (m_temp_buffer.substring(0,3).equals("!--"))
	    && (m_temp_buffer.substring(l-2,l).equals("--")));

  } // end containsCompleteHTMLComment

  /**
   * Returns <code>true</code> if the temporary buffer contains an XML construct, i.e. a tag that
   * contains a ':', and may or may not have a leading '/'.  (The leading and trailing angle brackets
   * are assumed.)
   *
   * @return See above.
   */
  private final boolean containsXMLConstruct()
  {
    int ptr = 0;
    if ((m_temp_buffer.length()>1) && (m_temp_buffer.charAt(0)=='/'))
      ptr++;
    while (ptr<m_temp_buffer.length())
    { // look for a ':' in the "tag name" portion of the HTML tag
      char ch = m_temp_buffer.charAt(ptr++);
      if (ch==':')
	return true;
      if (Character.isWhitespace(ch))
	break;

    } // end while

    return false;

  } // end containsXMLConstruct

  private final void finishTag()
  {
    if (containsHTMLComment())
    { // we are dealing with at least the start of an HTML comment - is it a complete one?
      if (!(containsCompleteHTMLComment()))
	return;  // we have an HTML comment but it's not complete yet - bail out

      if (!(m_config.getDiscardHTMLComments()))
      { // output the comment in the raw
	List filters = m_config.getRawOutputFilters();
	emitChar('<',filters,false);
	emitString(m_temp_buffer.toString(),filters,false);
	emitChar('>',filters,false);

      } // end if

      // clear our state and return to parsing
      m_temp_buffer.setLength(0);
      m_state = ST_WHITESPACE;
      return;

    } // end if

    if (handleAsHTML())
    { // the tag has been handled as an HTML tag - bail out immediately
      m_temp_buffer.setLength(0);
      m_state = ST_WHITESPACE;
      return;

    } // end if

    // now try to handle it using a tag rewriter
    MarkupData md = attemptRewrite(m_tag_rewriters,m_temp_buffer.toString());
    if (md!=null)
    { // we've got something marked up!  output it...
      emitMarkupData(md,'<','>');
      m_temp_buffer.setLength(0);
      m_state = ST_WHITESPACE;
      if (md.getRescan())
      { // shift to character state to parse the rescan stuff
	m_temp_buffer.setLength(0);
	m_temp_buffer.append('<');
	m_state = ST_CHARS;
	parse(md.getAll() + ">");

      } // end if
      return;

    } // end if

    if (m_config.getDiscardXMLConstructs() && containsXMLConstruct())
    { // the tag is an XML construct, and is to be discarded
      m_temp_buffer.setLength(0);
      m_state = ST_WHITESPACE;
      return;

    } // end if

    // This tag has been rejected!  We need to process it normally, as character data.
    String rejection = m_temp_buffer.toString();
    m_temp_buffer.setLength(0);
    m_temp_buffer.append('<');
    m_state = ST_CHARS;
    if (rejection.length()>0)
      parse(rejection);   // just run it through the parser, now that we've fixed up the state
    parse(">");

  } // end finishTag

  private final void finishParen()
  {
    // Try to handle the paren element using a paren rewriter.
    MarkupData md = attemptRewrite(m_paren_rewriters,m_temp_buffer.toString());
    if (md!=null)
    { // we've got something marked up!  output it...
      emitMarkupData(md,'(',')');
      m_temp_buffer.setLength(0);
      m_state = ST_WHITESPACE;
      m_paren_level = 0;
      if (md.getRescan())
      { // rescan the interior...
	m_temp_buffer.setLength(0);
	m_temp_buffer.append('(');
	m_state = ST_CHARS;
	parse(md.getAll() + ")");

      } // end if

      return;

    } // end if

    // This tag has been rejected!  We need to process it normally, as character data.
    String rejection = m_temp_buffer.toString();
    m_temp_buffer.setLength(0);
    m_temp_buffer.append('(');
    m_state = ST_CHARS;
    m_paren_level = 0;
    if (rejection.length()>0)
      parse(rejection);   // just run it through the parser, now that we've fixed up the state
    parse(")");

  } // end finishParen

  private final void parse(String str)
  {
    int i = 0;
    while (i<str.length())
    { // get the character from the string
      char ch = str.charAt(i);

      // now process differently based on the current parser state
      switch (m_state)
      {
	case ST_WHITESPACE:
	  { // Whitespace handling - look at the character
	    switch (ch)
	    {
	      case ' ':     // append spaces and tabs verbatim to the temp buffer
	      case '\t':
		m_temp_buffer.append(ch);
		i++;
		break;

	      case '\r':    // flush the whitespace and go to NEWLINE state
	      case '\n':
		doFlushWhitespace();
		m_state = ST_NEWLINE;
		m_temp_buffer.append(ch);
		i++;
		break;

	      case '<':
		doFlushWhitespace();  // flush the whitespace we've already got
		if (m_config.getProcessAngles())
		  m_state = ST_LEFTANGLE;
		else
		{ // just process as an ordinary character
		  m_state = ST_CHARS;
		  m_temp_buffer.append(ch);

		} // end else
		i++;
		break;

	      case '(':
		doFlushWhitespace();  // flush the whitespace we've already got
		if (m_config.getProcessParens())
		  m_state = ST_PAREN;
		else
		{ // just process as an ordinary character
		  m_state = ST_CHARS;
		  m_temp_buffer.append(ch);

		} // end else
		i++;
		break;

	      case '\\':  // backslash processing is complex - shift to ST_CHARS state to handle it
		doFlushWhitespace();
		m_state = ST_CHARS;
		break;

	      default:
		doFlushWhitespace();  // flush the whitespace we've already got
		m_state = ST_CHARS;
		m_temp_buffer.append(ch);
		i++;
		break;

	    } // end switch

	  } // end case
	  break;

	case ST_CHARS:
	  { // Character data handling - look at the character
	    switch (ch)
	    {
	      case ' ':   // whitespace - drop back to whitespace mode
	      case '\t':
		doFlushString();
		m_state = ST_WHITESPACE;
		m_temp_buffer.append(ch);
		i++;
		break;

	      case '\r':  // newline characters - flush out the buffer and go to NEWLINE state
	      case '\n':
		doFlushString();
		m_state = ST_NEWLINE;
		m_temp_buffer.append(ch);
		i++;
		break;

	      case '<':   // left angle bracket - may be a start-of-tag
		if (m_config.getProcessAngles())
		{ // this is a tag start - go to LEFTANGLE state
		  doFlushString();
		  m_state = ST_LEFTANGLE;

		} // end if
		else  // just handle it normally
		  m_temp_buffer.append(ch);
		i++;
		break;

		/* EJB 7/27/2004 - don't go into PAREN state from CHARS because this may be part of a smiley
	      case '(':   // left parenthesis - may be a start-of-paren
		if (m_config.getProcessParens())
		{ // we're going into Parens mode...
		  doFlushString();
		  m_paren_level = 0;
		  m_state = ST_PAREN;

		} // end if
		else  // just handle it normally
		  m_temp_buffer.append(ch);
		i++;
		break;
		*/

	      case '\\':
		if (i<(str.length()-1))
		{ // look at the character following the backslash
		  ch = str.charAt(++i);
		  if (((ch=='(') && m_config.getProcessParens()) || ((ch=='<') && m_config.getProcessAngles()))
		  { // append the escaped character, omitting the backslash
		    m_temp_buffer.append(ch);
		    i++;

		  } // end if
		  else // append the backslash and hit the new character
		    m_temp_buffer.append('\\');

		} // end if
		else
		{ // just append the backslash normally
		  m_temp_buffer.append('\\');
		  i++;

		} // end else
		break;

	      default:  // just append the next non-white character
		m_temp_buffer.append(ch);
		i++;
		break;
	      
	    } // end switch

	  } // end case
	  break;

	case ST_LEFTANGLE:
	  { // Left Angle processing - this isn't very difficult
	    switch (ch)
	    {
	      case ' ':
	      case '\t':
	      case '\r':
	      case '\n':  // output the < and click back to whitespace mode
		emitChar('<',m_config.getOutputFilters(),true);
		m_state = ST_WHITESPACE;
		break;

	      case '<':   // output the < and stay in this mode
		emitChar('<',m_config.getOutputFilters(),true);
		i++;
		break;

	      default:    // click over to TAG mode
		m_state = ST_TAG;
		m_temp_buffer.append(ch);
		i++;
		break;
	      
	    } // end switch

	  } // end case
	  break;

	case ST_TAG:
	  { // inside a tag - process the character here
	    switch (ch)
	    {
	      case '>':  // end tag
		finishTag();  // this changes the state, and maybe calls parse() recursively
		i++;
		break;

	      case '\'':  // go into "quote string" mode inside tag
	      case '\"':
		m_temp_buffer.append(ch);
		m_state = ST_TAGQUOTE;
		m_quote_char = ch;
		i++;
		break;

	      default:   // just append more data to the temp buffer
		m_temp_buffer.append(ch);
		i++;
		break;

	    } // end switch

	  } // end case
	  break;

	case ST_PAREN:
	  { // inside parentheses - try to build something up
	    switch (ch)
	    {
	      case '(':  // append the open parenthesis and kick it up a notch!
		m_temp_buffer.append(ch);
		m_paren_level++;
		i++;
		break;

	      case ')':
		if (m_paren_level==0)
		  finishParen();  // will change the parser state
		else
		{ // append the close parenthesis and kick it DOWN a notch
		  m_temp_buffer.append(ch);
		  m_paren_level--;

		} // end else
		i++;
		break;

	      default:
		m_temp_buffer.append(ch);
		i++;
		break;

	    } // end switch

	  } // end case
	  break;

	case ST_TAGQUOTE:
	  m_temp_buffer.append(ch);
	  if (ch==m_quote_char) // close quote seen - go back to ST_TAG state
	    m_state = ST_TAG;
	  i++;
	  break;

	case ST_NEWLINE:
	  if ((ch=='\r') || (ch=='\n'))
	  { // handle newlines
	    m_temp_buffer.append(ch);
	    i++;

	  } // end if
	  else // flush out the newlines and click back to whitespace mode
	    doFlushNewlines();
	  break;

	default:
	  throw new IllegalStateException("invalid parser state value");

      } // end switch

    } // end while (looking through string)

  } // end parse

  /*--------------------------------------------------------------------------------
   * Implementations from interface HTMLChecker
   *--------------------------------------------------------------------------------
   */

  public void append(String str) throws AlreadyFinishedException
  {
    if (m_finished)
      throw new AlreadyFinishedException();
    if (!m_started)
    { // initialize the parser state
      initState();
      m_started = true;

    } // end if

    if (str!=null)
      parse(str);  // parse things

  } // end append

  public void finish() throws AlreadyFinishedException
  {
    if (m_finished)
      throw new AlreadyFinishedException();
    if (!m_started)
    { // set up the initial parser state (so we don't kill ourselves later)
      initState();
      m_started = true;

    } // end if

    // This is the "end parse" loop, in which we resolve any funny state the parser has
    // found itself in and clear out the internal buffers.
    boolean running;
    do
    { // what we do depends on the parser state...
      running = false;  // make sure we stop unless running is set to true here

      switch (m_state)
      {
	case ST_WHITESPACE:
	case ST_NEWLINE:
	  break;    // discard any whitespace or newlines at the end of output

	case ST_CHARS:
	  running = doFlushString();  // flush out the temporary buffer
	  break;

	case ST_LEFTANGLE:  // just emit a '<' character
	  emitPossibleLineBreak();
	  emitChar('<',m_config.getOutputFilters(),true);
	  break;

	case ST_TAG:
	case ST_TAGQUOTE:
	  { // we won't finish this tag, so it's automagically rejected
	    String rejection = m_temp_buffer.toString();
	    m_temp_buffer.setLength(0);
	    m_temp_buffer.append('<');
	    m_state = ST_CHARS;

	    // now parse the tag contents again
	    if (rejection.length()>0)
	      parse(rejection);
	    
	    running = true;  // go back around for another try

	  } // end case
	  break;

	case ST_PAREN:
	  { // we won't finish this paren tag, so it's automagically rejected
	    String rejection = m_temp_buffer.toString();
	    m_temp_buffer.setLength(0);
	    m_temp_buffer.append('(');
	    m_state = ST_CHARS;
	    m_paren_level = 0;

	    // now parse the parenthesis contents again
	    if (rejection.length()>0)
	      parse(rejection);
	    
	    running = true;  // go back around for another try

	  } // end case
	  break;

      } // end switch

    } while (running);  // end do

    // Now close all the HTML tags that were left open.
    for (int i=(m_tag_stack.size()-1); i>=0; i--)
    { // get each element in the tag stack and append the appropriate closing tag
      Integer foo = (Integer)(m_tag_stack.get(i));
      SimpleTag tagobj = TagRepository.tagIndexToObject(foo.intValue());
      m_output_buffer.append(tagobj.makeClosingTag());

    } // end for

    // deallocate some excess crap and mark the object as finished
    killState();
    m_lines++;  // there's one extra line at the end
    m_finished = true;

  } // end finish

  public void reset()
  {
    m_started = false;
    m_finished = false;
    m_trigger_wbr = false;
    m_state = ST_WHITESPACE;
    m_quote_char = '\0';
    m_columns = 0;
    m_lines = 0;
    m_paren_level = 0;
    m_output_buffer = null;
    m_external_references.clear();
    m_internal_references.clear();
    killState();

    // Also reset all the counters.
    for (Iterator it=m_counters.values().iterator(); it.hasNext(); )
    { // reset each counter in turn
      CountingRewriter cr = (CountingRewriter)(it.next());
      cr.reset();

    } // end for

  } // end reset

  public String getValue() throws NotYetFinishedException
  {
    if (!m_finished)
      throw new NotYetFinishedException();
    return m_output_buffer.toString();

  } // end getValue

  public int getLength() throws NotYetFinishedException
  {
    if (!m_finished)
      throw new NotYetFinishedException();
    return m_output_buffer.length();

  } // end getLength

  public int getLines() throws NotYetFinishedException
  {
    if (!m_finished)
      throw new NotYetFinishedException();
    return m_lines;

  } // end getLines

  public int getCounter(String name) throws NotYetFinishedException
  {
    if (!m_finished)
      throw new NotYetFinishedException();
    CountingRewriter cr = (CountingRewriter)(m_counters.get(name));
    if (cr==null)
      return 0;
    else
      return cr.getCount();

  } // end getCounter

  public Object getContextValue(String name)
  {
    return m_context_data.get(name);

  } // end getContextValue

  public void setContextValue(String name, Object val)
  {
    m_context_data.put(name,val);

  } // end setContextValue

  public Set getExternalReferences() throws NotYetFinishedException
  {
    if (!m_finished)
      throw new NotYetFinishedException();
    if (m_external_references.isEmpty())
      return Collections.EMPTY_SET;
    HashSet rc = new HashSet(m_external_references);
    return Collections.unmodifiableSet(rc);

  } // end getExternalReferences

  public Set getInternalReferences() throws NotYetFinishedException
  {
    if (!m_finished)
      throw new NotYetFinishedException();
    if (m_internal_references.isEmpty())
      return Collections.EMPTY_SET;
    HashSet rc = new HashSet(m_internal_references);
    return Collections.unmodifiableSet(rc);

  } // end getInternalReferences

  /*--------------------------------------------------------------------------------
   * Implementations from interface HTMLCheckerBackend
   *--------------------------------------------------------------------------------
   */

  public String getCheckerAttrValue(String name)
  {
    if (name=="ANCHORTAIL")
      return m_config.getAnchorTail();

    throw new IllegalArgumentException("attribute \"" + name + "\" is not defined");

  } // end getCheckerAttrValue

  public void sendTagMessage(String msg)
  {
    if (msg=="NOBR")
    { // increment the no-break count
      m_nobreak_count++;
      return;

    } // end if

    if (msg=="/NOBR")
    { // decrement the no-break count
      m_nobreak_count--;
      return;

    } // end if

    if (msg=="WBR")
    { // trigger a word break
      m_trigger_wbr = true;
      return;

    } // end if

    throw new IllegalArgumentException("message \"" + msg + "\" is not defined");

  } // end sendTagMessage

  public Object getCheckerContextValue(String name)
  {
    return m_context_data.get(name);

  } // end getCheckerContextValue

  public void addExternalReference(URL ref)
  {
    m_external_references.add(ref);

  } // end addExternalReference

  public void addInternalReference(String ref)
  {
    m_internal_references.add(ref);

  } // end addInternalReference

  /*--------------------------------------------------------------------------------
   * Implementations from interface RewriterServices
   *--------------------------------------------------------------------------------
   */

  public String getRewriterAttrValue(String name)
  {
    return getCheckerAttrValue(name);

  } // end getRewriterAttrValue

  public Object getRewriterContextValue(String name)
  {
    return m_context_data.get(name);

  } // end getRewriterContextValue

  // addExternalReference is implemented as part of HTMLCheckerBackend

} // end class HTMLCheckerImpl
