/*
 * The contents of this file are subject to the Mozilla Public License Version 1.1
 * (the "License"); you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at <http://www.mozilla.org/MPL/>.
 * 
 * Software distributed under the License is distributed on an "AS IS" basis, WITHOUT
 * WARRANTY OF ANY KIND, either express or implied. See the License for the specific
 * language governing rights and limitations under the License.
 * 
 * The Original Code is the Venice Web Communities System.
 * 
 * The Initial Developer of the Original Code is Eric J. Bowersox <erbo@ricochet.com>,
 * for Silverwrist Design Studios.  Portions created by Eric J. Bowersox are
 * Copyright (C) 2004 Eric J. Bowersox/Silverwrist Design Studios.  All Rights Reserved.
 * 
 * Contributor(s): 
 */
package com.silverwrist.venice.core.impl;

import java.io.*;
import java.sql.*;
import java.text.*;
import java.util.*;
import java.util.zip.*;
import javax.xml.parsers.*;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.*;
import org.xml.sax.*;
import org.xml.sax.helpers.*;
import com.silverwrist.util.*;
import com.silverwrist.venice.core.ConferenceContext;
import com.silverwrist.venice.core.internals.EngineBackend;
import com.silverwrist.venice.db.*;
import com.silverwrist.venice.except.*;

class ConferencingImporter
{
  /*--------------------------------------------------------------------------------
   * Inner class containing topic data
   *--------------------------------------------------------------------------------
   */

  private class TopicData
  {
    /*====================================================================
     * Attributes
     *====================================================================
     */

    private int m_index;                 // topic index
    private boolean m_frozen = false;    // are we frozen?
    private boolean m_archived = false;  // are we archived?
    private StringBuffer m_name;         // topic name buffer

    /*====================================================================
     * Constructor
     *====================================================================
     */

    TopicData(Attributes attrs) throws SAXException
    {
      boolean index_seen = false;
      for (int i=0; i<attrs.getLength(); i++)
      { // grind the attributes into information
	String name = resolveName(attrs.getLocalName(i),attrs.getQName(i));
	if (name.equals("index"))
	{ // saw the index attribute
	  try
	  { // handle parsing the topic index
	    if (index_seen)
	      throw new SAXException("<topic> index specified twice");
	    m_index = Integer.parseInt(attrs.getValue(i));
	    index_seen = true;

	  } // end if
	  catch (NumberFormatException e)
	  { // this sucks!
	    throw new SAXException("invalid numeric format for <topic> index");

	  } // end catch

	} // end if
	else if (name.equals("frozen"))
	{ // parse the "frozen" attribute
	  if (StringUtil.isBooleanTrue(attrs.getValue(i)))
	    m_frozen = true;
	  else if (StringUtil.isBooleanFalse(attrs.getValue(i)))
	    m_frozen = false;
	  else
	    throw new SAXException("invalid boolean format for <topic> frozen");

	} // end else if
	else if (name.equals("archived"))
	{ // parse the "archived" attribute
	  if (StringUtil.isBooleanTrue(attrs.getValue(i)))
	    m_archived = true;
	  else if (StringUtil.isBooleanFalse(attrs.getValue(i)))
	    m_archived = false;
	  else
	    throw new SAXException("invalid boolean format for <topic> archived");

	} // end else if
	// else ignore any unknown attributes

      } // end for

      if (!index_seen)
	throw new SAXException("<topic> index not specified");

      m_name = new StringBuffer();

    } // end constructor

    /*====================================================================
     * External operations
     *====================================================================
     */

    final void appendNameData(char[] ch, int start, int length)
    {
      m_name.append(ch,start,length);

    } // end appendNameData

    final void resolveTopic() throws SAXException
    {
      Statement stmt_x = null;
      PreparedStatement stmt = null;
      boolean mid_create = false;

      try
      { // lock the tables we need
	stmt_x = m_conn.createStatement();
	stmt_x.executeUpdate("LOCK TABLES confs WRITE, topics WRITE;");

	// determine how to try to match the topic
	switch (m_match_method)
	{
	  case ConferenceContext.IMPORT_MATCH_NUM:
	    stmt = m_conn.prepareStatement("SELECT topicid FROM topics WHERE confid = ? AND num = ?;");
	    stmt.setInt(2,m_index);
	    break;

	  case ConferenceContext.IMPORT_MATCH_NAME:
	    stmt = m_conn.prepareStatement("SELECT topicid FROM topics WHERE confid = ? AND name = ?;");
	    stmt.setString(2,m_name.toString());
	    break;

	  default:
	    throw new SAXException("internal error: invalid match method");

	} // end switch

	stmt.setInt(1,m_conf_id);
	ResultSet rs = stmt.executeQuery();
	if (rs.next())
	{ // found a matching topic - save the ID and return
	  m_cur_topic = rs.getInt(1);
	  SQLUtil.shutdown(rs);
	  return;

	} // end if

	SQLUtil.shutdown(rs);
	if (!m_create_new)
	{ // the topic was not matched and cannot be created - ignore it
	  recordEvent("Topic <" + m_index + ".> (" + m_name.toString() + ") not matched or created - skipped.");
	  m_cur_topic = -1;
	  return;

	} // end if

	mid_create = true;

	// get the proper topic index
	SQLUtil.shutdown(stmt);
	stmt = m_conn.prepareStatement("SELECT top_topic + 1 FROM confs WHERE confid = ?;");
	stmt.setInt(1,m_conf_id);
	rs = stmt.executeQuery();
	if (rs.next())
	  m_index = rs.getInt(1);
	else
	  m_index = 1;
	SQLUtil.shutdown(rs);

	// prepare the new topic entry
	SQLUtil.shutdown(stmt);
	stmt = m_conn.prepareStatement("INSERT INTO topics (confid, num, creator_uid, top_message, frozen, archived, "
				       + "createdate, lastupdate, name) VALUES (?, ?, ?, -1, ?, ?, ?, ?, ?);");
	stmt.setInt(1,m_conf_id);
	stmt.setInt(2,m_index);
	stmt.setInt(3,m_uid);
	stmt.setInt(4,m_frozen ? 1 : 0);
	stmt.setInt(5,m_archived ? 1 : 0);
	java.util.Date now = new java.util.Date();
	SQLUtil.setFullDateTime(stmt,6,now);
	SQLUtil.setFullDateTime(stmt,7,now);
	stmt.setString(8,m_name.toString());
	stmt.executeUpdate();

	// get the new topic ID
	rs = stmt_x.executeQuery("SELECT LAST_INSERT_ID();");
	if (rs.next())
	  m_cur_topic = rs.getInt(1);
	else
	  throw new SAXException("internal error: can\'t get new topic ID");

	// update the last update and top topic fields
	SQLUtil.shutdown(stmt);
	stmt = m_conn.prepareStatement("UPDATE confs SET lastupdate = ?, top_topic = ? WHERE confid = ?;");
	SQLUtil.setFullDateTime(stmt,1,now);
	stmt.setInt(2,m_index);
	stmt.setInt(3,m_conf_id);
	stmt.executeUpdate();

      } // end try
      catch (SQLException e)
      { // record the event and continue
	if (mid_create)
	  recordEvent("error creating topic \"" + m_name.toString() + "\"",e);
	else
	  recordEvent("error resolving topic <" + m_index + ".> (" + m_name.toString() + ")",e);
	m_cur_topic = -1;

      } // end catch
      finally
      { // shut down things
	SQLUtil.unlockTables(m_conn);
	SQLUtil.shutdown(stmt);
	SQLUtil.shutdown(stmt_x);

      } // end finally

    } // end resolveTopic

  } // end class TopicData

  /*--------------------------------------------------------------------------------
   * Inner class containing post data
   *--------------------------------------------------------------------------------
   */

  private class PostData
  {
    /*====================================================================
     * Attributes
     *====================================================================
     */

    private long m_id_in;
    private long m_parent_in;
    private int m_index;
    private int m_line_count;
    private String m_author;
    private java.util.Date m_date;
    private boolean m_hidden = false;
    private String m_scribble_by = null;
    private java.util.Date m_scribble_date = null;
    private StringBuffer m_pseud;
    private StringBuffer m_text;
    private int m_att_length = -1;
    private String m_att_type = null;
    private String m_att_filename = null;
    private File m_att_data = null;

    /*====================================================================
     * Constructor
     *====================================================================
     */

    PostData(Attributes attrs) throws SAXException
    {
      boolean id_seen = false, parent_seen = false, index_seen = false, linecount_seen = false, author_seen = false;
      boolean date_seen = false;
      for (int i=0; i<attrs.getLength(); i++)
      { // grind the attributes into information
	String name = resolveName(attrs.getLocalName(i),attrs.getQName(i));
	if (name.equals("id"))
	{ // get the post ID
	  try
	  { // handle parsing the topic index
	    if (id_seen)
	      throw new SAXException("<post> ID specified twice");
	    m_id_in = Long.parseLong(attrs.getValue(i));
	    id_seen = true;

	  } // end if
	  catch (NumberFormatException e)
	  { // this sucks!
	    throw new SAXException("invalid numeric format for <post> ID");

	  } // end catch

	} // end if
	else if (name.equals("parent"))
	{ // get the parent ID
	  try
	  { // handle parsing the topic index
	    if (parent_seen)
	      throw new SAXException("<post> parent ID specified twice");
	    m_parent_in = Long.parseLong(attrs.getValue(i));
	    parent_seen = true;

	  } // end if
	  catch (NumberFormatException e)
	  { // this sucks!
	    throw new SAXException("invalid numeric format for <post> parent ID");

	  } // end catch

	} // end else if
	else if (name.equals("index"))
	{ // get the index
	  try
	  { // handle parsing the topic index
	    if (index_seen)
	      throw new SAXException("<post> index specified twice");
	    m_index = Integer.parseInt(attrs.getValue(i));
	    index_seen = true;

	  } // end if
	  catch (NumberFormatException e)
	  { // this sucks!
	    throw new SAXException("invalid numeric format for <post> index");

	  } // end catch

	} // end else if
	else if (name.equals("lines"))
	{ // get the number of lines
	  try
	  { // handle parsing the topic index
	    if (linecount_seen)
	      throw new SAXException("<post> line count specified twice");
	    m_line_count = Integer.parseInt(attrs.getValue(i));
	    linecount_seen = true;

	  } // end if
	  catch (NumberFormatException e)
	  { // this sucks!
	    throw new SAXException("invalid numeric format for <post> line count");

	  } // end catch

	} // end else if
	else if (name.equals("author"))
	{ // get the post author as a string
	  if (author_seen)
	    throw new SAXException("<post> author name specified twice");
	  m_author = attrs.getValue(i);
	  author_seen = true;

	} // end else if
	else if (name.equals("date"))
	{ // get the post date
	  try
	  { // parse the ISO 8601 date
	    if (date_seen)
	      throw new SAXException("<post> date specified twice");
	    m_date = s_datefmt.parse(attrs.getValue(i));
	    date_seen = true;

	  } // end try
	  catch (ParseException e)
	  { // date parsing failed
	    throw new SAXException("invalid date format for <post> date",e);

	  } // end catch

	} // end else if
	else if (name.equals("hidden"))
	{ // parse the "hidden" attribute
	  if (StringUtil.isBooleanTrue(attrs.getValue(i)))
	    m_hidden = true;
	  else if (StringUtil.isBooleanFalse(attrs.getValue(i)))
	    m_hidden = false;
	  else
	    throw new SAXException("invalid boolean format for <post> hidden");

	} // end else if
	// else ignore any unknown attributes
 
      } // end for

      if (!id_seen)
	throw new SAXException("<post> ID not specified");
      if (!parent_seen)
	throw new SAXException("<post> parent ID not specified");
      if (!index_seen)
	throw new SAXException("<post> index not specified");
      if (!linecount_seen)
	throw new SAXException("<post> line count not specified");
      if (!author_seen)
	throw new SAXException("<post> author name not specified");
      if (!date_seen)
	throw new SAXException("<post> date not specified");

      m_pseud = new StringBuffer();
      m_text = new StringBuffer();

    } // end constructor

    /*====================================================================
     * External operations
     *====================================================================
     */

    final void dispose()
    {
      if (m_att_data!=null)
      { // delete the temp file
	m_att_data.delete();
	m_att_data = null;

      } // end if

    } // end dispose

    final void markScribbled(Attributes attrs) throws SAXException
    {
      boolean by_seen = false, date_seen = false;
      for (int i=0; i<attrs.getLength(); i++)
      { // grind the attributes into information
	String name = resolveName(attrs.getLocalName(i),attrs.getQName(i));
	if (name.equals("by"))
	{ // get the user name that scribbled it
	  if (by_seen)
	    throw new SAXException("<scribbled> user name specified twice");
	  m_scribble_by = attrs.getValue(i);
	  by_seen = true;

	} // end if
	else if (name.equals("date"))
	{ // get the scribbled date
	  try
	  { // parse the ISO 8601 date
	    if (date_seen)
	      throw new SAXException("<scribbled> date specified twice");
	    m_scribble_date = s_datefmt.parse(attrs.getValue(i));
	    date_seen = true;

	  } // end try
	  catch (ParseException e)
	  { // date parsing failed
	    throw new SAXException("invalid date format for <scribbled> date",e);

	  } // end catch

	} // end else if
	// else ignore any unknown attributes

      } // end for
      
      if (!by_seen)
	throw new SAXException("<scribbled> user name not specified");
      if (!date_seen)
	throw new SAXException("<scribbled> date not specified");

    } // end markScribbled

    final void appendPseud(char[] ch, int start, int length)
    {
      m_pseud.append(ch,start,length);

    } // end appendPseud

    final void appendText(char[] ch, int start, int length)
    {
      m_text.append(ch,start,length);

    } // end appendText

    final OutputStream openAttachment(Attributes attrs) throws SAXException
    {
      boolean length_seen = false, type_seen = false, filename_seen = false;
      for (int i=0; i<attrs.getLength(); i++)
      { // grind the attributes into information
	String name = resolveName(attrs.getLocalName(i),attrs.getQName(i));
	if (name.equals("length"))
	{ // get the length
	  try
	  { // handle parsing the length
	    if (length_seen)
	      throw new SAXException("<attachment> length specified twice");
	    m_att_length = Integer.parseInt(attrs.getValue(i));
	    length_seen = true;

	  } // end if
	  catch (NumberFormatException e)
	  { // this sucks!
	    throw new SAXException("invalid numeric format for <attachment> length");

	  } // end catch

	} // end if
	else if (name.equals("type"))
	{ // get the MIME type
	  if (type_seen)
	    throw new SAXException("<attachment> MIME type specified twice");
	  m_att_type = attrs.getValue(i);
	  type_seen = true;

	} // end else if
	else if (name.equals("filename"))
	{ // get the filename
	  if (filename_seen)
	    throw new SAXException("<attachment> filename specified twice");
	  m_att_filename = attrs.getValue(i);
	  filename_seen = true;

	} // end else if
	// else ignore any unknown attributes

      } // end for

      if (!length_seen)
	throw new SAXException("<attachment> length not specified");
      if (!type_seen)
	throw new SAXException("<attachment> MIME type not specified");
      if (!filename_seen)
	throw new SAXException("<attachment> filename not specified");

      boolean need_delete = false;
      try
      { // create the attachment data file and open it
	m_att_data = File.createTempFile("Attachment",null);
	need_delete = true;
	return new FileOutputStream(m_att_data);

      } // end try
      catch (IOException e)
      { // record the event and continue
	recordEvent("unable to open temporary file for attachment \"" + m_att_filename + "\"",e);
	m_att_length = -1;
	m_att_type = null;
	m_att_filename = null;
	if (need_delete)
	  m_att_data.delete();
	m_att_data = null;

      } // end catch

      return null;

    } // end openAttachment

    final void postThis() throws SAXException
    {
      /*
    private String m_att_type = null;
    private String m_att_filename = null;
       */

      // get the creator UID
      int creator_uid = getUIDForUser(m_author);
      if (creator_uid==-1)
      { // add a note to the beginning of the text and change the creator UID
	m_text.insert(0,"<i>(Originally posted by \"" + m_author + "\")</i>\r\n\r\n");
	m_line_count += 2;
	creator_uid = m_uid;

      } // end if

      // get the scribble UID
      int scribble_uid = -1;
      if (m_scribble_by!=null)
      { // get the UID of the scribbling user
	scribble_uid = getUIDForUser(m_scribble_by);
	if (scribble_uid==-1)
	  scribble_uid = m_uid;

      } // end if

      Statement stmt_x = null;
      PreparedStatement stmt = null;
      try
      { // lock the tables we'll need
	stmt_x = m_conn.createStatement();
	stmt_x.executeUpdate("LOCK TABLES confs WRITE, confsettings WRITE, topics WRITE, topicsettings WRITE, "
			     + "posts WRITE, postdata WRITE, postattach WRITE;");

	// get the new index for the post
	stmt = m_conn.prepareStatement("SELECT top_message + 1 FROM topics WHERE topicid = ?;");
	stmt.setInt(1,m_cur_topic);
	ResultSet rs = stmt.executeQuery();
	if (rs.next())
	  m_index = rs.getInt(1);
	else
	  m_index = 0;
	SQLUtil.shutdown(rs);

	// prepare the initial post entry
	SQLUtil.shutdown(stmt);
	stmt = m_conn.prepareStatement("INSERT INTO posts (parent, topicid, num, linecount, creator_uid, posted, "
				       + "hidden, pseud) VALUES (?, ?, ?, ?, ?, ?, ?, ?);");
	stmt.setLong(1,getPostMapping(m_parent_in));
	stmt.setInt(2,m_cur_topic);
	stmt.setInt(3,m_index);
	stmt.setInt(4,m_line_count);
	stmt.setInt(5,creator_uid);
	SQLUtil.setFullDateTime(stmt,6,m_date);
	stmt.setInt(7,m_hidden ? 1 : 0);
	stmt.setString(8,m_pseud.toString());
	stmt.executeUpdate();

	// get the new post ID
	long pid;
	rs = stmt.executeQuery("SELECT LAST_INSERT_ID();");
	if (rs.next())
	  pid = rs.getLong(1);
	else
	  throw new SAXException("internal error: cannot get new post ID");
	setPostMapping(m_id_in,pid);

	// add the post data
	SQLUtil.shutdown(stmt);
	stmt = m_conn.prepareStatement("INSERT INTO postdata(postid, data) VALUES (?, ?);");
	stmt.setLong(1,pid);
	stmt.setString(2,m_text.toString());
	stmt.executeUpdate();

	if (m_scribble_date!=null)
	{ // update the post to reflect scribble dates
	  SQLUtil.shutdown(stmt);
	  stmt = m_conn.prepareStatement("UPDATE posts SET scribble_uid = ?, scribble_date = ? WHERE postid = ?;");
	  stmt.setInt(1,scribble_uid);
	  SQLUtil.setFullDateTime(stmt,2,m_scribble_date);
	  stmt.setLong(3,pid);
	  stmt.executeUpdate();
	  
	} // end if

	// update the topic to reflect the last update
	SQLUtil.shutdown(stmt);
	stmt = m_conn.prepareStatement("UPDATE topics SET top_message = ?, lastupdate = ? WHERE topicid = ?;");
	stmt.setInt(1,m_index);
	SQLUtil.setFullDateTime(stmt,2,m_date);
	stmt.setInt(3,m_cur_topic);
	stmt.executeUpdate();

	// update the topic settings to reflect the last post date
	SQLUtil.shutdown(stmt);
	stmt = m_conn.prepareStatement("UPDATE topicsettings SET last_post = ? WHERE topicid = ? AND uid = ?;");
	SQLUtil.setFullDateTime(stmt,1,m_date);
	stmt.setInt(2,m_cur_topic);
	stmt.setInt(3,creator_uid);
	if (stmt.executeUpdate()<1)
	{ // We didn't have a topic settings record yet; create one.	
	  SQLUtil.shutdown(stmt);
	  stmt = m_conn.prepareStatement("INSERT INTO topicsettings (topicid, uid, last_post) VALUES (?, ?, ?);");
	  stmt.setInt(1,m_cur_topic);
	  stmt.setInt(2,creator_uid);
	  SQLUtil.setFullDateTime(stmt,3,m_date);
	  stmt.executeUpdate();
	  
	} // end if

	// Update the "last update" date of the conference.
	SQLUtil.shutdown(stmt);
	stmt = m_conn.prepareStatement("UPDATE confs SET lastupdate = ? WHERE confid = ?;");
	SQLUtil.setFullDateTime(stmt,1,m_date);
	stmt.setInt(2,m_conf_id);

	// Update the "last post" timestamp in conference settings.
	SQLUtil.shutdown(stmt);
	stmt = m_conn.prepareStatement("UPDATE confsettings SET last_read = ?, last_post = ? WHERE confid = ? "
				       + "AND uid = ?;");
	SQLUtil.setFullDateTime(stmt,1,m_date);
	SQLUtil.setFullDateTime(stmt,2,m_date);
	stmt.setInt(3,m_conf_id);
	stmt.setInt(4,creator_uid);
	if (stmt.executeUpdate()<1)
	{ // no "confsettings" record found - create one
	  SQLUtil.shutdown(stmt);
	  stmt = m_conn.prepareStatement("INSERT INTO confsettings (confid, uid, last_read, last_post) "
					 + "VALUES (?, ?, ?, ?);");
	  stmt.setInt(1,m_conf_id);
	  stmt.setInt(2,creator_uid);
	  SQLUtil.setFullDateTime(stmt,3,m_date);
	  SQLUtil.setFullDateTime(stmt,4,m_date);
	  
	} // end if
	
	if (m_att_type!=null)
	{ // there's an attachment to be stored!  First see if we need to compress the data.
	  InputStream stm = null;
	  int stg_method, stmlen;
	  try
	  { // handle all the IO manipulations here
	    if (m_engine.isNoCompressMimeType(m_att_type))
	    { // don't compress, just copy the file data directly
	      stm = new FileInputStream(m_att_data);
	      stmlen = m_att_length;
	      stg_method = 0;

	    } // end if
	    else
	    { // create the buffer to compress into
	      ByteArrayOutputStream bytestm = new ByteArrayOutputStream(m_att_length);
	      GZIPOutputStream gzipstm = new GZIPOutputStream(bytestm);
	      IOUtil.copy(new FileInputStream(m_att_data),gzipstm);
	      gzipstm.finish();
	      byte[] buffer = bytestm.toByteArray();
	      stmlen = buffer.length;
	      stm = new ByteArrayInputStream(buffer);
	      IOUtil.shutdown(gzipstm);
	      stg_method = 1;

	    } // end else

	    // add the attachment record to the database!
	    stmt = m_conn.prepareStatement("INSERT INTO postattach(postid, datalen, last_hit, stgmethod, "
					   + "filename, mimetype, data) VALUES (?, ?, ?, ?, ?, ?, ?);");
	    stmt.setLong(1,pid);
	    stmt.setInt(2,m_att_length);
	    SQLUtil.setFullDateTime(stmt,3,new java.util.Date());
	    stmt.setInt(4,stg_method);
	    stmt.setString(5,m_att_filename);
	    stmt.setString(6,m_att_type);
	    stmt.setBinaryStream(7,stm,stmlen);
	    stmt.executeUpdate();

	  } // end try
	  catch (IOException e)
	  { // bail out from attachment if we screw this up
	    recordEvent("error getting/compressing attachment data",e);
	    return;

	  } // end catch

	} // end if

      } // end try
      catch (SQLException e)
      { // record a database error here
	recordEvent("error posting message",e);

      } // end catch
      finally
      { // close everything down
	SQLUtil.unlockTables(m_conn);
	SQLUtil.shutdown(stmt_x);
	SQLUtil.shutdown(stmt);

      } // end finally

    } // end postThis

  } // end class PostData

  /*--------------------------------------------------------------------------------
   * Inner class containing XML listener
   *--------------------------------------------------------------------------------
   */

  private class Listener extends DefaultHandler
  {
    /*====================================================================
     * Attributes
     *====================================================================
     */

    private TopicData m_topicdata = null;
    private PostData m_postdata = null;
    private OutputStream m_attachment = null;
    private byte[] m_leftover = null;

    /*====================================================================
     * Constructor
     *====================================================================
     */

    Listener()
    {
      super();

    } // end constructor

    /*====================================================================
     * Internal operations
     *====================================================================
     */

    private final void handleAttachmentChars(char[] ch, int start, int length) throws SAXException
    {
      if (m_attachment==null)
	return;  // no attachment - don't bother

      byte[] code_data = null;
      try
      { // convert from an array of characters to an array of bytes
	String tmp = new String(ch,start,length);
	code_data = tmp.getBytes("US-ASCII");

      } // end try
      catch (UnsupportedEncodingException e)
      { // this should never happen
	throw new SAXException("?!?!?!? WTF? US-ASCII is OK!",e);

      } // end catch

      int len_leftover = (m_leftover!=null) ? m_leftover.length : 0;
      if (len_leftover==0)
	m_leftover = null;
      int t = (len_leftover + code_data.length) / 4;
      int ilen = t * 4;
      int len_new_leftover = (len_leftover + code_data.length) - ilen;

      byte[] input_data = null;
      if (ilen>0)
      { // create a perfect input group of 4-byte blocks
	input_data = new byte[ilen];
	if (len_leftover>0)
	  System.arraycopy(m_leftover,0,input_data,0,len_leftover);
	System.arraycopy(code_data,0,input_data,len_leftover,ilen - len_leftover);

      } // end if

      if (len_new_leftover>0)
      { // set the leftovers aside
	m_leftover = new byte[len_new_leftover];
	System.arraycopy(code_data,code_data.length - len_new_leftover,m_leftover,0,len_new_leftover);

      } // end if
      else  // no leftovers
	m_leftover = null;

      if (ilen==0)
	return;  // no data to write

      try
      { // decode the base-64 data and write it!
	m_attachment.write(Base64.decodeBase64(input_data));

      } // end try
      catch (IOException e)
      { // error during the decode or write
	throw new SAXException("error decoding and writing attachment data",e);

      } // end catch

    } // end handleAttachmentChars

    private final void finishAttachment() throws SAXException
    {
      if ((m_leftover!=null) && (m_leftover.length==0))
	m_leftover = null;
      if (m_leftover==null)
	return;

      try
      { // decode the base-64 data and write it!
	m_attachment.write(Base64.decodeBase64(m_leftover));

      } // end try
      catch (IOException e)
      { // error during the decode or write
	throw new SAXException("error decoding and writing attachment data",e);

      } // end catch
      finally
      { // null out the leftover array
	m_leftover = null;

      } // end finally

    } // end finishAttachment

    /*====================================================================
     * Overrides from class DefaultHandler
     *====================================================================
     */

    public void setDocumentLocator(Locator locator)
    {
      m_locator = locator;

    } // end setDocumentLocator

    public void startDocument()
    {
      m_state = ST_PRESTART;
      m_cur_topic = -1;

    } // end startDocument

    public void endDocument() throws SAXException
    {
      if (m_state!=ST_END)
	throw new SAXException("no content found");
      m_locator = null;

    } // end endDocument

    public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException
    {
      String name = resolveName(localName,qName);
      switch (m_state)
      {
	case ST_PRESTART:
	  if (!(name.equals("vcif")))
	    throw new SAXException("<vcif> expected");
	  m_state = ST_INSTART;
	  break;

	case ST_INSTART:
	  if (!(name.equals("topic")))
	    throw new SAXException("<topic> expected");
	  m_topicdata = new TopicData(attributes);
	  m_state = ST_INTOPIC_1;
	  break;

	case ST_INTOPIC_1:
	  if (!(name.equals("topicname")))
	    throw new SAXException("<topicname> expected");
	  m_state = ST_INTOPIC_NAME;
	  break;

	case ST_INTOPIC:
	  if (!(name.equals("post")))
	    throw new SAXException("<post> expected");
	  if (m_cur_topic>0)
	    m_postdata = new PostData(attributes);
	  m_state = ST_INPOST;
	  break;

	case ST_INPOST:
	  if (name.equals("scribbled"))
	  { // mark post as scribbled
	    if (m_postdata!=null)
	      m_postdata.markScribbled(attributes);
	    m_state = ST_SCRIBBLETAG;

	  } // end if
	  else if (name.equals("pseud"))
	    m_state = ST_PSEUD;
	  else if (name.equals("text"))
	    m_state = ST_TEXT;
	  else if (name.equals("attachment"))
	  { // open the attachment
	    if (m_postdata!=null)
	      m_attachment = m_postdata.openAttachment(attributes);
	    m_state = ST_ATTACHMENT;

	  } // end else if
	  else
	    throw new SAXException("<scribbled>, <pseud>, <text>, or <attachment> expected");
	  break;

	default:
	  throw new SAXException("startElement state error (state = " + m_state + ", elt = \"" + name + "\")");

      } // end switch

    } // end startElement

    public void endElement(String uri, String localName, String qName) throws SAXException
    {
      String name = resolveName(localName,qName);
      switch (m_state)
      {
	case ST_INSTART:
	  if (!(name.equals("vcif")))
	    throw new SAXException("</vcif> expected");
	  m_state = ST_END;
	  break;

	case ST_INTOPIC_NAME:
	  if (!(name.equals("topicname")))
	    throw new SAXException("</topicname> expected");
	  if (m_topicdata==null)
	    throw new SAXException("internal error: no topic data");
	  m_topicdata.resolveTopic();
	  m_topicdata = null;
	  m_state = ST_INTOPIC;
	  break;

	case ST_INTOPIC:
	  if (!(name.equals("topic")))
	    throw new SAXException("</topic> expected");
	  m_cur_topic = -1;
	  m_state = ST_INSTART;
	  break;

	case ST_INPOST:
	  if (!(name.equals("post")))
	    throw new SAXException("</post> expected");
	  if (m_postdata!=null)
	  { // post complete - make it
	    try
	    { // post the message!
	      m_postdata.postThis();

	    } // end try
	    finally
	    { // make sure we dispose the postdata
	      m_postdata.dispose();
	      m_postdata = null;

	    } // end finally

	  } // end if

	  m_state = ST_INTOPIC;
	  break;

	case ST_SCRIBBLETAG:
	  if (!(name.equals("scribbled")))
	    throw new SAXException("</scribbled> expected");
	  m_state = ST_INPOST;
	  break;

	case ST_PSEUD:
	  if (!(name.equals("pseud")))
	    throw new SAXException("</pseud> expected");
	  m_state = ST_INPOST;
	  break;

	case ST_TEXT:
	  if (!(name.equals("text")))
	    throw new SAXException("</text> expected");
	  m_state = ST_INPOST;
	  break;

	case ST_ATTACHMENT:
	  if (!(name.equals("attachment")))
	    throw new SAXException("</attachment> expected");
	  try
	  { // finish the attachment
	    finishAttachment();

	  } // end try
	  finally
	  { // close it all down
	    IOUtil.shutdown(m_attachment);
	    m_attachment = null;
	    m_state = ST_INPOST;

	  } // end finally
	  break;

	default:
	  throw new SAXException("endElement state error (state = " + m_state + ", elt = \"" + name + "\")");

      } // end switch

    } // end endElement

    public void characters(char[] ch, int start, int length) throws SAXException
    {
      switch (m_state)
      {
	case ST_INTOPIC_NAME:
	  if (m_topicdata!=null)
	    m_topicdata.appendNameData(ch,start,length);
	  break;

	case ST_PSEUD:
	  if (m_postdata!=null)
	    m_postdata.appendPseud(ch,start,length);
	  break;

	case ST_TEXT:
	  if (m_postdata!=null)
	    m_postdata.appendText(ch,start,length);
	  break;

	case ST_ATTACHMENT:
	  handleAttachmentChars(ch,start,length);
	  break;

	default:
	  break;  // ignore characters

      } // end switch

    } // end characters

    public void warning(SAXParseException e) throws SAXException
    {
      recordEvent("parser warning",e);
      super.warning(e);

    } // end warning

    public void error(SAXParseException e) throws SAXException
    {
      recordEvent("parser error",e);
      super.error(e);

    } // end error

    public void fatalError(SAXParseException e) throws SAXException
    {
      recordEvent("parser fatal error",e);
      super.fatalError(e);

    } // end fatalError

  } // end class Listener
  
  /*--------------------------------------------------------------------------------
   * Static data members
   *--------------------------------------------------------------------------------
   */

  private static Logger logger = Logger.getLogger(ConferencingImporter.class);

  private static final DateFormat s_datefmt;

  private static final int ST_PRESTART = 0;
  private static final int ST_INSTART = 1;
  private static final int ST_END = 2;
  private static final int ST_INTOPIC_1 = 3;
  private static final int ST_INTOPIC_NAME = 4;
  private static final int ST_INTOPIC = 5;
  private static final int ST_INPOST = 6;
  private static final int ST_SCRIBBLETAG = 7;
  private static final int ST_PSEUD = 8;
  private static final int ST_TEXT = 9;
  private static final int ST_ATTACHMENT = 10;

  /*--------------------------------------------------------------------------------
   * Attributes
   *--------------------------------------------------------------------------------
   */

  private int m_uid;                   // UID we're creating stuff with
  private int m_conf_id;               // conference ID we're importing to
  private int m_match_method;          // conference match method
  private boolean m_create_new;        // create new topic if it doesn't exist?
  private Connection m_conn;           // SQL database connection
  private EngineBackend m_engine;      // the engine
  private LinkedList m_events = null;  // the events reported by the parser
  private Locator m_locator = null;    // locator for events
  private int m_state;                 // current state of the parser
  private int m_cur_topic;             // current topic ID
  private HashMap m_post_map;          // mapping from old to new post IDs

  /*--------------------------------------------------------------------------------
   * Constructor
   *--------------------------------------------------------------------------------
   */

  ConferencingImporter(int uid, int conf_id, int match_method, boolean create_new, Connection conn,
		       EngineBackend engine)
  {
    m_uid = uid;
    m_conf_id = conf_id;
    m_match_method = match_method;
    m_create_new = create_new;
    m_conn = conn;
    m_engine = engine;
    m_post_map = new HashMap();

  } // end constructor

  /*--------------------------------------------------------------------------------
   * Internal operations
   *--------------------------------------------------------------------------------
   */

  private static final String resolveName(String local_name, String q_name)
  {
    if ((local_name!=null) && (local_name.length()>0))
      return local_name;
    else
      return q_name;

  } // end resolveName

  private final int getUIDForUser(String username)
  {
    PreparedStatement stmt = null;
    ResultSet rs = null;
    try
    { // straightforward query
      stmt = m_conn.prepareStatement("SELECT uid FROM users WHERE username = ?;");
      stmt.setString(1,username);
      rs = stmt.executeQuery();
      if (rs.next())
	return rs.getInt(1);
      else
	return -1;
      
    } // end try
    catch (SQLException e)
    { // whoops: big error!
      recordEvent("error getting UID for user name \"" + username + "\"",e);
      return -1;

    } // end catch
    finally
    { // shut everything down
      SQLUtil.shutdown(rs);
      SQLUtil.shutdown(stmt);

    } // end finally

  } // end getUIDForUser

  private final void recordEvent(String message)
  {
    StringBuffer buf = new StringBuffer("[");
    buf.append(m_locator.getLineNumber()).append(", ").append(m_locator.getColumnNumber()).append("] ");
    buf.append(message);
    if (m_events==null)
      m_events = new LinkedList();
    m_events.addLast(buf.toString());

  } // end recordEvent

  private final void recordEvent(String message, Throwable t)
  {
    StringBuffer buf = new StringBuffer("[");
    buf.append(m_locator.getLineNumber()).append(", ").append(m_locator.getColumnNumber()).append("] ");
    buf.append(message).append(" - ").append(t.getMessage());
    if (m_events==null)
      m_events = new LinkedList();
    m_events.addLast(buf.toString());

  } // end recordEvent

  private final long getPostMapping(long old)
  {
    if (old==0)
      return 0;
    Long rc = (Long)(m_post_map.get(new Long(old)));
    return (rc==null) ? 0 : rc.longValue();

  } // end getPostMapping

  private final void setPostMapping(long old, long nu)
  {
    m_post_map.put(new Long(old),new Long(nu));

  } // end setPostMapping

  /*--------------------------------------------------------------------------------
   * External operations
   *--------------------------------------------------------------------------------
   */

  final List importMessages(InputStream xmlstm) throws DataException
  {
    try
    { // create a SAX parser and let it loose on the input data with our listener
      SAXParserFactory fact = SAXParserFactory.newInstance();
      fact.setNamespaceAware(false);
      fact.setValidating(false);
      SAXParser parser = fact.newSAXParser();
      parser.parse(xmlstm,new Listener());

    } // end try
    catch (ParserConfigurationException e)
    { // configuration error
      throw new DataException("Error configuring XML parser for message import: " + e.getMessage(),e);

    } // end catch
    catch (SAXException e)
    { // give an error message
      throw new DataException("Error importing messages: " + e.getMessage(),e);

    } // end catch
    catch (IOException e)
    { // I/O error in parsing!
      throw new DataException("Error importing messages: " + e.getMessage(),e);

    } // end catch

    if (m_events==null)
      return Collections.EMPTY_LIST;
    ArrayList rc = new ArrayList(m_events);
    m_events = null;
    return Collections.unmodifiableList(rc);

  } // end importMessages

  /*--------------------------------------------------------------------------------
   * Static initializer
   *--------------------------------------------------------------------------------
   */

  static
  { // create an ISO 8601 date formatter
    SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd'T'HHmmss");
    df.setTimeZone(new SimpleTimeZone(0,"UTC"));
    df.setLenient(false);
    s_datefmt = df;

  } // end static initializer

} // end class ConferencingImporter
