amy/venice-main-classic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added "blacklist" of attachment MIME types that won't be compressed; a couple

Browse Source 
of other minor tweaks
This commit is contained in:
Eric J. Bowersox
2001-04-18 06:01:45 +00:00
parent 0a7aec5103
commit 17d9a58867
8 changed files with 192 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/com/silverwrist/venice/core/impl/EngineBackend.java
View File

@@ -97,4 +97,6 @@ public interface EngineBackend
public abstract boolean isValidRandomAuthString(String s);
public abstract boolean isNoCompressMimeType(String type);
} // end interface EngineBackend

85
src/com/silverwrist/venice/core/impl/TopicMessageUserContextImpl.java
View File

@@ -35,7 +35,7 @@ class TopicMessageUserContextImpl implements TopicMessageContext
*--------------------------------------------------------------------------------
*/
private static Category logger = Category.getInstance(TopicMessageUserContextImpl.class.getName());
private static Category logger = Category.getInstance(TopicMessageUserContextImpl.class);
private static final int MAX_ATTACH = 1048576; // TODO: should be a configurable parameter
@@ -813,6 +813,9 @@ class TopicMessageUserContextImpl implements TopicMessageContext
public void attachData(String m_type, String file, int length, InputStream data)
throws AccessError, DataException
{
if (logger.isDebugEnabled())
logger.debug("attachData to msg " + postid + ": (" + m_type + ", " + file + ", " + length + ")");
if (mimetype!=null)
{ // the message already has an attachment
logger.error("tried to attach data to a message that already has it!");
@@ -865,36 +868,57 @@ class TopicMessageUserContextImpl implements TopicMessageContext
// Compress the attachment data in memory.
InputStream real_data = null;
int real_length = 0;
try
{ // create a compressor into a memory buffer
ByteArrayOutputStream bytestm = new ByteArrayOutputStream(length);
GZIPOutputStream gzipstm = new GZIPOutputStream(bytestm);
int tmp_stgmethod;
// do the usual read/write loop to get the data compressed
byte[] buffer = new byte[4096];
int rd = data.read(buffer);
while (rd>=0)
{ // write, then read
if (rd>0)
gzipstm.write(buffer,0,rd);
rd = data.read(buffer);
if (engine.isNoCompressMimeType(m_type))
{ // don't compress me, just store me normally
logger.debug("stored as NORMAL data (no compression)");
real_data = data;
real_length = length;
tmp_stgmethod = 0; // indicates uncompressed data
} // end while
} // end if
else
{ // OK, we want to compress the data
logger.debug("stored as COMPRESSED data");
// finish compression, then get the data and the real length
gzipstm.finish();
buffer = bytestm.toByteArray();
real_length = buffer.length;
real_data = new ByteArrayInputStream(buffer);
gzipstm.close();
try
{ // create a compressor into a memory buffer
ByteArrayOutputStream bytestm = new ByteArrayOutputStream(length);
GZIPOutputStream gzipstm = new GZIPOutputStream(bytestm);
} // end try
catch (IOException ioe)
{ // the big error
logger.error("IOException in compressor loop: " + ioe.getMessage(),ioe);
throw new DataException("Failure in compressing read data: " + ioe.getMessage());
// do the usual read/write loop to get the data compressed
byte[] buffer = new byte[4096];
int rd = data.read(buffer);
while (rd>=0)
{ // write, then read
if (rd>0)
gzipstm.write(buffer,0,rd);
rd = data.read(buffer);
} // end catch
} // end while
// finish compression, then get the data and the real length
gzipstm.finish();
buffer = bytestm.toByteArray();
real_length = buffer.length;
real_data = new ByteArrayInputStream(buffer);
gzipstm.close();
if (logger.isDebugEnabled())
logger.debug("org size " + length + ", compr size = " + real_length + " ("
+ String.valueOf(100 - ((real_length * 100) / length)) + "% reduction)");
} // end try
catch (IOException ioe)
{ // the big error
logger.error("IOException in compressor loop: " + ioe.getMessage(),ioe);
throw new DataException("Failure in compressing read data: " + ioe.getMessage());
} // end catch
tmp_stgmethod = 1; // indicates compressed data
} // end else
Connection conn = null;
AuditRecord ar = null;
@@ -917,9 +941,11 @@ class TopicMessageUserContextImpl implements TopicMessageContext
StringBuffer sql = new StringBuffer("INSERT INTO postattach (postid, datalen, last_hit, stgmethod, "
+ "filename, mimetype, data) VALUES (");
sql.append(postid).append(", ").append(length).append(", '");
sql.append(SQLUtil.encodeDate(new java.util.Date())).append("', 1, '");
sql.append(SQLUtil.encodeDate(new java.util.Date())).append("', ").append(tmp_stgmethod).append(", '");
sql.append(SQLUtil.encodeString(file)).append("', '").append(SQLUtil.encodeString(m_type));
sql.append("', ?);");
if (logger.isDebugEnabled())
logger.debug("SQL: " + sql.toString());
// Prepare the statement, set the BLOB parameter, and execute it.
PreparedStatement stmt = conn.prepareStatement(sql.toString());
@@ -927,15 +953,16 @@ class TopicMessageUserContextImpl implements TopicMessageContext
stmt.executeUpdate();
// Save off the local attachment values.
logger.debug("... attachment completed successfully");
datalen = length;
filename = file;
mimetype = m_type;
stgmethod = 1;
stgmethod = tmp_stgmethod;
// Generate an audit record indicating what we did.
ar = new AuditRecord(AuditRecord.UPLOAD_ATTACHMENT,conf.realUID(),conf.userRemoteAddress(),
conf.realSIGID(),"conf=" + conf.realConfID() + ",post=" + postid,
"len=" + length + ",type=" + m_type + ",name=" + file);
"len=" + length + ",type=" + m_type + ",name=" + file + ",method=" + tmp_stgmethod);
} // end try
catch (SQLException e)

60
src/com/silverwrist/venice/core/impl/VeniceEngineImpl.java
View File

@@ -382,7 +382,7 @@ public class VeniceEngineImpl implements VeniceEngine, EngineBackend
*--------------------------------------------------------------------------------
*/
private static Category logger = Category.getInstance(VeniceEngineImpl.class.getName());
private static Category logger = Category.getInstance(VeniceEngineImpl.class);
private static final String AUTH_ALPHABET =
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz./";
@@ -411,6 +411,7 @@ public class VeniceEngineImpl implements VeniceEngine, EngineBackend
private Hashtable sidebox_ids = new Hashtable(); // maps sidebox IDs to MasterSideBox objects
private LinkedList cache_fp_posts = new LinkedList(); // all posts that have been published to front page
private boolean cache_fp_posts_busy = false; // busy flag for above vector
private HashSet no_compress_types = new HashSet(); // the file types that can't be compressed
/*--------------------------------------------------------------------------------
* Constructor
@@ -559,6 +560,46 @@ public class VeniceEngineImpl implements VeniceEngine, EngineBackend
} // end if
} // end for
Element upload_sect = root_h.getSubElement("upload");
if (upload_sect==null)
{ // unable to find the "upload" section
logger.fatal("config document has no <upload/> section");
throw new ConfigException("no <upload/> section found in config file",root);
} // end if
// Look for a "no-compress" blacklist.
DOMElementHelper upload_sect_h = new DOMElementHelper(upload_sect);
Element no_compress_sect = upload_sect_h.getSubElement("no-compress");
if (no_compress_sect!=null)
{ // Initialize the no-compress list.
NodeList nc_nodes = no_compress_sect.getChildNodes();
for (i=0; i<nc_nodes.getLength(); i++)
{ // retrieve all entries from the list
Node ncn = nc_nodes.item(i);
if (ncn.getNodeType()==Node.ELEMENT_NODE)
{ // get the element
DOMElementHelper ncel = new DOMElementHelper((Element)ncn);
if (ncel.getElement().getNodeName().equals("type"))
{ // add a new element to the list
no_compress_types.add(ncel.getElementText().trim().toLowerCase());
} // end if
else
{ // this is bad - bail out!
logger.fatal("spurious element in <no-compress/> section");
throw new ConfigException("<no-compress/> section contains spurious element",no_compress_sect);
} // end else
} // end if
// else skip it
} // end for
} // end if
// else no list, just proceed
Element msg_sect = root_h.getSubElement("messages");
if (msg_sect==null)
@@ -1960,4 +2001,21 @@ public class VeniceEngineImpl implements VeniceEngine, EngineBackend
} // end isValidRandomAuthString
public boolean isNoCompressMimeType(String type)
{
if (no_compress_types.isEmpty())
return false; // can't possibly be no-compress
// strip the MIME type name down to basics
int psem = type.indexOf(';');
String real_type;
if (psem>=0)
real_type = type.substring(0,psem).trim().toLowerCase();
else
real_type = type.trim().toLowerCase();
return no_compress_types.contains(real_type);
} // end isNoCompressMimeType
} // end class VeniceEngineImpl

25
src/com/silverwrist/venice/servlets/Attachment.java
View File

@@ -34,7 +34,7 @@ public class Attachment extends VeniceServlet
*--------------------------------------------------------------------------------
*/
private static Category logger = Category.getInstance(Attachment.class.getName());
private static Category logger = Category.getInstance(Attachment.class);
/*--------------------------------------------------------------------------------
* Overrides from class HttpServlet
@@ -102,25 +102,44 @@ public class Attachment extends VeniceServlet
VeniceEngine engine, UserContext user, RenderData rdat)
throws ServletException, IOException, VeniceServletResult
{
logger.debug("Attachment.doVenicePost: entry");
// Get target URL for the operation
if (mphandler.isFileParam("target"))
{ // bogus target URL
logger.error("Internal Error: 'target' should be a normal param");
return new ErrorBox(null,"Internal Error: 'target' should be a normal param","top");
} // end if
String target = mphandler.getValue("target");
if (logger.isDebugEnabled())
logger.debug("Target URL: " + target);
setMyLocation(request,target);
// also check on file parameter status
if (!(mphandler.isFileParam("thefile")))
{ // bogus file parameter
logger.error("Internal Error: 'thefile' should be a file param");
return new ErrorBox(null,"Internal Error: 'thefile' should be a file param",target);
} // end if
// get the SIG
SIGContext sig = getSIGParameter(mphandler,user,true,"top");
if (logger.isDebugEnabled())
logger.debug("SIG param: #" + sig.getSIGID());
changeMenuSIG(request,sig);
// get the conference
ConferenceContext conf = getConferenceParameter(mphandler,sig,true,"top");
if (logger.isDebugEnabled())
logger.debug("Conference param: #" + conf.getConfID());
// get the message we want to use
TopicMessageContext msg = getMessageParameter(mphandler,conf,true,"top");
if (logger.isDebugEnabled())
logger.debug("Message param: #" + msg.getPostID());
try
{ // attach the data to the message!
@@ -130,20 +149,24 @@ public class Attachment extends VeniceServlet
} // end try
catch (ServletMultipartException smpe)
{ // error processing the file parameter data
logger.error("Servlet multipart error:",smpe);
return new ErrorBox(null,"Internal Error: " + smpe.getMessage(),target);
} // end catch
catch (AccessError ae)
{ // access error posting the attachment data
logger.error("Access error:",ae);
return new ErrorBox("Access Error",ae.getMessage(),target);
} // end catch
catch (DataException de)
{ // error storing the attachment in the database
logger.error("DataException:",de);
return new ErrorBox("Database Error","Database error storing attachment: " + de.getMessage(),target);
} // end catch
logger.debug("Attachment.doVenicePost: done!");
throw new RedirectResult(target); // get back, get back, get back to where you once belonged
} // end doVenicePost