echelon/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* ReusableGZIPInputStream: Fix 3 major bugs, all present since 2005:

Browse Source 
- Payloads an exact multiple of 512 bytes failed to decompress
    - Data at the end of the stream could be lost
    - read() returned -1 when the data was 0xff
This commit is contained in:
zzz
2011-12-11 22:59:37 +00:00
parent 8448001a17
commit e117e3310c
4 changed files with 158 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
core/java/src/net/i2p/util/LookaheadInputStream.java
View File

@ -17,6 +17,15 @@ public class LookaheadInputStream extends FilterInputStream {
private final byte[] _footerLookahead;
private static final InputStream _fakeInputStream = new ByteArrayInputStream(new byte[0]);
/**
* Configure a stream that hides a number of bytes from the reader.
* The last n bytes will never be available from read(),
* they can only be obtained from getFooter().
*
* initialize() MUST be called before doing any read() calls.
*
* @param lookaheadSize how many bytes to hide
*/
public LookaheadInputStream(int lookaheadSize) {
super(_fakeInputStream);
_footerLookahead = new byte[lookaheadSize];
@ -24,7 +33,11 @@ public class LookaheadInputStream extends FilterInputStream {
public boolean getEOFReached() { return _eofReached; }
/** blocking! */
/**
* Start the LookaheadInputStream with the given input stream.
* Resets everything if the LookaheadInputStream was previously used.
* WARNING - blocking until lookaheadSize bytes are read!
*/
public void initialize(InputStream src) throws IOException {
in = src;
_eofReached = false;
@ -47,6 +60,7 @@ public class LookaheadInputStream extends FilterInputStream {
return -1;
}
int rv = _footerLookahead[0];
// FIXME use an index!!!!!!!!!!!!
System.arraycopy(_footerLookahead, 1, _footerLookahead, 0, _footerLookahead.length-1);
_footerLookahead[_footerLookahead.length-1] = (byte)c;
if (rv < 0) rv += 256;

128
core/java/src/net/i2p/util/ResettableGZIPInputStream.java
View File

@ -1,6 +1,7 @@
package net.i2p.util;
import java.io.ByteArrayInputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.CRC32;
@ -19,8 +20,12 @@ import java.util.zip.InflaterInputStream;
public class ResettableGZIPInputStream extends InflaterInputStream {
private static final int FOOTER_SIZE = 8; // CRC32 + ISIZE
private static final boolean DEBUG = false;
/** keep a typesafe copy of (LookaheadInputStream)in */
/** See below for why this is necessary */
private final ExtraByteInputStream _extraByteInputStream;
/** keep a typesafe copy of this */
private final LookaheadInputStream _lookaheadStream;
private static final byte[] _oneDummyByte = new byte[1];
private final InputStream _sequenceStream = null;
private final CRC32 _crc32;
private final byte _buf1[] = new byte[1];
private boolean _complete;
@ -31,8 +36,15 @@ public class ResettableGZIPInputStream extends InflaterInputStream {
* decompress a stream.
*/
public ResettableGZIPInputStream() {
super(new LookaheadInputStream(FOOTER_SIZE), new Inflater(true));
_lookaheadStream = (LookaheadInputStream)in;
// compressedStream ->
// LookaheadInputStream that removes last 8 bytes ->
// ExtraByteInputStream that adds 1 byte ->
// InflaterInputStream
// See below for why this is necessary
super(new ExtraByteInputStream(new LookaheadInputStream(FOOTER_SIZE)),
new Inflater(true));
_extraByteInputStream = (ExtraByteInputStream)in;
_lookaheadStream = (LookaheadInputStream)_extraByteInputStream.getInputStream();
_crc32 = new CRC32();
}
@ -55,6 +67,7 @@ public class ResettableGZIPInputStream extends InflaterInputStream {
_complete = false;
_crc32.reset();
_buf1[0] = 0x0;
_extraByteInputStream.reset();
// blocking call to read the footer/lookahead, and use the compressed
// stream as the source for further lookahead bytes
_lookaheadStream.initialize(compressedStream);
@ -65,16 +78,10 @@ public class ResettableGZIPInputStream extends InflaterInputStream {
@Override
public int read() throws IOException {
if (_complete) {
// shortcircuit so the inflater doesn't try to refill
// with the footer's data (which would fail, causing ZLIB err)
return -1;
}
int read = read(_buf1, 0, 1);
if (read == -1)
return -1;
else
return _buf1[0];
return _buf1[0] & 0xff;
}
@Override
@ -82,6 +89,9 @@ public class ResettableGZIPInputStream extends InflaterInputStream {
return read(buf, 0, buf.length);
}
/**
*
*/
@Override
public int read(byte buf[], int off, int len) throws IOException {
if (_complete) {
@ -95,7 +105,12 @@ public class ResettableGZIPInputStream extends InflaterInputStream {
return -1;
} else {
_crc32.update(buf, off, read);
if (_lookaheadStream.getEOFReached()) {
// NO, we can't do use getEOFReached here
// 1) Just because the lookahead stream has hit EOF doesn't mean
// that the inflater has given us all the data yet,
// this would cause data loss at the end
//if (_lookaheadStream.getEOFReached()) {
if (inf.finished()) {
verifyFooter();
inf.reset(); // so it doesn't bitch about missing data...
_complete = true;
@ -275,6 +290,97 @@ public class ResettableGZIPInputStream extends InflaterInputStream {
}
}
/**
* Essentially a SequenceInputStream(in, new ByteArrayInputStream(new byte[1])),
* except that this is resettable.
*
* Unsupported:
* - available() doesn't include the extra byte
* - skip() doesn't skip the extra byte
*
* Why? otherwise the inflater finished() is wrong when the compressed payload
* (in between the 10 byte header and the 8 byte footer) is a multiple of 512 bytes,
* which caused read(buf, off, len) above to fail.
* Happened every time with 1042 byte compressed router infos, for example.
*
* Details:
*
* Warning with Inflater nowrap = true:
*
* "Note: When using the 'nowrap' option it is also necessary to provide an extra "dummy" byte as input.
* This is required by the ZLIB native library in order to support certain optimizations."
*
* http://docs.oracle.com/javase/1.5.0/docs/api/java/util/zip/Inflater.html
*
* This is for sure:
*
* "This is not nearly specific enough to be useful. Where in the compressed byte array is the
* extra 'dummy' byte" expected? What is it to contain? When calling setInput() is the 'len'
* argument incremented to include the dummy byte or not?"
*
* http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4795299
*
* This is useless:
*
* http://www.java-forums.org/new-java/38604-decompress-un-gzip-byte.html
*
* This seems to be the definitive answer:
*
* "The fix simply involves copying the byte array and tacking a single null byte on to the end."
*
* http://code.google.com/p/google-apps-sso-sample/issues/detail?id=8
*
*/
private static class ExtraByteInputStream extends FilterInputStream {
private static final byte DUMMY = 0;
private boolean _extraSent;
public ExtraByteInputStream(InputStream in) {
super(in);
}
@Override
public int read() throws IOException {
if (_extraSent)
return -1;
int rv = in.read();
if (rv >= 0)
return rv;
_extraSent = true;
return DUMMY;
}
@Override
public int read(byte buf[], int off, int len) throws IOException {
if (len == 0)
return 0;
if (_extraSent)
return -1;
int rv = in.read(buf, off, len);
if (rv >= 0)
return rv;
_extraSent = true;
buf[off] = DUMMY;
return 1;
}
@Override
public void close() throws IOException {
_extraSent = false;
in.close();
}
/** does NOT call in.reset() */
@Override
public void reset() {
_extraSent = false;
}
public InputStream getInputStream() {
return in;
}
}
/******
public static void main(String args[]) {
for (int i = 129; i < 64*1024; i++) {

25
history.txt
View File

@ -1,3 +1,28 @@
2011-12-12 zzz
* ExploreJob: Tweaks to handle DatabaseLookupMessage changes
* I2NP:
- Deprecate unused stream methods and I2NPMessageReader since
all transports provide encapsulation.
- Don't throw IOE from byte array methods
- Use cached null cert in GarlicClove
- Add method to limit size of buffer to read
- Don't check checksum at input, in most cases
- Reuse checksum at output, for unomodified pass-through messages
(but recalculating it now and logging on a mismatch for testing)
- Fix DatabaseLookupMessage to internally store the don't include peers as
a List, not a Set, so it doesn't get reordered and break the checksum
- Log cleanup
* NTCP:
- Zero-copy and limit size when handing buffer to I2NP
- Log hex dump message on I2NPMessageException, like in SSU
- Don't close connection on I2NPMessageException
* PortMapper: New service for registering application ports in the context
* ReusableGZIPInputStream: Fix 3 major bugs, all present since 2005:
- Payloads an exact multiple of 512 bytes failed to decompress
- Data at the end of the stream could be lost
- read() returned -1 when the data was 0xff
* SearchState: generics and cleanups
2011-12-11 sponge
* HTTP server tunnel, use log.WARN for 3 first minutes. (closes #460)

2
router/java/src/net/i2p/router/RouterVersion.java
View File

@ -18,7 +18,7 @@ public class RouterVersion {
/** deprecated */
public final static String ID = "Monotone";
public final static String VERSION = CoreVersion.VERSION;
public final static long BUILD = 19;
public final static long BUILD = 20;
/** for example "-test" */
public final static String EXTRA = "";