Remove the "buf = NULL" => skip requirement from pb_istream_t callbacks.

Rationale: it's easy to implement the callback wrong. Doing so introduces
io errors when unknown fields are present in the input. If code is not
tested with unknown fields, these bugs can remain hidden for long time.

Added a special case for the memory buffer stream, where it gives a small
speed benefit.

Added testcase for skipping fields with test_decode2 implementation.

Update issue 37
Status: FixedInGit

docs/concepts.rst

@@ -92,9 +92,8 @@ Writing to stdout::
 
 Input streams
 -------------
-For input streams, there are a few extra rules:
+For input streams, there is one extra rule:
 
-#) If buf is NULL, read from stream but don't store the data. This is used to skip unknown input.
 #) You don't need to know the length of the message in advance. After getting EOF error when reading, set bytes_left to 0 and return false. Pb_decode will detect this and if the EOF was in a proper position, it will return true.
 
 Here is the structure::

example/common.c

@@ -19,15 +19,6 @@ static bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count)
     int fd = (intptr_t)stream->state;
     int result;
     
-    if (buf == NULL)
-    {
-        /* Well, this is a really inefficient way to skip input. */
-        /* It is only used when there are unknown fields. */
-        char dummy;
-        while (count-- && recv(fd, &dummy, 1, 0) == 1);
-        return count == 0;
-    }
-    
     result = recv(fd, buf, count, MSG_WAITALL);
     
     if (result == 0)

pb_decode.c

@@ -36,18 +36,6 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = {
  * pb_istream *
  **************/
 
-bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count)
-{
-    if (stream->bytes_left < count)
-        PB_RETURN_ERROR(stream, "end-of-stream");
-    
-    if (!stream->callback(stream, buf, count))
-        PB_RETURN_ERROR(stream, "io error");
-    
-    stream->bytes_left -= count;
-    return true;
-}
-
 static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count)
 {
     uint8_t *source = (uint8_t*)stream->state;
@@ -59,6 +47,33 @@ static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t coun
     return true;
 }
 
+bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count)
+{
+	if (buf == NULL && stream->callback != buf_read)
+	{
+		/* Skip input bytes */
+		uint8_t tmp[16];
+		while (count > 16)
+		{
+			if (!pb_read(stream, tmp, 16))
+				return false;
+			
+			count -= 16;
+		}
+		
+		return pb_read(stream, tmp, count);
+	}
+
+    if (stream->bytes_left < count)
+        PB_RETURN_ERROR(stream, "end-of-stream");
+    
+    if (!stream->callback(stream, buf, count))
+        PB_RETURN_ERROR(stream, "io error");
+    
+    stream->bytes_left -= count;
+    return true;
+}
+
 pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize)
 {
     pb_istream_t stream;

pb_decode.h

@@ -19,12 +19,10 @@
  * Rules for callback:
  * 1) Return false on IO errors. This will cause decoding to abort.
  * 
- * 2) If buf is NULL, read but don't store bytes ("skip input").
+ * 2) You can use state to store your own data (e.g. buffer pointer),
- * 
- * 3) You can use state to store your own data (e.g. buffer pointer),
  * and rely on pb_read to verify that no-body reads past bytes_left.
  * 
- * 4) Your callback may be used with substreams, in which case bytes_left
+ * 3) Your callback may be used with substreams, in which case bytes_left
  * is different than from the main stream. Don't use bytes_left to compute
  * any pointers.
  */

tests/Makefile

@@ -70,6 +70,9 @@ run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 te
 	[ "`./test_encode2 | ./test_decode2`" = \
 	"`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ]
 	
+	[ "`./test_decode2 < person_with_extra_field.pb`" = \
+	"`cat person_with_extra_field.txt`" ]
+	
 	[ "`./test_encode_callbacks | ./test_decode_callbacks`" = \
 	"`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ]
 

tests/person_with_extra_field.txt

@@ -0,0 +1,3 @@
+name: "Test Person 99"
+id: 99
+email: "test@person.com"

tests/test_decode2.c

@@ -59,13 +59,6 @@ bool callback(pb_istream_t *stream, uint8_t *buf, size_t count)
     FILE *file = (FILE*)stream->state;
     bool status;
     
-    if (buf == NULL)
-    {
-       /* Skipping data */
-        while (count-- && fgetc(file) != EOF);
-        return count == 0;
-    }
-    
     status = (fread(buf, 1, count, file) == count);
     
     if (feof(file))