Modified nanopb_generator.py to generate includes for other .proto files.

Implementation was suggested by extremeblue99.
Fixes issue 4.

docs/concepts.rst

@@ -158,7 +158,10 @@ required bytes data = 1 [(nanopb).max_size = 40];
                                                                                 | Person_data_t data;
 =============================================================================== =======================
 
-The maximum lengths are checked in runtime. If string/bytes/array exceeds the allocated length, *pb_decode* will return false. 
+The maximum lengths are checked in runtime. If string/bytes/array exceeds the allocated length, *pb_decode* will return false.
+
+Note: for the *bytes* datatype, the field length checking may not be exact.
+The compiler may add some padding to the *pb_bytes_t* structure, and the nanopb runtime doesn't know how much of the structure size is padding. Therefore it uses the whole length of the structure for storing data, which is not very smart but shouldn't cause problems. In practise, this means that if you specify *(nanopb).max_size=5* on a *bytes* field, you may be able to store 6 bytes there. For the *string* field type, the length limit is exact.
 
 Field callbacks
 ===============

docs/reference.rst

@@ -376,7 +376,12 @@ Because of memory concerns, the detection of missing required fields is not perf
 
     Each field decoder reads and decodes a single value. For arrays, the decoder is called repeatedly.
 
-    You can use the decoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible with most of the field decoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type you pass to *dest*.
+    You can use the decoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible 
+    with the *varint* field decoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type 
+    you pass to *dest*, e.g. *field.data_size = sizeof(int);*. Other fields in the *pb_field_t* don't matter.
+
+    The field decoder interface is a bit messy as a result of the interface required inside the nanopb library.
+    Eventually they may be replaced by separate wrapper functions with a more friendly interface.
 
 pb_dec_varint
 -------------
@@ -403,12 +408,12 @@ pb_dec_fixed32
 --------------
 Field decoder for PB_LTYPE_FIXED32. ::
 
-    bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest);
+    bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest);
 
-:stream:        Input stream to read from. 1-10 bytes will be read.
+:stream:        Input stream to read from. 4 bytes will be read.
 :field:         Not used.
-:dest:          Pointer to destination integer. Must have size of *field->data_size* bytes.
+:dest:          Pointer to destination *int32_t*, *uint32_t* or *float*.
-:returns:       True on success, false on IO errors or if `pb_decode_varint`_ fails.
+:returns:       True on success, false on IO errors.
 
 This function reads 4 bytes from the input stream.
 On big endian architectures, it then reverses the order of the bytes.
@@ -420,6 +425,11 @@ Field decoder for PB_LTYPE_FIXED64. ::
 
     bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest);
 
+:stream:        Input stream to read from. 8 bytes will be read.
+:field:         Not used.
+:dest:          Pointer to destination *int64_t*, *uint64_t* or *double*.
+:returns:       True on success, false on IO errors.
+
 Same as `pb_dec_fixed32`_, except this reads 8 bytes.
 
 pb_dec_bytes
@@ -428,6 +438,9 @@ Field decoder for PB_LTYPE_BYTES. Reads a length-prefixed block of bytes. ::
 
     bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest);
 
+**Note:** This is an internal function that is not useful in decoder callbacks. To read bytes fields in callbacks, use 
+*stream->bytes_left* and `pb_read`_.
+
 :stream:        Input stream to read from.
 :field:         Field description structure. Only *field->data_size* matters.
 :dest:          Pointer to a structure similar to pb_bytes_array_t.
@@ -441,6 +454,9 @@ Field decoder for PB_LTYPE_STRING. Reads a length-prefixed string. ::
 
     bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest);
 
+**Note:** This is an internal function that is not useful in decoder callbacks. To read string fields in callbacks, use 
+*stream->bytes_left* and `pb_read`_.
+
 :stream:        Input stream to read from.
 :field:         Field description structure. Only *field->data_size* matters.
 :dest:          Pointer to a character array of size *field->data_size*.
@@ -454,6 +470,9 @@ Field decoder for PB_LTYPE_SUBMESSAGE. Calls `pb_decode`_ to perform the actual
 
     bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest)
 
+**Note:** This is an internal function that is not useful in decoder callbacks. To read submessage fields in callbacks, use 
+`pb_decode`_ directly.
+
 :stream:        Input stream to read from.
 :field:         Field description structure. Only *field->ptr* matters.
 :dest:          Pointer to the destination structure.

generator/nanopb_generator.py

@@ -14,8 +14,8 @@ datatypes = {
     FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED32'),
     FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT'),
     FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT'),
-    FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED'),
+    FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED32'),
-    FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED'),
+    FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED64'),
     FieldD.TYPE_SINT32: ('int32_t', 'PB_LTYPE_SVARINT'),
     FieldD.TYPE_SINT64: ('int64_t', 'PB_LTYPE_SVARINT'),
     FieldD.TYPE_UINT32: ('uint32_t', 'PB_LTYPE_VARINT'),
@@ -219,9 +219,6 @@ class Field:
             result += '\n    pb_membersize(%s, %s[0]),' % (self.struct_name, self.name)
             result += ('\n    pb_membersize(%s, %s) / pb_membersize(%s, %s[0]),'
                        % (self.struct_name, self.name, self.struct_name, self.name))
-        elif self.htype != 'PB_HTYPE_CALLBACK' and self.ltype == 'PB_LTYPE_BYTES':
-            result += '\n    pb_membersize(%s, bytes),' % self.ctype
-            result += ' 0,'
         else:
             result += '\n    pb_membersize(%s, %s),' % (self.struct_name, self.name)
             result += ' 0,'
@@ -350,7 +347,7 @@ def sort_dependencies(messages):
         if msgname in message_by_name:
             yield message_by_name[msgname]
 
-def generate_header(headername, enums, messages):
+def generate_header(dependencies, headername, enums, messages):
     '''Generate content for a header file.
     Generates strings, which should be concatenated and stored to file.
     '''
@@ -362,6 +359,11 @@ def generate_header(headername, enums, messages):
     yield '#define _PB_%s_\n' % symbol
     yield '#include <pb.h>\n\n'
     
+    for dependency in dependencies:
+        noext = os.path.splitext(dependency)[0]
+        yield '#include "%s.pb.h"\n' % noext
+    yield '\n'
+    
     yield '/* Enum definitions */\n'
     for enum in enums:
         yield str(enum) + '\n\n'
@@ -407,7 +409,7 @@ if __name__ == '__main__':
         print "Output fill be written to file.pb.h and file.pb.c"
         sys.exit(1)
     
-    data = open(sys.argv[1]).read()
+    data = open(sys.argv[1], 'rb').read()
     fdesc = descriptor.FileDescriptorSet.FromString(data)
     enums, messages = parse_file(fdesc.file[0])
     
@@ -418,8 +420,13 @@ if __name__ == '__main__':
     
     print "Writing to " + headername + " and " + sourcename
     
+    # List of .proto files that should not be included in the C header file
+    # even if they are mentioned in the source .proto.
+    excludes = ['nanopb.proto']
+    dependencies = [d for d in fdesc.file[0].dependency if d not in excludes]
+    
     header = open(headername, 'w')
-    for part in generate_header(headerbasename, enums, messages):
+    for part in generate_header(dependencies, headerbasename, enums, messages):
         header.write(part)
 
     source = open(sourcename, 'w')

pb_decode.c

@@ -509,7 +509,8 @@ bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, voi
         return false;
     x->size = temp;
     
-    if (x->size > field->data_size)
+    /* Check length, noting the space taken by the size_t header. */
+    if (x->size > field->data_size - offsetof(pb_bytes_array_t, bytes))
         return false;
     
     return pb_read(stream, x->bytes, x->size);
@@ -522,6 +523,7 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo
     if (!pb_decode_varint32(stream, &size))
         return false;
     
+    /* Check length, noting the null terminator */
     if (size > field->data_size - 1)
         return false;
     

tests/Makefile

@@ -1,12 +1,12 @@
 CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage
 LDFLAGS=--coverage
-DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h
+DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h
 TESTS=test_decode1 test_encode1 decode_unittests encode_unittests
 
 all: breakpoints $(TESTS) run_unittests
 
 clean:
-	rm -f $(TESTS) person.pb* *.o *.gcda *.gcno
+	rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno
 
 %.o: %.c
 %.o: %.c $(DEPS)
@@ -19,8 +19,10 @@ pb_decode.o: ../pb_decode.c $(DEPS)
 
 test_decode1: test_decode1.o pb_decode.o person.pb.o
 test_decode2: test_decode2.o pb_decode.o person.pb.o
+test_decode3: test_decode3.o pb_decode.o alltypes.pb.o
 test_encode1: test_encode1.o pb_encode.o person.pb.o
 test_encode2: test_encode2.o pb_encode.o person.pb.o
+test_encode3: test_encode3.o pb_encode.o alltypes.pb.o
 test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o
 test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o
 decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o
@@ -39,7 +41,7 @@ coverage: run_unittests
 	gcov pb_encode.gcda
 	gcov pb_decode.gcda
 
-run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_decode1 test_decode2 test_encode_callbacks test_decode_callbacks
+run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks
 	rm -f *.gcda
 	
 	./decode_unittests > /dev/null
@@ -57,5 +59,8 @@ run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_
 	[ "`./test_encode_callbacks | ./test_decode_callbacks`" = \
 	"`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ]
 
+	./test_encode3 | ./test_decode3
+	./test_encode3 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null
+
 run_fuzztest: test_decode2
 	bash -c 'I=1; while true; do cat /dev/urandom | ./test_decode2 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done'

tests/alltypes.proto

@@ -0,0 +1,40 @@
+import "nanopb.proto";
+
+message SubMessage {
+    required string substuff1 = 1 [(nanopb).max_size = 16];
+    required int32 substuff2 = 2;
+}
+
+enum MyEnum {
+    First = 1;
+    Second = 2;
+    Truth = 42;
+}
+
+message AllTypes {
+    required int32      req_int32   = 1;
+    required int64      req_int64   = 2;
+    required uint32     req_uint32  = 3;
+    required uint64     req_uint64  = 4;
+    required sint32     req_sint32  = 5;
+    required sint64     req_sint64  = 6;
+    required bool       req_bool    = 7;
+    
+    required fixed32    req_fixed32 = 8;
+    required sfixed32   req_sfixed32= 9;
+    required float      req_float   = 10;
+    
+    required fixed64    req_fixed64 = 11;
+    required sfixed64   req_sfixed64= 12;
+    required double     req_double  = 13;
+    
+    required string     req_string  = 14 [(nanopb).max_size = 16];
+    required bytes      req_bytes   = 15 [(nanopb).max_size = 16];
+    required SubMessage req_submsg  = 16;
+    required MyEnum     req_enum    = 17;
+    
+    // Just to make sure that the size of the fields has been calculated
+    // properly, i.e. otherwise a bug in last field might not be detected.
+    required int32      end = 99;
+}
+

tests/decode_unittests.c

@@ -167,14 +167,22 @@ int main()
     {
         pb_istream_t s;
         struct { size_t size; uint8_t bytes[5]; } d;
-        pb_field_t f = {1, PB_LTYPE_BYTES, 0, 0, 5, 0, 0};
+        pb_field_t f = {1, PB_LTYPE_BYTES, 0, 0, sizeof(d), 0, 0};
         
         COMMENT("Test pb_dec_bytes")
         TEST((s = S("\x00"), pb_dec_bytes(&s, &f, &d) && d.size == 0))
         TEST((s = S("\x01\xFF"), pb_dec_bytes(&s, &f, &d) && d.size == 1 && d.bytes[0] == 0xFF))
-        TEST((s = S("\x06xxxxxx"), !pb_dec_bytes(&s, &f, &d)))
         TEST((s = S("\x05xxxxx"), pb_dec_bytes(&s, &f, &d) && d.size == 5))
         TEST((s = S("\x05xxxx"), !pb_dec_bytes(&s, &f, &d)))
+        
+        /* Note: the size limit on bytes-fields is not strictly obeyed, as
+         * the compiler may add some padding to the struct. Using this padding
+         * is not a very good thing to do, but it is difficult to avoid when
+         * we use only a single uint8_t to store the size of the field.
+         * Therefore this tests against a 10-byte string, while otherwise even
+         * 6 bytes should error out.
+         */
+        TEST((s = S("\x10xxxxxxxxxx"), !pb_dec_bytes(&s, &f, &d)))
     }
     
     {

tests/test_decode3.c

@@ -0,0 +1,70 @@
+/* Tests the decoding of all types. Currently only in the 'required' variety.
+ * This is the counterpart of test_encode3.
+ * Run e.g. ./test_encode3 | ./test_decode3
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <pb_decode.h>
+#include "alltypes.pb.h"
+
+#define TEST(x) if (!(x)) { \
+    printf("Test " #x " failed.\n"); \
+    return false; \
+    }
+
+/* This function is called once from main(), it handles
+   the decoding and checks the fields. */
+bool check_alltypes(pb_istream_t *stream)
+{
+    AllTypes alltypes = {};
+    
+    if (!pb_decode(stream, AllTypes_fields, &alltypes))
+        return false;
+    
+    TEST(alltypes.req_int32         == 1001);
+    TEST(alltypes.req_int64         == 1002);
+    TEST(alltypes.req_uint32        == 1003);
+    TEST(alltypes.req_uint64        == 1004);
+    TEST(alltypes.req_sint32        == 1005);
+    TEST(alltypes.req_sint64        == 1006);
+    TEST(alltypes.req_bool          == true);
+    
+    TEST(alltypes.req_fixed32       == 1008);
+    TEST(alltypes.req_sfixed32      == 1009);
+    TEST(alltypes.req_float         == 1010.0f);
+    
+    TEST(alltypes.req_fixed64       == 1011);
+    TEST(alltypes.req_sfixed64      == 1012);
+    TEST(alltypes.req_double        == 1013.0f);
+    
+    TEST(strcmp(alltypes.req_string, "1014") == 0);
+    TEST(alltypes.req_bytes.size == 4);
+    TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0);
+    TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0);
+    TEST(alltypes.req_submsg.substuff2 == 1016);
+    TEST(alltypes.req_enum == MyEnum_Truth);
+    
+    TEST(alltypes.end == 1099);
+    
+    return true;
+}
+
+int main()
+{
+    /* Read the data into buffer */
+    uint8_t buffer[512];
+    size_t count = fread(buffer, 1, sizeof(buffer), stdin);
+    
+    /* Construct a pb_istream_t for reading from the buffer */
+    pb_istream_t stream = pb_istream_from_buffer(buffer, count);
+    
+    /* Decode and print out the stuff */
+    if (!check_alltypes(&stream))
+    {
+        printf("Parsing failed.\n");
+        return 1;
+    } else {
+        return 0;
+    }
+}

tests/test_encode3.c

@@ -0,0 +1,50 @@
+/* Attempts to test all the datatypes supported by ProtoBuf.
+ * Currently only tests the 'required' variety.
+ */
+
+#include <stdio.h>
+#include <pb_encode.h>
+#include "alltypes.pb.h"
+
+int main()
+{
+    /* Initialize the structure with constants */
+    AllTypes alltypes = {
+        1001,
+        1002,
+        1003,
+        1004,
+        1005,
+        1006,
+        true,
+        
+        1008,
+        1009,
+        1010.0f,
+        
+        1011,
+        1012,
+        1013.0,
+        
+        "1014",
+        {4, "1015"},
+        {"1016", 1016},
+        MyEnum_Truth,
+        
+        1099
+    };
+    
+    uint8_t buffer[512];
+    pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+    
+    /* Now encode it and check if we succeeded. */
+    if (pb_encode(&stream, AllTypes_fields, &alltypes))
+    {
+        fwrite(buffer, 1, stream.bytes_written, stdout);
+        return 0; /* Success */
+    }
+    else
+    {
+        return 1; /* Failure */
+    }
+}