Publishing nanopb-0.2.9.2

Also backports cdbf51 generator bugfix.

.gitignore

@@ -5,17 +5,24 @@
 *.pb.c
 *.pb.h
 *.pb
+*.pyc
+*_pb2.py
 *~
 *.tar.gz
 .sconsign.dblite
+config.log
+.sconf_temp
+tests/build
 julkaisu.txt
+dist
 docs/*.html
 docs/generator_flow.png
-example/client
-example/server
-example_avr_double/decode_double
-example_avr_double/encode_double
-example_avr_double/test_conversions
-example_unions/decode
-example_unions/encode
+examples/simple/simple
+examples/network_server/client
+examples/network_server/server
+examples/using_double_on_avr/decode_double
+examples/using_double_on_avr/encode_double
+examples/using_double_on_avr/test_conversions
+examples/using_union_messages/decode
+examples/using_union_messages/encode
 generator/nanopb_pb2.pyc

CHANGELOG.txt

@@ -1,3 +1,54 @@
+nanopb-0.2.9.2 (2015-01-24)
+ Fix memory leaks with PB_ENABLE_MALLOC with some submessage hierarchies (issue 138)
+ Fix compilation error with generated initializers for repeated pointer fields
+
+nanopb-0.2.9.1 (2014-09-11)
+ Fix security issue due to size_t overflows. (issue 132)
+ Fix memory leak with duplicated fields and PB_ENABLE_MALLOC
+ Fix crash if pb_release() is called twice.
+
+nanopb-0.2.9 (2014-08-09)
+ NOTE: If you are using the -e option with the generator, you have
+ to prepend . to the argument to get the same behaviour as before.
+
+ Do not automatically add a dot with generator -e option. (issue 122)
+ Fix problem with .options file and extension fields. (issue 125)
+ Don't use SIZE_MAX macro, as it is not in C89. (issue 120)
+ Generate #defines for initializing message structures. (issue 79)
+ Add skip_message option to generator. (issue 121)
+ Add PB_PACKED_STRUCT support for Keil MDK-ARM toolchain (issue 119)
+ Give better messages about the .options file path. (issue 124)
+ Improved tests
+
+nanopb-0.2.8 (2014-05-20)
+ Fix security issue with PB_ENABLE_MALLOC. (issue 117)
+ Add option to not add timestamps to .pb.h and .pb.c preambles. (issue 115)
+ Documentation updates
+ Improved tests
+
+nanopb-0.2.7 (2014-04-07)
+ Fix bug with default values for extension fields (issue 111)
+ Fix some MISRA-C warnings (issue 91)
+ Implemented optional malloc() support (issue 80)
+ Changed pointer-type bytes field datatype
+ Add a "found" field to pb_extension_t (issue 112)
+ Add convenience function pb_get_encoded_size() (issue 16)
+
+nanopb-0.2.6 (2014-02-15)
+ Fix generator error with bytes callback fields (issue 99)
+ Fix warnings about large integer constants (issue 102)
+ Add comments to where STATIC_ASSERT is used (issue 96)
+ Add warning about unknown field names on .options (issue 105)
+ Move descriptor.proto to google/protobuf subdirectory (issue 104)
+ Improved tests
+
+nanopb-0.2.5 (2014-01-01)
+ Fix a bug with encoding negative values in int32 fields (issue 97)
+ Create binary packages of the generator + dependencies (issue 47)
+ Add support for pointer-type fields to the encoder (part of issue 80)
+ Fixed path in FindNanopb.cmake (issue 94)
+ Improved tests
+
 nanopb-0.2.4 (2013-11-07)
  Remove the deprecated NANOPB_INTERNALS functions from public API.
  Document the security model.

README

@@ -1,11 +0,0 @@
-Nanopb is a small code-size Protocol Buffers implementation.
-
-Homepage: http://kapsi.fi/~jpa/nanopb/
-
-To compile the library, you'll need these libraries:
-protobuf-compiler python-protobuf libprotobuf-dev
-
-The only runtime dependencies are memset() and memcpy().
-
-To run the tests, run make under the tests folder.
-If it completes without error, everything is fine.

README.txt

@@ -0,0 +1,61 @@
+Nanopb is a small code-size Protocol Buffers implementation in ansi C. It is
+especially suitable for use in microcontrollers, but fits any memory
+restricted system.
+
+Homepage: http://kapsi.fi/~jpa/nanopb/
+
+
+
+
+Using the nanopb library
+========================
+To use the nanopb library, you need to do two things:
+
+1) Compile your .proto files for nanopb, using protoc.
+2) Include pb_encode.c and pb_decode.c in your project.
+
+The easiest way to get started is to study the project in "examples/simple".
+It contains a Makefile, which should work directly under most Linux systems.
+However, for any other kind of build system, see the manual steps in
+README.txt in that folder.
+
+
+
+Using the Protocol Buffers compiler (protoc)
+============================================
+The nanopb generator is implemented as a plugin for the Google's own protoc
+compiler. This has the advantage that there is no need to reimplement the
+basic parsing of .proto files. However, it does mean that you need the
+Google's protobuf library in order to run the generator.
+
+If you have downloaded a binary package for nanopb (either Windows, Linux or
+Mac OS X version), the 'protoc' binary is included in the 'generator-bin'
+folder. In this case, you are ready to go. Simply run this command:
+
+    generator-bin/protoc --nanopb_out=. myprotocol.proto
+
+However, if you are using a git checkout or a plain source distribution, you
+need to provide your own version of protoc and the Google's protobuf library.
+On Linux, the necessary packages are protobuf-compiler and python-protobuf.
+On Windows, you can either build Google's protobuf library from source or use
+one of the binary distributions of it. In either case, if you use a separate
+protoc, you need to manually give the path to nanopb generator:
+
+    protoc --plugin=protoc-gen-nanopb=nanopb/generator/protoc-gen-nanopb ...
+
+
+
+Running the tests
+=================
+If you want to perform further development of the nanopb core, or to verify
+its functionality using your compiler and platform, you'll want to run the
+test suite. The build rules for the test suite are implemented using Scons,
+so you need to have that installed. To run the tests:
+
+    cd tests
+    scons
+
+This will show the progress of various test cases. If the output does not
+end in an error, the test cases were successful.
+
+

docs/index.rst

@@ -47,7 +47,6 @@ Features and limitations
 
 **Limitations**
 
-#) User must provide callbacks when decoding arrays or strings without maximum size. Malloc support could be added as a separate module.
 #) Some speed has been sacrificed for code size.
 #) Encoding is focused on writing to streams. For memory buffers only it could be made more efficient.
 #) The deprecated Protocol Buffers feature called "groups" is not supported.
@@ -104,11 +103,12 @@ requires a few header files to be available:
 #) *stdbool.h*, for definition of *bool*
 
 If these header files do not come with your compiler, you can use the
-file *compat/pb_syshdr.h* instead. It contains an example of how to provide
+file *extra/pb_syshdr.h* instead. It contains an example of how to provide
 the dependencies. You may have to edit it a bit to suit your custom platform.
 
-To use the pb_syshdr.h, define *PB_SYSTEM_HEADER* to be the name of your custom
-header file. It should provide all the dependencies listed above.
+To use the pb_syshdr.h, define *PB_SYSTEM_HEADER* as *"pb_syshdr.h"* (including the quotes).
+Similarly, you can provide a custom include file, which should provide all the dependencies
+listed above.
 
 Running the test cases
 ======================

docs/reference.rst

@@ -24,10 +24,8 @@ __BIG_ENDIAN__                 Set this if your platform stores integers and
                                floats in big-endian format. Mixed-endian
                                systems (different layout for ints and floats)
                                are currently not supported.
-NANOPB_INTERNALS               Set this to expose the field encoder functions
-                               that are hidden since nanopb-0.1.3. Starting
-                               with nanopb-0.2.4, this flag does nothing. Use
-                               the newer functions that have better interface.
+PB_ENABLE_MALLOC               Set this to enable dynamic allocation support
+                               in the decoder.
 PB_MAX_REQUIRED_FIELDS         Maximum number of required fields to check for
                                presence. Default value is 64. Increases stack
                                usage 1 byte per every 8 fields. Compiler
@@ -77,8 +75,9 @@ max_count                      Allocated number of entries in arrays
                                (*repeated* fields).
 type                           Type of the generated field. Default value
                                is *FT_DEFAULT*, which selects automatically.
-                               You can use *FT_CALLBACK*, *FT_STATIC* or
-                               *FT_IGNORE* to force a callback field, a static
+                               You can use *FT_CALLBACK*, *FT_POINTER*,
+                               *FT_STATIC* or *FT_IGNORE* to force a callback
+                               field, a dynamically allocated field, a static
                                field or to completely ignore the field.
 long_names                     Prefix the enum name to the enum value in
                                definitions, i.e. *EnumName_EnumValue*. Enabled
@@ -417,6 +416,17 @@ Encodes the contents of a structure as a protocol buffers message and writes it
 
 Normally pb_encode simply walks through the fields description array and serializes each field in turn. However, submessages must be serialized twice: first to calculate their size and then to actually write them to output. This causes some constraints for callback fields, which must return the same data on every call.
 
+pb_encode_delimited
+-------------------
+Calculates the length of the message, encodes it as varint and then encodes the message. ::
+
+    bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct);
+
+(parameters are the same as for `pb_encode`_.)
+
+A common way to indicate the message length in Protocol Buffers is to prefix it with a varint.
+This function does this, and it is compatible with *parseDelimitedFrom* in Google's protobuf library.
+
 .. sidebar:: Encoding fields manually
 
     The functions with names *pb_encode_\** are used when dealing with callback fields. The typical reason for using callbacks is to have an array of unlimited size. In that case, `pb_encode`_ will call your callback function, which in turn will call *pb_encode_\** functions repeatedly to write out values.
@@ -579,6 +589,10 @@ In addition to EOF, the pb_decode implementation supports terminating a message
 
 For optional fields, this function applies the default value and sets *has_<field>* to false if the field is not present.
 
+If *PB_ENABLE_MALLOC* is defined, this function may allocate storage for any pointer type fields.
+In this case, you have to call `pb_release`_ to release the memory after you are done with the message.
+On error return `pb_decode` will release the memory itself.
+
 pb_decode_noinit
 ----------------
 Same as `pb_decode`_, except does not apply the default values to fields. ::
@@ -589,6 +603,35 @@ Same as `pb_decode`_, except does not apply the default values to fields. ::
 
 The destination structure should be filled with zeros before calling this function. Doing a *memset* manually can be slightly faster than using `pb_decode`_ if you don't need any default values.
 
+In addition to decoding a single message, this function can be used to merge two messages, so that
+values from previous message will remain if the new message does not contain a field.
+
+This function *will not* release the message even on error return. If you use *PB_ENABLE_MALLOC*,
+you will need to call `pb_release`_ yourself.
+
+pb_decode_delimited
+-------------------
+Same as `pb_decode`_, except that it first reads a varint with the length of the message. ::
+
+    bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct);
+
+(parameters are the same as for `pb_decode`_.)
+
+A common method to indicate message size in Protocol Buffers is to prefix it with a varint.
+This function is compatible with *writeDelimitedTo* in the Google's Protocol Buffers library.
+
+pb_release
+----------
+Releases any dynamically allocated fields.
+
+    void pb_release(const pb_field_t fields[], void *dest_struct);
+
+:fields:        A field description array. Usually autogenerated.
+:dest_struct:   Pointer to structure where data will be stored.
+
+This function is only available if *PB_ENABLE_MALLOC* is defined. It will release any
+pointer type fields in the structure and set the pointers to NULL.
+
 pb_skip_varint
 --------------
 Skip a varint_ encoded integer without decoding it. ::

examples/network_server/Makefile

@@ -1,19 +1,17 @@
-CFLAGS = -ansi -Wall -Werror -g -O0
+# Include the nanopb provided Makefile rules
+include ../../extra/nanopb.mk
 
-# Path to the nanopb root folder
-NANOPB_DIR = ../..
-DEPS = $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \
-       $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h $(NANOPB_DIR)/pb.h
+# Compiler flags to enable all warnings & debug info
+CFLAGS = -ansi -Wall -Werror -g -O0
 CFLAGS += -I$(NANOPB_DIR)
 
 all: server client
 
+.SUFFIXES:
+
 clean:
 	rm -f server client fileproto.pb.c fileproto.pb.h
 
-%: %.c $(DEPS) fileproto.pb.h fileproto.pb.c
-	$(CC) $(CFLAGS) -o $@ $< $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c fileproto.pb.c common.c
+%: %.c common.c fileproto.pb.c
+	$(CC) $(CFLAGS) -o $@ $^ $(NANOPB_CORE)
 
-fileproto.pb.c fileproto.pb.h: fileproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py
-	protoc -ofileproto.pb $<
-	python $(NANOPB_DIR)/generator/nanopb_generator.py fileproto.pb

examples/network_server/client.c

@@ -23,9 +23,13 @@
 #include "fileproto.pb.h"
 #include "common.h"
 
+/* This callback function will be called once for each filename received
+ * from the server. The filenames will be printed out immediately, so that
+ * no memory has to be allocated for them.
+ */
 bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void **arg)
 {
-    FileInfo fileinfo;
+    FileInfo fileinfo = {};
     
     if (!pb_decode(stream, FileInfo_fields, &fileinfo))
         return false;
@@ -35,14 +39,20 @@ bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void **ar
     return true;
 }
 
+/* This function sends a request to socket 'fd' to list the files in
+ * directory given in 'path'. The results received from server will
+ * be printed to stdout.
+ */
 bool listdir(int fd, char *path)
 {
-    ListFilesRequest request;
-    ListFilesResponse response;
-    pb_istream_t input = pb_istream_from_socket(fd);
+    /* Construct and send the request to server */
+    {
+        ListFilesRequest request = {};
         pb_ostream_t output = pb_ostream_from_socket(fd);
         uint8_t zero = 0;
         
+        /* In our protocol, path is optional. If it is not given,
+         * the server will list the root directory. */
         if (path == NULL)
         {
             request.has_path = false;
@@ -59,15 +69,25 @@ bool listdir(int fd, char *path)
             strcpy(request.path, path);
         }
         
+        /* Encode the request. It is written to the socket immediately
+         * through our custom stream. */
         if (!pb_encode(&output, ListFilesRequest_fields, &request))
         {
-        fprintf(stderr, "Encoding failed.\n");
+            fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&output));
             return false;
         }
         
         /* We signal the end of request with a 0 tag. */
         pb_write(&output, &zero, 1);
+    }
     
+    /* Read back the response from server */
+    {
+        ListFilesResponse response = {};
+        pb_istream_t input = pb_istream_from_socket(fd);
+        
+        /* Give a pointer to our callback function, which will handle the
+         * filenames as they arrive. */
         response.file.funcs.decode = &printfile_callback;
         
         if (!pb_decode(&input, ListFilesResponse_fields, &response))
@@ -76,11 +96,14 @@ bool listdir(int fd, char *path)
             return false;
         }
         
+        /* If the message from server decodes properly, but directory was
+         * not found on server side, we get path_error == true. */
         if (response.path_error)
         {
             fprintf(stderr, "Server reported error.\n");
             return false;
         }
+    }
     
     return true;
 }
@@ -96,6 +119,7 @@ int main(int argc, char **argv)
     
     sockfd = socket(AF_INET, SOCK_STREAM, 0);
     
+    /* Connect to server running on localhost:1234 */
     memset(&servaddr, 0, sizeof(servaddr));
     servaddr.sin_family = AF_INET;
     servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
@@ -107,9 +131,11 @@ int main(int argc, char **argv)
         return 1;
     }
     
+    /* Send the directory listing request */
     if (!listdir(sockfd, path))
         return 2;
     
+    /* Close connection */
     close(sockfd);
     
     return 0;

examples/network_server/server.c

@@ -23,11 +23,16 @@
 #include "fileproto.pb.h"
 #include "common.h"
 
+/* This callback function will be called once during the encoding.
+ * It will write out any number of FileInfo entries, without consuming unnecessary memory.
+ * This is accomplished by fetching the filenames one at a time and encoding them
+ * immediately.
+ */
 bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
 {
     DIR *dir = (DIR*) *arg;
     struct dirent *file;
-    FileInfo fileinfo;
+    FileInfo fileinfo = {};
     
     while ((file = readdir(dir)) != NULL)
     {
@@ -35,9 +40,12 @@ bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * cons
         strncpy(fileinfo.name, file->d_name, sizeof(fileinfo.name));
         fileinfo.name[sizeof(fileinfo.name) - 1] = '\0';
         
+        /* This encodes the header for the field, based on the constant info
+         * from pb_field_t. */
         if (!pb_encode_tag_for_field(stream, field))
             return false;
         
+        /* This encodes the data for the field, based on our FileInfo structure. */
         if (!pb_encode_submessage(stream, FileInfo_fields, &fileinfo))
             return false;
     }
@@ -45,13 +53,18 @@ bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * cons
     return true;
 }
 
+/* Handle one arriving client connection.
+ * Clients are expected to send a ListFilesRequest, terminated by a '0'.
+ * Server will respond with a ListFilesResponse message.
+ */
 void handle_connection(int connfd)
 {
-    ListFilesRequest request;
-    ListFilesResponse response;
+    DIR *directory = NULL;
+    
+    /* Decode the message from the client and open the requested directory. */
+    {
+        ListFilesRequest request = {};
         pb_istream_t input = pb_istream_from_socket(connfd);
-    pb_ostream_t output = pb_ostream_from_socket(connfd);
-    DIR *directory;
         
         if (!pb_decode(&input, ListFilesRequest_fields, &request))
         {
@@ -60,19 +73,26 @@ void handle_connection(int connfd)
         }
         
         directory = opendir(request.path);
-    
         printf("Listing directory: %s\n", request.path);
+    }
+    
+    /* List the files in the directory and transmit the response to client */
+    {
+        ListFilesResponse response = {};
+        pb_ostream_t output = pb_ostream_from_socket(connfd);
         
         if (directory == NULL)
         {
             perror("opendir");
             
+            /* Directory was not found, transmit error status */
             response.has_path_error = true;
             response.path_error = true;
             response.file.funcs.encode = NULL;
         }
         else
         {
+            /* Directory was found, transmit filenames */
             response.has_path_error = false;
             response.file.funcs.encode = &listdir_callback;
             response.file.arg = directory;
@@ -80,8 +100,12 @@ void handle_connection(int connfd)
         
         if (!pb_encode(&output, ListFilesResponse_fields, &response))
         {
-        printf("Encoding failed.\n");
+            printf("Encoding failed: %s\n", PB_GET_ERROR(&output));
         }
+    }
+    
+    if (directory != NULL)
+        closedir(directory);
 }
 
 int main(int argc, char **argv)
@@ -90,8 +114,8 @@ int main(int argc, char **argv)
     struct sockaddr_in servaddr;
     int reuse = 1;
     
+    /* Listen on localhost:1234 for TCP connections */
     listenfd = socket(AF_INET, SOCK_STREAM, 0);
-    
     setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));
     
     memset(&servaddr, 0, sizeof(servaddr));
@@ -112,6 +136,7 @@ int main(int argc, char **argv)
     
     for(;;)
     {
+        /* Wait for a client */
         connfd = accept(listenfd, NULL, NULL);
         
         if (connfd < 0)
@@ -128,4 +153,6 @@ int main(int argc, char **argv)
         
         close(connfd);
     }
+    
+    return 0;
 }

examples/simple/Makefile

@@ -1,8 +1,8 @@
+# Include the nanopb provided Makefile rules
+include ../../extra/nanopb.mk
+
 # Compiler flags to enable all warnings & debug info
 CFLAGS = -Wall -Werror -g -O0
-
-# Path to the nanopb root folder
-NANOPB_DIR = ../..
 CFLAGS += -I$(NANOPB_DIR)
 
 # C source code files that are required
@@ -17,6 +17,5 @@ simple: $(CSRC)
 
 # Build rule for the protocol
 simple.pb.c: simple.proto
-	protoc -osimple.pb simple.proto
-	python $(NANOPB_DIR)/generator/nanopb_generator.py simple.pb
+	$(PROTOC) $(PROTOC_OPTS) --nanopb_out=. simple.proto
 

examples/simple/README.txt

@@ -18,10 +18,9 @@ On Linux, simply type "make" to build the example. After that, you can
 run it with the command: ./simple
 
 On other platforms, you first have to compile the protocol definition using
-the following two commands::
+the following command::
 
-  protoc -osimple.pb simple.proto
-  python nanopb_generator.py simple.pb
+  ../../generator-bin/protoc --nanopb_out=. simple.proto
 
 After that, add the following four files to your project and compile:
 

examples/using_double_on_avr/Makefile

@@ -1,27 +1,22 @@
-CFLAGS = -Wall -Werror -g -O0
+# Include the nanopb provided Makefile rules
+include ../../extra/nanopb.mk
 
-# Path to the nanopb root directory
-NANOPB_DIR = ../..
-DEPS = double_conversion.c $(NANOPB_DIR)/pb.h \
-	$(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \
-	$(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h
+# Compiler flags to enable all warnings & debug info
+CFLAGS = -Wall -Werror -g -O0
 CFLAGS += -I$(NANOPB_DIR)
 
 all: run_tests
 
+.SUFFIXES:
+
 clean:
 	rm -f test_conversions encode_double decode_double doubleproto.pb.c doubleproto.pb.h
 
 test_conversions: test_conversions.c double_conversion.c
 	$(CC) $(CFLAGS) -o $@ $^
 
-%: %.c $(DEPS) doubleproto.pb.h doubleproto.pb.c
-	$(CC) $(CFLAGS) -o $@ $< double_conversion.c \
-		$(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c doubleproto.pb.c
-
-doubleproto.pb.c doubleproto.pb.h: doubleproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py
-	protoc -odoubleproto.pb $<
-	python $(NANOPB_DIR)/generator/nanopb_generator.py doubleproto.pb
+%: %.c double_conversion.c doubleproto.pb.c
+	$(CC) $(CFLAGS) -o $@ $^ $(NANOPB_CORE)
 
 run_tests: test_conversions encode_double decode_double
 	./test_conversions

examples/using_union_messages/Makefile

@@ -1,9 +1,8 @@
-CFLAGS = -ansi -Wall -Werror -g -O0
+# Include the nanopb provided Makefile rules
+include ../../extra/nanopb.mk
 
-# Path to the nanopb root folder
-NANOPB_DIR = ../..
-DEPS = $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \
-       $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h $(NANOPB_DIR)/pb.h
+# Compiler flags to enable all warnings & debug info
+CFLAGS = -ansi -Wall -Werror -g -O0
 CFLAGS += -I$(NANOPB_DIR)
 
 all: encode decode
@@ -11,12 +10,11 @@ all: encode decode
 	./encode 2 | ./decode
 	./encode 3 | ./decode
 
+.SUFFIXES:
+
 clean:
 	rm -f encode unionproto.pb.h unionproto.pb.c
 
-%: %.c $(DEPS) unionproto.pb.h unionproto.pb.c
-	$(CC) $(CFLAGS) -o $@ $< $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c unionproto.pb.c
+%: %.c unionproto.pb.c
+	$(CC) $(CFLAGS) -o $@ $^ $(NANOPB_CORE)
 
-unionproto.pb.h unionproto.pb.c: unionproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py
-	protoc -ounionproto.pb $<
-	python $(NANOPB_DIR)/generator/nanopb_generator.py unionproto.pb

extra/FindNanopb.cmake

@@ -1,4 +1,5 @@
-# Locate and configure the nanopb library.
+# This is an example script for use with CMake projects for locating and configuring
+# the nanopb library.
 #
 # The following varialbes have to be set:
 #
@@ -148,7 +149,7 @@ function(NANOPB_GENERATE_CPP SRCS HDRS)
              "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
       COMMAND python
       ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb
-      DEPENDS ${FIL_WE}.pb
+      DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb"
       COMMENT "Running nanopb generator on ${FIL_WE}.pb"
       VERBATIM )
   endforeach()

extra/nanopb.mk

@@ -0,0 +1,37 @@
+# This is an include file for Makefiles. It provides rules for building
+# .pb.c and .pb.h files out of .proto, as well the path to nanopb core.
+
+# Path to the nanopb root directory
+NANOPB_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))../)
+
+# Files for the nanopb core
+NANOPB_CORE = $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_decode.c
+
+# Check if we are running on Windows
+ifdef windir
+WINDOWS = 1
+endif
+ifdef WINDIR
+WINDOWS = 1
+endif
+
+# Check whether to use binary version of nanopb_generator or the
+# system-supplied python interpreter.
+ifneq "$(wildcard $(NANOPB_DIR)/generator-bin)" ""
+	# Binary package
+	PROTOC = $(NANOPB_DIR)/generator-bin/protoc
+	PROTOC_OPTS = 
+else
+	# Source only or git checkout
+	PROTOC = protoc
+	ifdef WINDOWS
+		PROTOC_OPTS = --plugin=protoc-gen-nanopb=$(NANOPB_DIR)/generator/protoc-gen-nanopb.bat
+	else
+		PROTOC_OPTS = --plugin=protoc-gen-nanopb=$(NANOPB_DIR)/generator/protoc-gen-nanopb
+	endif
+endif
+
+# Rule for building .pb.c and .pb.h
+%.pb.c %.pb.h: %.proto $(wildcard %.options)
+	$(PROTOC) $(PROTOC_OPTS) --nanopb_out=. $<
+

extra/pb_syshdr.h

@@ -1,7 +1,7 @@
 /* This is an example of a header file for platforms/compilers that do
  * not come with stdint.h/stddef.h/stdbool.h/string.h. To use it, define
  * PB_SYSTEM_HEADER as "pb_syshdr.h", including the quotes, and add the
- * compat folder to your include path.
+ * extra folder to your include path.
  *
  * It is very likely that you will need to customize this file to suit
  * your platform. For any compiler that supports C99, this file should
@@ -53,6 +53,16 @@ typedef int bool;
 
 #endif
 
+/* stdlib.h subset */
+#ifdef PB_ENABLE_MALLOC
+#ifdef HAVE_STDLIB_H
+#include <stdlib.h>
+#else
+void *realloc(void *ptr, size_t size);
+void free(void *ptr);
+#endif
+#endif
+
 /* string.h subset */
 #ifdef HAVE_STRING_H
 #include <string.h>

generator/Makefile

@@ -1,5 +0,0 @@
-nanopb_pb2.py: nanopb.proto
-	protoc --python_out=. -I /usr/include -I . nanopb.proto
-
-plugin_pb2.py: plugin.proto
-	protoc --python_out=. -I /usr/include -I . plugin.proto

generator/nanopb_generator.py

@@ -1,36 +1,42 @@
 #!/usr/bin/python
 
 '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.'''
-nanopb_version = "nanopb-0.2.4"
+nanopb_version = "nanopb-0.2.9.2"
+
+import sys
+
+try:
+    # Add some dummy imports to keep packaging tools happy.
+    import google, distutils.util # bbfreeze seems to need these
+    import pkg_resources # pyinstaller / protobuf 2.5 seem to need these
+except:
+    # Don't care, we will error out later if it is actually important.
+    pass
 
 try:
-    import google.protobuf.descriptor_pb2 as descriptor
     import google.protobuf.text_format as text_format
+    import google.protobuf.descriptor_pb2 as descriptor
 except:
-    print
-    print "*************************************************************"
-    print "*** Could not import the Google protobuf Python libraries ***"
-    print "*** Try installing package 'python-protobuf' or similar.  ***"
-    print "*************************************************************"
-    print
+    sys.stderr.write('''
+         *************************************************************
+         *** Could not import the Google protobuf Python libraries ***
+         *** Try installing package 'python-protobuf' or similar.  ***
+         *************************************************************
+    ''' + '\n')
     raise
 
 try:
-    import nanopb_pb2
+    import proto.nanopb_pb2 as nanopb_pb2
+    import proto.plugin_pb2 as plugin_pb2
 except:
-    print
-    print "***************************************************************"
-    print "*** Could not import the precompiled nanopb_pb2.py.         ***"
-    print "*** Run 'make' in the 'generator' folder to update the file.***"
-    print "***************************************************************"
-    print
+    sys.stderr.write('''
+         ********************************************************************
+         *** Failed to import the protocol definitions for generator.     ***
+         *** You have to run 'make' in the nanopb/generator/proto folder. ***
+         ********************************************************************
+    ''' + '\n')
     raise
 
-
-
-
-
-
 # ---------------------------------------------------------------------------
 #                     Generation of single fields
 # ---------------------------------------------------------------------------
@@ -46,7 +52,7 @@ datatypes = {
     FieldD.TYPE_FIXED32:    ('uint32_t', 'FIXED32',     4),
     FieldD.TYPE_FIXED64:    ('uint64_t', 'FIXED64',     8),
     FieldD.TYPE_FLOAT:      ('float',    'FLOAT',       4),
-    FieldD.TYPE_INT32:      ('int32_t',  'INT32',       5),
+    FieldD.TYPE_INT32:      ('int32_t',  'INT32',      10),
     FieldD.TYPE_INT64:      ('int64_t',  'INT64',      10),
     FieldD.TYPE_SFIXED32:   ('int32_t',  'SFIXED32',    4),
     FieldD.TYPE_SFIXED64:   ('int64_t',  'SFIXED64',    8),
@@ -163,6 +169,7 @@ class Field:
         self.max_count = None
         self.array_decl = ""
         self.enc_size = None
+        self.ctype = None
         
         # Parse field options
         if field_options.HasField("max_size"):
@@ -189,6 +196,34 @@ class Field:
         else:
             raise NotImplementedError(desc.label)
         
+        # Check if the field can be implemented with static allocation
+        # i.e. whether the data size is known.
+        if desc.type == FieldD.TYPE_STRING and self.max_size is None:
+            can_be_static = False
+        
+        if desc.type == FieldD.TYPE_BYTES and self.max_size is None:
+            can_be_static = False
+        
+        # Decide how the field data will be allocated
+        if field_options.type == nanopb_pb2.FT_DEFAULT:
+            if can_be_static:
+                field_options.type = nanopb_pb2.FT_STATIC
+            else:
+                field_options.type = nanopb_pb2.FT_CALLBACK
+        
+        if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static:
+            raise Exception("Field %s is defined as static, but max_size or "
+                            "max_count is not given." % self.name)
+        
+        if field_options.type == nanopb_pb2.FT_STATIC:
+            self.allocation = 'STATIC'
+        elif field_options.type == nanopb_pb2.FT_POINTER:
+            self.allocation = 'POINTER'
+        elif field_options.type == nanopb_pb2.FT_CALLBACK:
+            self.allocation = 'CALLBACK'
+        else:
+            raise NotImplementedError(field_options.type)
+        
         # Decide the C data type to use in the struct.
         if datatypes.has_key(desc.type):
             self.ctype, self.pbtype, self.enc_size = datatypes[desc.type]
@@ -200,19 +235,18 @@ class Field:
             self.enc_size = 5 # protoc rejects enum values > 32 bits
         elif desc.type == FieldD.TYPE_STRING:
             self.pbtype = 'STRING'
-            if self.max_size is None:
-                can_be_static = False
-            else:
+            self.ctype = 'char'
+            if self.allocation == 'STATIC':
                 self.ctype = 'char'
                 self.array_decl += '[%d]' % self.max_size
                 self.enc_size = varint_max_size(self.max_size) + self.max_size
         elif desc.type == FieldD.TYPE_BYTES:
             self.pbtype = 'BYTES'
-            if self.max_size is None:
-                can_be_static = False
-            else:
+            if self.allocation == 'STATIC':
                 self.ctype = self.struct_name + self.name + 't'
                 self.enc_size = varint_max_size(self.max_size) + self.max_size
+            elif self.allocation == 'POINTER':
+                self.ctype = 'pb_bytes_array_t'
         elif desc.type == FieldD.TYPE_MESSAGE:
             self.pbtype = 'MESSAGE'
             self.ctype = self.submsgname = names_from_type_name(desc.type_name)
@@ -220,34 +254,30 @@ class Field:
         else:
             raise NotImplementedError(desc.type)
         
-        if field_options.type == nanopb_pb2.FT_DEFAULT:
-            if can_be_static:
-                field_options.type = nanopb_pb2.FT_STATIC
-            else:
-                field_options.type = nanopb_pb2.FT_CALLBACK
-        
-        if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static:
-            raise Exception("Field %s is defined as static, but max_size or max_count is not given." % self.name)
-        
-        if field_options.type == nanopb_pb2.FT_STATIC:
-            self.allocation = 'STATIC'
-        elif field_options.type == nanopb_pb2.FT_CALLBACK:
-            self.allocation = 'CALLBACK'
-            self.ctype = 'pb_callback_t'
-            self.array_decl = ''
-        else:
-            raise NotImplementedError(field_options.type)
-    
     def __cmp__(self, other):
         return cmp(self.tag, other.tag)
     
     def __str__(self):
-        if self.rules == 'OPTIONAL' and self.allocation == 'STATIC':
-            result = '    bool has_' + self.name + ';\n'
-        elif self.rules == 'REPEATED' and self.allocation == 'STATIC':
-            result = '    size_t ' + self.name + '_count;\n'
-        else:
         result = ''
+        if self.allocation == 'POINTER':
+            if self.rules == 'REPEATED':
+                result += '    size_t ' + self.name + '_count;\n'
+            
+            if self.pbtype == 'MESSAGE':
+                # Use struct definition, so recursive submessages are possible
+                result += '    struct _%s *%s;' % (self.ctype, self.name)
+            elif self.rules == 'REPEATED' and self.pbtype in ['STRING', 'BYTES']:
+                # String/bytes arrays need to be defined as pointers to pointers
+                result += '    %s **%s;' % (self.ctype, self.name)
+            else:
+                result += '    %s *%s;' % (self.ctype, self.name)
+        elif self.allocation == 'CALLBACK':
+            result += '    pb_callback_t %s;' % self.name
+        else:
+            if self.rules == 'OPTIONAL' and self.allocation == 'STATIC':
+                result += '    bool has_' + self.name + ';\n'
+            elif self.rules == 'REPEATED' and self.allocation == 'STATIC':
+                result += '    size_t ' + self.name + '_count;\n'
             result += '    %s %s%s;' % (self.ctype, self.name, self.array_decl)
         return result
     
@@ -262,30 +292,64 @@ class Field:
             result = None
         return result
     
+    def get_initializer(self, null_init):
+        '''Return literal expression for this field's default value.'''
+        
+        if self.pbtype == 'MESSAGE':
+            if null_init:
+                return '%s_init_zero' % self.ctype
+            else:
+                return '%s_init_default' % self.ctype
+        
+        if self.default is None or null_init:
+            if self.pbtype == 'STRING':
+                return '""'
+            elif self.pbtype == 'BYTES':
+                return '{0, {0}}'
+            elif self.pbtype == 'ENUM':
+                return '(%s)0' % self.ctype
+            else:
+                return '0'
+        
+        default = str(self.default)
+        
+        if self.pbtype == 'STRING':
+            default = default.encode('utf-8').encode('string_escape')
+            default = default.replace('"', '\\"')
+            default = '"' + default + '"'
+        elif self.pbtype == 'BYTES':
+            data = default.decode('string_escape')
+            data = ['0x%02x' % ord(c) for c in data]
+            if len(data) == 0:
+                default = '{0, {0}}'
+            else:
+                default = '{%d, {%s}}' % (len(data), ','.join(data))
+        elif self.pbtype in ['FIXED32', 'UINT32']:
+            default += 'u'
+        elif self.pbtype in ['FIXED64', 'UINT64']:
+            default += 'ull'
+        elif self.pbtype in ['SFIXED64', 'INT64']:
+            default += 'll'
+        
+        return default
+    
     def default_decl(self, declaration_only = False):
         '''Return definition for this field's default value.'''
         if self.default is None:
             return None
 
-        ctype, default = self.ctype, self.default
+        ctype = self.ctype
+        default = self.get_initializer(False)
         array_decl = ''
         
         if self.pbtype == 'STRING':
             if self.allocation != 'STATIC':
                 return None # Not implemented
-        
             array_decl = '[%d]' % self.max_size
-            default = str(self.default).encode('string_escape')
-            default = default.replace('"', '\\"')
-            default = '"' + default + '"'
         elif self.pbtype == 'BYTES':
             if self.allocation != 'STATIC':
                 return None # Not implemented
         
-            data = self.default.decode('string_escape')
-            data = ['0x%02x' % ord(c) for c in data]
-            default = '{%d, {%s}}' % (len(data), ','.join(data))
-        
         if declaration_only:
             return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl)
         else:
@@ -303,7 +367,7 @@ class Field:
         result  = '    PB_FIELD2(%3d, ' % self.tag
         result += '%-8s, ' % self.pbtype
         result += '%s, ' % self.rules
-        result += '%s, ' % self.allocation
+        result += '%-8s, ' % self.allocation
         result += '%s, ' % ("FIRST" if not prev_field_name else "OTHER")
         result += '%s, ' % self.struct_name
         result += '%s, ' % self.name
@@ -315,6 +379,8 @@ class Field:
             result += '0)'
         elif self.pbtype in ['BYTES', 'STRING'] and self.allocation != 'STATIC':
             result += '0)' # Arbitrary size default values not implemented
+        elif self.rules == 'OPTEXT':
+            result += '0)' # Default value for extensions is not implemented
         else:
             result += '&%s_default)' % (self.struct_name + self.name)
         
@@ -515,6 +581,34 @@ class Message:
                 result += types + '\n'
         return result
     
+    def get_initializer(self, null_init):
+        if not self.ordered_fields:
+            return '{0}'
+    
+        parts = []
+        for field in self.ordered_fields:
+            if field.allocation == 'STATIC':
+                if field.rules == 'REPEATED':
+                    parts.append('0')
+                    parts.append('{'
+                                 + ', '.join([field.get_initializer(null_init)] * field.max_count)
+                                 + '}')
+                elif field.rules == 'OPTIONAL':
+                    parts.append('false')
+                    parts.append(field.get_initializer(null_init))
+                else:
+                    parts.append(field.get_initializer(null_init))
+            elif field.allocation == 'POINTER':
+                if field.rules == 'REPEATED':
+                    parts.append('0')
+                parts.append('NULL')
+            elif field.allocation == 'CALLBACK':
+                if field.pbtype == 'EXTENSION':
+                    parts.append('NULL')
+                else:
+                    parts.append('{{NULL}, NULL}')
+        return '{' + ', '.join(parts) + '}'
+    
     def default_decl(self, declaration_only = False):
         result = ""
         for field in self.fields:
@@ -601,13 +695,17 @@ def parse_file(fdesc, file_options):
     
     for names, message in iterate_messages(fdesc, base_name):
         message_options = get_nanopb_suboptions(message, file_options, names)
+        
+        if message_options.skip_message:
+            continue
+        
         messages.append(Message(names, message, message_options))
         for enum in message.enum_type:
             enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name)
             enums.append(Enum(names, enum, enum_options))
     
     for names, extension in iterate_extensions(fdesc, base_name):
-        field_options = get_nanopb_suboptions(extension, file_options, names)
+        field_options = get_nanopb_suboptions(extension, file_options, names + extension.name)
         if field_options.type != nanopb_pb2.FT_IGNORE:
             extensions.append(ExtensionField(names, extension, field_options))
     
@@ -669,6 +767,9 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio
     '''
     
     yield '/* Automatically generated nanopb header */\n'
+    if options.notimestamp:
+        yield '/* Generated by %s */\n\n' % (nanopb_version)
+    else:
         yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
     
     symbol = make_identifier(headername)
@@ -683,7 +784,7 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio
     
     for dependency in dependencies:
         noext = os.path.splitext(dependency)[0]
-        yield options.genformat % (noext + '.' + options.extension + '.h')
+        yield options.genformat % (noext + options.extension + '.h')
         yield '\n'
 
     yield '#ifdef __cplusplus\n'
@@ -710,6 +811,15 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio
         yield msg.default_decl(True)
     yield '\n'
     
+    yield '/* Initializer values for message structs */\n'
+    for msg in messages:
+        identifier = '%s_init_default' % msg.name
+        yield '#define %-40s %s\n' % (identifier, msg.get_initializer(False))
+    for msg in messages:
+        identifier = '%s_init_zero' % msg.name
+        yield '#define %-40s %s\n' % (identifier, msg.get_initializer(True))
+    yield '\n'
+    
     yield '/* Field tags (for use in manual encoding/decoding) */\n'
     for msg in sort_dependencies(messages):
         for field in msg.fields:
@@ -742,6 +852,9 @@ def generate_source(headername, enums, messages, extensions, options):
     '''Generate content for a source file.'''
     
     yield '/* Automatically generated nanopb constant definitions */\n'
+    if options.notimestamp:
+        yield '/* Generated by %s */\n\n' % (nanopb_version)
+    else:
         yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
     yield options.genformat % (headername)
     yield '\n'
@@ -786,16 +899,6 @@ def generate_source(headername, enums, messages, extensions, options):
     if worst > 255 or checks:
         yield '\n/* Check that field information fits in pb_field_t */\n'
         
-        if worst < 65536:
-            yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n'
-            if worst > 255:
-                yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field
-            else:
-                assertion = ' && '.join(str(c) + ' < 256' for c in checks)
-                msgs = '_'.join(str(n) for n in checks_msgnames)
-                yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
-            yield '#endif\n\n'
-        
         if worst > 65535 or checks:
             yield '#if !defined(PB_FIELD_32BIT)\n'
             if worst > 65535:
@@ -803,8 +906,32 @@ def generate_source(headername, enums, messages, extensions, options):
             else:
                 assertion = ' && '.join(str(c) + ' < 65536' for c in checks)
                 msgs = '_'.join(str(n) for n in checks_msgnames)
+                yield '/* If you get an error here, it means that you need to define PB_FIELD_32BIT\n'
+                yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n'
+                yield ' * \n'
+                yield ' * The reason you need to do this is that some of your messages contain tag\n'
+                yield ' * numbers or field sizes that are larger than what can fit in 8 or 16 bit\n'
+                yield ' * field descriptors.\n'
+                yield ' */\n'
                 yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
-            yield '#endif\n'
+            yield '#endif\n\n'
+        
+        if worst < 65536:
+            yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n'
+            if worst > 255:
+                yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field
+            else:
+                assertion = ' && '.join(str(c) + ' < 256' for c in checks)
+                msgs = '_'.join(str(n) for n in checks_msgnames)
+                yield '/* If you get an error here, it means that you need to define PB_FIELD_16BIT\n'
+                yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n'
+                yield ' * \n'
+                yield ' * The reason you need to do this is that some of your messages contain tag\n'
+                yield ' * numbers or field sizes that are larger than what can fit in the default\n'
+                yield ' * 8 bit descriptors.\n'
+                yield ' */\n'
+                yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
+            yield '#endif\n\n'
     
     # Add check for sizeof(double)
     has_double = False
@@ -850,6 +977,7 @@ class Globals:
     '''Ugly global variables, should find a good way to pass these.'''
     verbose_options = False
     separate_options = []
+    matched_namemasks = set()
 
 def get_nanopb_suboptions(subdesc, options, name):
     '''Get copy of options, and merge information from subdesc.'''
@@ -860,6 +988,7 @@ def get_nanopb_suboptions(subdesc, options, name):
     dotname = '.'.join(name.parts)
     for namemask, options in Globals.separate_options:
         if fnmatch(dotname, namemask):
+            Globals.matched_namemasks.add(namemask)
             new_options.MergeFrom(options)
     
     # Handle options defined in .proto
@@ -879,8 +1008,8 @@ def get_nanopb_suboptions(subdesc, options, name):
         new_options.MergeFrom(ext)
     
     if Globals.verbose_options:
-        print "Options for " + dotname + ":"
-        print text_format.MessageToString(new_options)
+        sys.stderr.write("Options for " + dotname + ": ")
+        sys.stderr.write(text_format.MessageToString(new_options) + "\n")
     
     return new_options
 
@@ -899,8 +1028,8 @@ optparser = OptionParser(
              "Output will be written to file.pb.h and file.pb.c.")
 optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", default=[],
     help="Exclude file from generated #include list.")
-optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default="pb",
-    help="Set extension to use instead of 'pb' for generated files. [default: %default]")
+optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default=".pb",
+    help="Set extension to use instead of '.pb' for generated files. [default: %default]")
 optparser.add_option("-f", "--options-file", dest="options_file", metavar="FILE", default="%s.options",
     help="Set name of a separate generator options file.")
 optparser.add_option("-Q", "--generated-include-format", dest="genformat",
@@ -909,6 +1038,8 @@ optparser.add_option("-Q", "--generated-include-format", dest="genformat",
 optparser.add_option("-L", "--library-include-format", dest="libformat",
     metavar="FORMAT", default='#include <%s>\n',
     help="Set format string to use for including the nanopb pb.h header. [default: %default]")
+optparser.add_option("-T", "--no-timestamp", dest="notimestamp", action="store_true", default=False,
+    help="Don't add timestamp to .pb.h and .pb.c preambles")
 optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False,
     help="Don't print anything except errors.")
 optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
@@ -938,28 +1069,38 @@ def process_file(filename, fdesc, options):
         fdesc = descriptor.FileDescriptorSet.FromString(data).file[0]
     
     # Check if there is a separate .options file
+    had_abspath = False
     try:
         optfilename = options.options_file % os.path.splitext(filename)[0]
     except TypeError:
         # No %s specified, use the filename as-is
         optfilename = options.options_file
-    
-    if options.verbose:
-        print 'Reading options from ' + optfilename
+        had_abspath = True
 
     if os.path.isfile(optfilename):
+        if options.verbose:
+            sys.stderr.write('Reading options from ' + optfilename + '\n')
+
         Globals.separate_options = read_options_file(open(optfilename, "rU"))
     else:
+        # If we are given a full filename and it does not exist, give an error.
+        # However, don't give error when we automatically look for .options file
+        # with the same name as .proto.
+        if options.verbose or had_abspath:
+            sys.stderr.write('Options file not found: ' + optfilename)
+
         Globals.separate_options = []
 
+    Globals.matched_namemasks = set()
+    
     # Parse the file
     file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename]))
     enums, messages, extensions = parse_file(fdesc, file_options)
 
     # Decide the file names
     noext = os.path.splitext(filename)[0]
-    headername = noext + '.' + options.extension + '.h'
-    sourcename = noext + '.' + options.extension + '.c'
+    headername = noext + options.extension + '.h'
+    sourcename = noext + options.extension + '.c'
     headerbasename = os.path.basename(headername)
     
     # List of .proto files that should not be included in the C header file
@@ -973,6 +1114,14 @@ def process_file(filename, fdesc, options):
     sourcedata = ''.join(generate_source(headerbasename, enums,
                                          messages, extensions, options))
 
+    # Check if there were any lines in .options that did not match a member
+    unmatched = [n for n,o in Globals.separate_options if n not in Globals.matched_namemasks]
+    if unmatched and not options.quiet:
+        sys.stderr.write("Following patterns in " + optfilename + " did not match any fields: "
+                         + ', '.join(unmatched) + "\n")
+        if not Globals.verbose_options:
+            sys.stderr.write("Use  protoc --nanopb-out=-v:.   to see a list of the field names.\n")
+
     return {'headername': headername, 'headerdata': headerdata,
             'sourcename': sourcename, 'sourcedata': sourcedata}
     
@@ -994,7 +1143,8 @@ def main_cli():
         results = process_file(filename, None, options)
         
         if not options.quiet:
-            print "Writing to " + results['headername'] + " and " + results['sourcename']
+            sys.stderr.write("Writing to " + results['headername'] + " and "
+                             + results['sourcename'] + "\n")
     
         open(results['headername'], 'w').write(results['headerdata'])
         open(results['sourcename'], 'w').write(results['sourcedata'])        
@@ -1002,7 +1152,13 @@ def main_cli():
 def main_plugin():
     '''Main function when invoked as a protoc plugin.'''
 
-    import plugin_pb2
+    import sys
+    if sys.platform == "win32":
+        import os, msvcrt
+        # Set stdin and stdout to binary mode
+        msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
+        msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+    
     data = sys.stdin.read()
     request = plugin_pb2.CodeGeneratorRequest.FromString(data)
     
@@ -1010,10 +1166,7 @@ def main_plugin():
     args = shlex.split(request.parameter)
     options, dummy = optparser.parse_args(args)
     
-    # We can't go printing stuff to stdout
-    Globals.verbose_options = False
-    options.verbose = False
-    options.quiet = True
+    Globals.verbose_options = options.verbose
     
     response = plugin_pb2.CodeGeneratorResponse()
     
@@ -1034,7 +1187,7 @@ def main_plugin():
 
 if __name__ == '__main__':
     # Check if we are running as a plugin under protoc
-    if 'protoc-gen-' in sys.argv[0]:
+    if 'protoc-gen-' in sys.argv[0] or '--protoc-plugin' in sys.argv:
         main_plugin()
     else:
         main_cli()

generator/nanopb_pb2.py

@@ -1,158 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-
-from google.protobuf import descriptor
-from google.protobuf import message
-from google.protobuf import reflection
-from google.protobuf import descriptor_pb2
-# @@protoc_insertion_point(imports)
-
-
-import google.protobuf.descriptor_pb2
-
-DESCRIPTOR = descriptor.FileDescriptor(
-  name='nanopb.proto',
-  package='',
-  serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*J\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions')
-
-_FIELDTYPE = descriptor.EnumDescriptor(
-  name='FieldType',
-  full_name='FieldType',
-  filename=None,
-  file=DESCRIPTOR,
-  values=[
-    descriptor.EnumValueDescriptor(
-      name='FT_DEFAULT', index=0, number=0,
-      options=None,
-      type=None),
-    descriptor.EnumValueDescriptor(
-      name='FT_CALLBACK', index=1, number=1,
-      options=None,
-      type=None),
-    descriptor.EnumValueDescriptor(
-      name='FT_STATIC', index=2, number=2,
-      options=None,
-      type=None),
-    descriptor.EnumValueDescriptor(
-      name='FT_IGNORE', index=3, number=3,
-      options=None,
-      type=None),
-  ],
-  containing_type=None,
-  options=None,
-  serialized_start=199,
-  serialized_end=273,
-)
-
-
-FT_DEFAULT = 0
-FT_CALLBACK = 1
-FT_STATIC = 2
-FT_IGNORE = 3
-
-NANOPB_FILEOPT_FIELD_NUMBER = 1010
-nanopb_fileopt = descriptor.FieldDescriptor(
-  name='nanopb_fileopt', full_name='nanopb_fileopt', index=0,
-  number=1010, type=11, cpp_type=10, label=1,
-  has_default_value=False, default_value=None,
-  message_type=None, enum_type=None, containing_type=None,
-  is_extension=True, extension_scope=None,
-  options=None)
-NANOPB_MSGOPT_FIELD_NUMBER = 1010
-nanopb_msgopt = descriptor.FieldDescriptor(
-  name='nanopb_msgopt', full_name='nanopb_msgopt', index=1,
-  number=1010, type=11, cpp_type=10, label=1,
-  has_default_value=False, default_value=None,
-  message_type=None, enum_type=None, containing_type=None,
-  is_extension=True, extension_scope=None,
-  options=None)
-NANOPB_ENUMOPT_FIELD_NUMBER = 1010
-nanopb_enumopt = descriptor.FieldDescriptor(
-  name='nanopb_enumopt', full_name='nanopb_enumopt', index=2,
-  number=1010, type=11, cpp_type=10, label=1,
-  has_default_value=False, default_value=None,
-  message_type=None, enum_type=None, containing_type=None,
-  is_extension=True, extension_scope=None,
-  options=None)
-NANOPB_FIELD_NUMBER = 1010
-nanopb = descriptor.FieldDescriptor(
-  name='nanopb', full_name='nanopb', index=3,
-  number=1010, type=11, cpp_type=10, label=1,
-  has_default_value=False, default_value=None,
-  message_type=None, enum_type=None, containing_type=None,
-  is_extension=True, extension_scope=None,
-  options=None)
-
-
-_NANOPBOPTIONS = descriptor.Descriptor(
-  name='NanoPBOptions',
-  full_name='NanoPBOptions',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    descriptor.FieldDescriptor(
-      name='max_size', full_name='NanoPBOptions.max_size', index=0,
-      number=1, type=5, cpp_type=1, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='max_count', full_name='NanoPBOptions.max_count', index=1,
-      number=2, type=5, cpp_type=1, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='type', full_name='NanoPBOptions.type', index=2,
-      number=3, type=14, cpp_type=8, label=1,
-      has_default_value=True, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='long_names', full_name='NanoPBOptions.long_names', index=3,
-      number=4, type=8, cpp_type=7, label=1,
-      has_default_value=True, default_value=True,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='packed_struct', full_name='NanoPBOptions.packed_struct', index=4,
-      number=5, type=8, cpp_type=7, label=1,
-      has_default_value=True, default_value=False,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  extension_ranges=[],
-  serialized_start=51,
-  serialized_end=197,
-)
-
-_NANOPBOPTIONS.fields_by_name['type'].enum_type = _FIELDTYPE
-DESCRIPTOR.message_types_by_name['NanoPBOptions'] = _NANOPBOPTIONS
-
-class NanoPBOptions(message.Message):
-  __metaclass__ = reflection.GeneratedProtocolMessageType
-  DESCRIPTOR = _NANOPBOPTIONS
-  
-  # @@protoc_insertion_point(class_scope:NanoPBOptions)
-
-nanopb_fileopt.message_type = _NANOPBOPTIONS
-google.protobuf.descriptor_pb2.FileOptions.RegisterExtension(nanopb_fileopt)
-nanopb_msgopt.message_type = _NANOPBOPTIONS
-google.protobuf.descriptor_pb2.MessageOptions.RegisterExtension(nanopb_msgopt)
-nanopb_enumopt.message_type = _NANOPBOPTIONS
-google.protobuf.descriptor_pb2.EnumOptions.RegisterExtension(nanopb_enumopt)
-nanopb.message_type = _NANOPBOPTIONS
-google.protobuf.descriptor_pb2.FieldOptions.RegisterExtension(nanopb)
-# @@protoc_insertion_point(module_scope)

generator/plugin_pb2.py

@@ -1,161 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-
-from google.protobuf import descriptor
-from google.protobuf import message
-from google.protobuf import reflection
-from google.protobuf import descriptor_pb2
-# @@protoc_insertion_point(imports)
-
-
-import google.protobuf.descriptor_pb2
-
-DESCRIPTOR = descriptor.FileDescriptor(
-  name='plugin.proto',
-  package='google.protobuf.compiler',
-  serialized_pb='\n\x0cplugin.proto\x12\x18google.protobuf.compiler\x1a google/protobuf/descriptor.proto\"}\n\x14\x43odeGeneratorRequest\x12\x18\n\x10\x66ile_to_generate\x18\x01 \x03(\t\x12\x11\n\tparameter\x18\x02 \x01(\t\x12\x38\n\nproto_file\x18\x0f \x03(\x0b\x32$.google.protobuf.FileDescriptorProto\"\xaa\x01\n\x15\x43odeGeneratorResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\x42\n\x04\x66ile\x18\x0f \x03(\x0b\x32\x34.google.protobuf.compiler.CodeGeneratorResponse.File\x1a>\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x17\n\x0finsertion_point\x18\x02 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x0f \x01(\t')
-
-
-
-
-_CODEGENERATORREQUEST = descriptor.Descriptor(
-  name='CodeGeneratorRequest',
-  full_name='google.protobuf.compiler.CodeGeneratorRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    descriptor.FieldDescriptor(
-      name='file_to_generate', full_name='google.protobuf.compiler.CodeGeneratorRequest.file_to_generate', index=0,
-      number=1, type=9, cpp_type=9, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='parameter', full_name='google.protobuf.compiler.CodeGeneratorRequest.parameter', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=unicode("", "utf-8"),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='proto_file', full_name='google.protobuf.compiler.CodeGeneratorRequest.proto_file', index=2,
-      number=15, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  extension_ranges=[],
-  serialized_start=76,
-  serialized_end=201,
-)
-
-
-_CODEGENERATORRESPONSE_FILE = descriptor.Descriptor(
-  name='File',
-  full_name='google.protobuf.compiler.CodeGeneratorResponse.File',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    descriptor.FieldDescriptor(
-      name='name', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.name', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=unicode("", "utf-8"),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='insertion_point', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.insertion_point', index=1,
-      number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=unicode("", "utf-8"),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='content', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.content', index=2,
-      number=15, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=unicode("", "utf-8"),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  extension_ranges=[],
-  serialized_start=312,
-  serialized_end=374,
-)
-
-_CODEGENERATORRESPONSE = descriptor.Descriptor(
-  name='CodeGeneratorResponse',
-  full_name='google.protobuf.compiler.CodeGeneratorResponse',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  fields=[
-    descriptor.FieldDescriptor(
-      name='error', full_name='google.protobuf.compiler.CodeGeneratorResponse.error', index=0,
-      number=1, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=unicode("", "utf-8"),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    descriptor.FieldDescriptor(
-      name='file', full_name='google.protobuf.compiler.CodeGeneratorResponse.file', index=1,
-      number=15, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-  ],
-  extensions=[
-  ],
-  nested_types=[_CODEGENERATORRESPONSE_FILE, ],
-  enum_types=[
-  ],
-  options=None,
-  is_extendable=False,
-  extension_ranges=[],
-  serialized_start=204,
-  serialized_end=374,
-)
-
-_CODEGENERATORREQUEST.fields_by_name['proto_file'].message_type = google.protobuf.descriptor_pb2._FILEDESCRIPTORPROTO
-_CODEGENERATORRESPONSE_FILE.containing_type = _CODEGENERATORRESPONSE;
-_CODEGENERATORRESPONSE.fields_by_name['file'].message_type = _CODEGENERATORRESPONSE_FILE
-DESCRIPTOR.message_types_by_name['CodeGeneratorRequest'] = _CODEGENERATORREQUEST
-DESCRIPTOR.message_types_by_name['CodeGeneratorResponse'] = _CODEGENERATORRESPONSE
-
-class CodeGeneratorRequest(message.Message):
-  __metaclass__ = reflection.GeneratedProtocolMessageType
-  DESCRIPTOR = _CODEGENERATORREQUEST
-  
-  # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorRequest)
-
-class CodeGeneratorResponse(message.Message):
-  __metaclass__ = reflection.GeneratedProtocolMessageType
-  
-  class File(message.Message):
-    __metaclass__ = reflection.GeneratedProtocolMessageType
-    DESCRIPTOR = _CODEGENERATORRESPONSE_FILE
-    
-    # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse.File)
-  DESCRIPTOR = _CODEGENERATORRESPONSE
-  
-  # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse)
-
-# @@protoc_insertion_point(module_scope)

generator/proto/Makefile

@@ -0,0 +1,4 @@
+all: nanopb_pb2.py plugin_pb2.py
+
+%_pb2.py: %.proto
+	protoc --python_out=. $<

generator/proto/google/protobuf/descriptor.proto

@@ -0,0 +1,620 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// http://code.google.com/p/protobuf/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+//
+// The messages in this file describe the definitions found in .proto files.
+// A valid .proto file can be translated directly to a FileDescriptorProto
+// without any other information (e.g. without reading its imports).
+
+
+
+package google.protobuf;
+option java_package = "com.google.protobuf";
+option java_outer_classname = "DescriptorProtos";
+
+// descriptor.proto must be optimized for speed because reflection-based
+// algorithms don't work during bootstrapping.
+option optimize_for = SPEED;
+
+// The protocol compiler can output a FileDescriptorSet containing the .proto
+// files it parses.
+message FileDescriptorSet {
+  repeated FileDescriptorProto file = 1;
+}
+
+// Describes a complete .proto file.
+message FileDescriptorProto {
+  optional string name = 1;       // file name, relative to root of source tree
+  optional string package = 2;    // e.g. "foo", "foo.bar", etc.
+
+  // Names of files imported by this file.
+  repeated string dependency = 3;
+  // Indexes of the public imported files in the dependency list above.
+  repeated int32 public_dependency = 10;
+  // Indexes of the weak imported files in the dependency list.
+  // For Google-internal migration only. Do not use.
+  repeated int32 weak_dependency = 11;
+
+  // All top-level definitions in this file.
+  repeated DescriptorProto message_type = 4;
+  repeated EnumDescriptorProto enum_type = 5;
+  repeated ServiceDescriptorProto service = 6;
+  repeated FieldDescriptorProto extension = 7;
+
+  optional FileOptions options = 8;
+
+  // This field contains optional information about the original source code.
+  // You may safely remove this entire field whithout harming runtime
+  // functionality of the descriptors -- the information is needed only by
+  // development tools.
+  optional SourceCodeInfo source_code_info = 9;
+}
+
+// Describes a message type.
+message DescriptorProto {
+  optional string name = 1;
+
+  repeated FieldDescriptorProto field = 2;
+  repeated FieldDescriptorProto extension = 6;
+
+  repeated DescriptorProto nested_type = 3;
+  repeated EnumDescriptorProto enum_type = 4;
+
+  message ExtensionRange {
+    optional int32 start = 1;
+    optional int32 end = 2;
+  }
+  repeated ExtensionRange extension_range = 5;
+
+  optional MessageOptions options = 7;
+}
+
+// Describes a field within a message.
+message FieldDescriptorProto {
+  enum Type {
+    // 0 is reserved for errors.
+    // Order is weird for historical reasons.
+    TYPE_DOUBLE         = 1;
+    TYPE_FLOAT          = 2;
+    // Not ZigZag encoded.  Negative numbers take 10 bytes.  Use TYPE_SINT64 if
+    // negative values are likely.
+    TYPE_INT64          = 3;
+    TYPE_UINT64         = 4;
+    // Not ZigZag encoded.  Negative numbers take 10 bytes.  Use TYPE_SINT32 if
+    // negative values are likely.
+    TYPE_INT32          = 5;
+    TYPE_FIXED64        = 6;
+    TYPE_FIXED32        = 7;
+    TYPE_BOOL           = 8;
+    TYPE_STRING         = 9;
+    TYPE_GROUP          = 10;  // Tag-delimited aggregate.
+    TYPE_MESSAGE        = 11;  // Length-delimited aggregate.
+
+    // New in version 2.
+    TYPE_BYTES          = 12;
+    TYPE_UINT32         = 13;
+    TYPE_ENUM           = 14;
+    TYPE_SFIXED32       = 15;
+    TYPE_SFIXED64       = 16;
+    TYPE_SINT32         = 17;  // Uses ZigZag encoding.
+    TYPE_SINT64         = 18;  // Uses ZigZag encoding.
+  };
+
+  enum Label {
+    // 0 is reserved for errors
+    LABEL_OPTIONAL      = 1;
+    LABEL_REQUIRED      = 2;
+    LABEL_REPEATED      = 3;
+    // TODO(sanjay): Should we add LABEL_MAP?
+  };
+
+  optional string name = 1;
+  optional int32 number = 3;
+  optional Label label = 4;
+
+  // If type_name is set, this need not be set.  If both this and type_name
+  // are set, this must be either TYPE_ENUM or TYPE_MESSAGE.
+  optional Type type = 5;
+
+  // For message and enum types, this is the name of the type.  If the name
+  // starts with a '.', it is fully-qualified.  Otherwise, C++-like scoping
+  // rules are used to find the type (i.e. first the nested types within this
+  // message are searched, then within the parent, on up to the root
+  // namespace).
+  optional string type_name = 6;
+
+  // For extensions, this is the name of the type being extended.  It is
+  // resolved in the same manner as type_name.
+  optional string extendee = 2;
+
+  // For numeric types, contains the original text representation of the value.
+  // For booleans, "true" or "false".
+  // For strings, contains the default text contents (not escaped in any way).
+  // For bytes, contains the C escaped value.  All bytes >= 128 are escaped.
+  // TODO(kenton):  Base-64 encode?
+  optional string default_value = 7;
+
+  optional FieldOptions options = 8;
+}
+
+// Describes an enum type.
+message EnumDescriptorProto {
+  optional string name = 1;
+
+  repeated EnumValueDescriptorProto value = 2;
+
+  optional EnumOptions options = 3;
+}
+
+// Describes a value within an enum.
+message EnumValueDescriptorProto {
+  optional string name = 1;
+  optional int32 number = 2;
+
+  optional EnumValueOptions options = 3;
+}
+
+// Describes a service.
+message ServiceDescriptorProto {
+  optional string name = 1;
+  repeated MethodDescriptorProto method = 2;
+
+  optional ServiceOptions options = 3;
+}
+
+// Describes a method of a service.
+message MethodDescriptorProto {
+  optional string name = 1;
+
+  // Input and output type names.  These are resolved in the same way as
+  // FieldDescriptorProto.type_name, but must refer to a message type.
+  optional string input_type = 2;
+  optional string output_type = 3;
+
+  optional MethodOptions options = 4;
+}
+
+
+// ===================================================================
+// Options
+
+// Each of the definitions above may have "options" attached.  These are
+// just annotations which may cause code to be generated slightly differently
+// or may contain hints for code that manipulates protocol messages.
+//
+// Clients may define custom options as extensions of the *Options messages.
+// These extensions may not yet be known at parsing time, so the parser cannot
+// store the values in them.  Instead it stores them in a field in the *Options
+// message called uninterpreted_option. This field must have the same name
+// across all *Options messages. We then use this field to populate the
+// extensions when we build a descriptor, at which point all protos have been
+// parsed and so all extensions are known.
+//
+// Extension numbers for custom options may be chosen as follows:
+// * For options which will only be used within a single application or
+//   organization, or for experimental options, use field numbers 50000
+//   through 99999.  It is up to you to ensure that you do not use the
+//   same number for multiple options.
+// * For options which will be published and used publicly by multiple
+//   independent entities, e-mail protobuf-global-extension-registry@google.com
+//   to reserve extension numbers. Simply provide your project name (e.g.
+//   Object-C plugin) and your porject website (if available) -- there's no need
+//   to explain how you intend to use them. Usually you only need one extension
+//   number. You can declare multiple options with only one extension number by
+//   putting them in a sub-message. See the Custom Options section of the docs
+//   for examples:
+//   http://code.google.com/apis/protocolbuffers/docs/proto.html#options
+//   If this turns out to be popular, a web service will be set up
+//   to automatically assign option numbers.
+
+
+message FileOptions {
+
+  // Sets the Java package where classes generated from this .proto will be
+  // placed.  By default, the proto package is used, but this is often
+  // inappropriate because proto packages do not normally start with backwards
+  // domain names.
+  optional string java_package = 1;
+
+
+  // If set, all the classes from the .proto file are wrapped in a single
+  // outer class with the given name.  This applies to both Proto1
+  // (equivalent to the old "--one_java_file" option) and Proto2 (where
+  // a .proto always translates to a single class, but you may want to
+  // explicitly choose the class name).
+  optional string java_outer_classname = 8;
+
+  // If set true, then the Java code generator will generate a separate .java
+  // file for each top-level message, enum, and service defined in the .proto
+  // file.  Thus, these types will *not* be nested inside the outer class
+  // named by java_outer_classname.  However, the outer class will still be
+  // generated to contain the file's getDescriptor() method as well as any
+  // top-level extensions defined in the file.
+  optional bool java_multiple_files = 10 [default=false];
+
+  // If set true, then the Java code generator will generate equals() and
+  // hashCode() methods for all messages defined in the .proto file. This is
+  // purely a speed optimization, as the AbstractMessage base class includes
+  // reflection-based implementations of these methods.
+  optional bool java_generate_equals_and_hash = 20 [default=false];
+
+  // Generated classes can be optimized for speed or code size.
+  enum OptimizeMode {
+    SPEED = 1;        // Generate complete code for parsing, serialization,
+                      // etc.
+    CODE_SIZE = 2;    // Use ReflectionOps to implement these methods.
+    LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime.
+  }
+  optional OptimizeMode optimize_for = 9 [default=SPEED];
+
+  // Sets the Go package where structs generated from this .proto will be
+  // placed.  There is no default.
+  optional string go_package = 11;
+
+
+
+  // Should generic services be generated in each language?  "Generic" services
+  // are not specific to any particular RPC system.  They are generated by the
+  // main code generators in each language (without additional plugins).
+  // Generic services were the only kind of service generation supported by
+  // early versions of proto2.
+  //
+  // Generic services are now considered deprecated in favor of using plugins
+  // that generate code specific to your particular RPC system.  Therefore,
+  // these default to false.  Old code which depends on generic services should
+  // explicitly set them to true.
+  optional bool cc_generic_services = 16 [default=false];
+  optional bool java_generic_services = 17 [default=false];
+  optional bool py_generic_services = 18 [default=false];
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message MessageOptions {
+  // Set true to use the old proto1 MessageSet wire format for extensions.
+  // This is provided for backwards-compatibility with the MessageSet wire
+  // format.  You should not use this for any other reason:  It's less
+  // efficient, has fewer features, and is more complicated.
+  //
+  // The message must be defined exactly as follows:
+  //   message Foo {
+  //     option message_set_wire_format = true;
+  //     extensions 4 to max;
+  //   }
+  // Note that the message cannot have any defined fields; MessageSets only
+  // have extensions.
+  //
+  // All extensions of your type must be singular messages; e.g. they cannot
+  // be int32s, enums, or repeated messages.
+  //
+  // Because this is an option, the above two restrictions are not enforced by
+  // the protocol compiler.
+  optional bool message_set_wire_format = 1 [default=false];
+
+  // Disables the generation of the standard "descriptor()" accessor, which can
+  // conflict with a field of the same name.  This is meant to make migration
+  // from proto1 easier; new code should avoid fields named "descriptor".
+  optional bool no_standard_descriptor_accessor = 2 [default=false];
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message FieldOptions {
+  // The ctype option instructs the C++ code generator to use a different
+  // representation of the field than it normally would.  See the specific
+  // options below.  This option is not yet implemented in the open source
+  // release -- sorry, we'll try to include it in a future version!
+  optional CType ctype = 1 [default = STRING];
+  enum CType {
+    // Default mode.
+    STRING = 0;
+
+    CORD = 1;
+
+    STRING_PIECE = 2;
+  }
+  // The packed option can be enabled for repeated primitive fields to enable
+  // a more efficient representation on the wire. Rather than repeatedly
+  // writing the tag and type for each element, the entire array is encoded as
+  // a single length-delimited blob.
+  optional bool packed = 2;
+
+
+
+  // Should this field be parsed lazily?  Lazy applies only to message-type
+  // fields.  It means that when the outer message is initially parsed, the
+  // inner message's contents will not be parsed but instead stored in encoded
+  // form.  The inner message will actually be parsed when it is first accessed.
+  //
+  // This is only a hint.  Implementations are free to choose whether to use
+  // eager or lazy parsing regardless of the value of this option.  However,
+  // setting this option true suggests that the protocol author believes that
+  // using lazy parsing on this field is worth the additional bookkeeping
+  // overhead typically needed to implement it.
+  //
+  // This option does not affect the public interface of any generated code;
+  // all method signatures remain the same.  Furthermore, thread-safety of the
+  // interface is not affected by this option; const methods remain safe to
+  // call from multiple threads concurrently, while non-const methods continue
+  // to require exclusive access.
+  //
+  //
+  // Note that implementations may choose not to check required fields within
+  // a lazy sub-message.  That is, calling IsInitialized() on the outher message
+  // may return true even if the inner message has missing required fields.
+  // This is necessary because otherwise the inner message would have to be
+  // parsed in order to perform the check, defeating the purpose of lazy
+  // parsing.  An implementation which chooses not to check required fields
+  // must be consistent about it.  That is, for any particular sub-message, the
+  // implementation must either *always* check its required fields, or *never*
+  // check its required fields, regardless of whether or not the message has
+  // been parsed.
+  optional bool lazy = 5 [default=false];
+
+  // Is this field deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for accessors, or it will be completely ignored; in the very least, this
+  // is a formalization for deprecating fields.
+  optional bool deprecated = 3 [default=false];
+
+  // EXPERIMENTAL.  DO NOT USE.
+  // For "map" fields, the name of the field in the enclosed type that
+  // is the key for this map.  For example, suppose we have:
+  //   message Item {
+  //     required string name = 1;
+  //     required string value = 2;
+  //   }
+  //   message Config {
+  //     repeated Item items = 1 [experimental_map_key="name"];
+  //   }
+  // In this situation, the map key for Item will be set to "name".
+  // TODO: Fully-implement this, then remove the "experimental_" prefix.
+  optional string experimental_map_key = 9;
+
+  // For Google-internal migration only. Do not use.
+  optional bool weak = 10 [default=false];
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message EnumOptions {
+
+  // Set this option to false to disallow mapping different tag names to a same
+  // value.
+  optional bool allow_alias = 2 [default=true];
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message EnumValueOptions {
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message ServiceOptions {
+
+  // Note:  Field numbers 1 through 32 are reserved for Google's internal RPC
+  //   framework.  We apologize for hoarding these numbers to ourselves, but
+  //   we were already using them long before we decided to release Protocol
+  //   Buffers.
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message MethodOptions {
+
+  // Note:  Field numbers 1 through 32 are reserved for Google's internal RPC
+  //   framework.  We apologize for hoarding these numbers to ourselves, but
+  //   we were already using them long before we decided to release Protocol
+  //   Buffers.
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+
+// A message representing a option the parser does not recognize. This only
+// appears in options protos created by the compiler::Parser class.
+// DescriptorPool resolves these when building Descriptor objects. Therefore,
+// options protos in descriptor objects (e.g. returned by Descriptor::options(),
+// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions
+// in them.
+message UninterpretedOption {
+  // The name of the uninterpreted option.  Each string represents a segment in
+  // a dot-separated name.  is_extension is true iff a segment represents an
+  // extension (denoted with parentheses in options specs in .proto files).
+  // E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents
+  // "foo.(bar.baz).qux".
+  message NamePart {
+    required string name_part = 1;
+    required bool is_extension = 2;
+  }
+  repeated NamePart name = 2;
+
+  // The value of the uninterpreted option, in whatever type the tokenizer
+  // identified it as during parsing. Exactly one of these should be set.
+  optional string identifier_value = 3;
+  optional uint64 positive_int_value = 4;
+  optional int64 negative_int_value = 5;
+  optional double double_value = 6;
+  optional bytes string_value = 7;
+  optional string aggregate_value = 8;
+}
+
+// ===================================================================
+// Optional source code info
+
+// Encapsulates information about the original source file from which a
+// FileDescriptorProto was generated.
+message SourceCodeInfo {
+  // A Location identifies a piece of source code in a .proto file which
+  // corresponds to a particular definition.  This information is intended
+  // to be useful to IDEs, code indexers, documentation generators, and similar
+  // tools.
+  //
+  // For example, say we have a file like:
+  //   message Foo {
+  //     optional string foo = 1;
+  //   }
+  // Let's look at just the field definition:
+  //   optional string foo = 1;
+  //   ^       ^^     ^^  ^  ^^^
+  //   a       bc     de  f  ghi
+  // We have the following locations:
+  //   span   path               represents
+  //   [a,i)  [ 4, 0, 2, 0 ]     The whole field definition.
+  //   [a,b)  [ 4, 0, 2, 0, 4 ]  The label (optional).
+  //   [c,d)  [ 4, 0, 2, 0, 5 ]  The type (string).
+  //   [e,f)  [ 4, 0, 2, 0, 1 ]  The name (foo).
+  //   [g,h)  [ 4, 0, 2, 0, 3 ]  The number (1).
+  //
+  // Notes:
+  // - A location may refer to a repeated field itself (i.e. not to any
+  //   particular index within it).  This is used whenever a set of elements are
+  //   logically enclosed in a single code segment.  For example, an entire
+  //   extend block (possibly containing multiple extension definitions) will
+  //   have an outer location whose path refers to the "extensions" repeated
+  //   field without an index.
+  // - Multiple locations may have the same path.  This happens when a single
+  //   logical declaration is spread out across multiple places.  The most
+  //   obvious example is the "extend" block again -- there may be multiple
+  //   extend blocks in the same scope, each of which will have the same path.
+  // - A location's span is not always a subset of its parent's span.  For
+  //   example, the "extendee" of an extension declaration appears at the
+  //   beginning of the "extend" block and is shared by all extensions within
+  //   the block.
+  // - Just because a location's span is a subset of some other location's span
+  //   does not mean that it is a descendent.  For example, a "group" defines
+  //   both a type and a field in a single declaration.  Thus, the locations
+  //   corresponding to the type and field and their components will overlap.
+  // - Code which tries to interpret locations should probably be designed to
+  //   ignore those that it doesn't understand, as more types of locations could
+  //   be recorded in the future.
+  repeated Location location = 1;
+  message Location {
+    // Identifies which part of the FileDescriptorProto was defined at this
+    // location.
+    //
+    // Each element is a field number or an index.  They form a path from
+    // the root FileDescriptorProto to the place where the definition.  For
+    // example, this path:
+    //   [ 4, 3, 2, 7, 1 ]
+    // refers to:
+    //   file.message_type(3)  // 4, 3
+    //       .field(7)         // 2, 7
+    //       .name()           // 1
+    // This is because FileDescriptorProto.message_type has field number 4:
+    //   repeated DescriptorProto message_type = 4;
+    // and DescriptorProto.field has field number 2:
+    //   repeated FieldDescriptorProto field = 2;
+    // and FieldDescriptorProto.name has field number 1:
+    //   optional string name = 1;
+    //
+    // Thus, the above path gives the location of a field name.  If we removed
+    // the last element:
+    //   [ 4, 3, 2, 7 ]
+    // this path refers to the whole field declaration (from the beginning
+    // of the label to the terminating semicolon).
+    repeated int32 path = 1 [packed=true];
+
+    // Always has exactly three or four elements: start line, start column,
+    // end line (optional, otherwise assumed same as start line), end column.
+    // These are packed into a single field for efficiency.  Note that line
+    // and column numbers are zero-based -- typically you will want to add
+    // 1 to each before displaying to a user.
+    repeated int32 span = 2 [packed=true];
+
+    // If this SourceCodeInfo represents a complete declaration, these are any
+    // comments appearing before and after the declaration which appear to be
+    // attached to the declaration.
+    //
+    // A series of line comments appearing on consecutive lines, with no other
+    // tokens appearing on those lines, will be treated as a single comment.
+    //
+    // Only the comment content is provided; comment markers (e.g. //) are
+    // stripped out.  For block comments, leading whitespace and an asterisk
+    // will be stripped from the beginning of each line other than the first.
+    // Newlines are included in the output.
+    //
+    // Examples:
+    //
+    //   optional int32 foo = 1;  // Comment attached to foo.
+    //   // Comment attached to bar.
+    //   optional int32 bar = 2;
+    //
+    //   optional string baz = 3;
+    //   // Comment attached to baz.
+    //   // Another line attached to baz.
+    //
+    //   // Comment attached to qux.
+    //   //
+    //   // Another line attached to qux.
+    //   optional double qux = 4;
+    //
+    //   optional string corge = 5;
+    //   /* Block comment attached
+    //    * to corge.  Leading asterisks
+    //    * will be removed. */
+    //   /* Block comment attached to
+    //    * grault. */
+    //   optional int32 grault = 6;
+    optional string leading_comments = 3;
+    optional string trailing_comments = 4;
+  }
+}

generator/proto/nanopb.proto

@@ -12,6 +12,7 @@ option java_package = "fi.kapsi.koti.jpa.nanopb";
 enum FieldType {
     FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible.
     FT_CALLBACK = 1; // Always generate a callback field.
+    FT_POINTER = 4; // Always generate a dynamically allocated field.
     FT_STATIC = 2; // Generate a static field or raise an exception if not possible.
     FT_IGNORE = 3; // Ignore the field completely.
 }
@@ -36,6 +37,9 @@ message NanoPBOptions {
   // Note: this cannot be used on CPUs that break on unaligned
   // accesses to variables.
   optional bool packed_struct = 5 [default = false];
+  
+  // Skip this message
+  optional bool skip_message = 6 [default = false];
 }
 
 // Extensions to protoc 'Descriptor' type in order to define options

generator/protoc-gen-nanopb

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# This file is used to invoke nanopb_generator.py as a plugin
+# to protoc on Linux and other *nix-style systems.
+# Use it like this:
+# protoc --plugin=nanopb=..../protoc-gen-nanopb --nanopb_out=dir foo.proto
+#
+# Note that if you use the binary package of nanopb, the protoc
+# path is already set up properly and there is no need to give
+# --plugin= on the command line.
+
+MYPATH=$(dirname "$0")
+exec python "$MYPATH/nanopb_generator.py" --protoc-plugin

generator/protoc-gen-nanopb.bat

@@ -0,0 +1,12 @@
+@echo off
+:: This file is used to invoke nanopb_generator.py as a plugin
+:: to protoc on Windows.
+:: Use it like this:
+:: protoc --plugin=nanopb=..../protoc-gen-nanopb.bat --nanopb_out=dir foo.proto
+::
+:: Note that if you use the binary package of nanopb, the protoc
+:: path is already set up properly and there is no need to give
+:: --plugin= on the command line.
+
+set mydir=%~dp0
+python "%mydir%\nanopb_generator.py" --protoc-plugin

pb.h

@@ -10,6 +10,9 @@
  * uncommenting the lines, or on the compiler command line.      *
  *****************************************************************/
 
+/* Enable support for dynamically allocated fields */
+/* #define PB_ENABLE_MALLOC 1 */
+
 /* Define this if your CPU architecture is big endian, i.e. it
  * stores the most-significant byte first. */
 /* #define __BIG_ENDIAN__ 1 */
@@ -43,7 +46,7 @@
 
 /* Version of the nanopb library. Just in case you want to check it in
  * your own program. */
-#define NANOPB_VERSION nanopb-0.2.4
+#define NANOPB_VERSION nanopb-0.2.9.2
 
 /* Include all the system headers needed by nanopb. You will need the
  * definitions of the following:
@@ -63,6 +66,10 @@
 #include <stddef.h>
 #include <stdbool.h>
 #include <string.h>
+
+#ifdef PB_ENABLE_MALLOC
+#include <stdlib.h>
+#endif
 #endif
 
 /* Macro for defining packed structures (compiler dependent).
@@ -73,8 +80,8 @@
 #   define PB_PACKED_STRUCT_START
 #   define PB_PACKED_STRUCT_END
 #   define pb_packed __attribute__((packed))
-#elif defined(__ICCARM__)
-    /* For IAR ARM compiler */
+#elif defined(__ICCARM__) || defined(__CC_ARM)
+    /* For IAR ARM and Keil MDK-ARM compilers */
 #   define PB_PACKED_STRUCT_START _Pragma("pack(push, 1)")
 #   define PB_PACKED_STRUCT_END _Pragma("pack(pop)")
 #   define pb_packed
@@ -96,8 +103,14 @@
 #endif
 
 /* Compile-time assertion, used for checking compatible compilation options.
- * If this fails on your compiler for some reason, use #define STATIC_ASSERT
- * to disable it. */
+ * If this does not work properly on your compiler, use #define STATIC_ASSERT
+ * to disable it.
+ *
+ * But before doing that, check carefully the error message / place where it
+ * comes from to see if the error has a real cause. Unfortunately the error
+ * message is not always very clear to read, but you can see the reason better
+ * in the place where the STATIC_ASSERT macro was called.
+ */
 #ifndef STATIC_ASSERT
 #define STATIC_ASSERT(COND,MSG) typedef char STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND)?1:-1];
 #define STATIC_ASSERT_MSG(MSG, LINE, COUNTER) STATIC_ASSERT_MSG_(MSG, LINE, COUNTER)
@@ -116,11 +129,6 @@
 /* List of possible field types. These are used in the autogenerated code.
  * Least-significant 4 bits tell the scalar type
  * Most-significant 4 bits specify repeated/required/packed etc.
- * 
- * INT32 and UINT32 are treated the same, as are (U)INT64 and (S)FIXED*
- * These types are simply casted to correct field type when they are
- * assigned to the memory pointer.
- * SINT* is different, though, because it is zig-zag coded.
  */
 
 typedef uint8_t pb_type_t;
@@ -128,32 +136,33 @@ typedef uint8_t pb_type_t;
 /**** Field data types ****/
 
 /* Numeric types */
-#define PB_LTYPE_VARINT 0x00 /* int32, uint32, int64, uint64, bool, enum */
-#define PB_LTYPE_SVARINT 0x01 /* sint32, sint64 */
-#define PB_LTYPE_FIXED32 0x02 /* fixed32, sfixed32, float */
-#define PB_LTYPE_FIXED64 0x03 /* fixed64, sfixed64, double */
+#define PB_LTYPE_VARINT  0x00 /* int32, int64, enum, bool */
+#define PB_LTYPE_UVARINT 0x01 /* uint32, uint64 */
+#define PB_LTYPE_SVARINT 0x02 /* sint32, sint64 */
+#define PB_LTYPE_FIXED32 0x03 /* fixed32, sfixed32, float */
+#define PB_LTYPE_FIXED64 0x04 /* fixed64, sfixed64, double */
 
 /* Marker for last packable field type. */
-#define PB_LTYPE_LAST_PACKABLE 0x03
+#define PB_LTYPE_LAST_PACKABLE 0x04
 
 /* Byte array with pre-allocated buffer.
  * data_size is the length of the allocated PB_BYTES_ARRAY structure. */
-#define PB_LTYPE_BYTES 0x04
+#define PB_LTYPE_BYTES 0x05
 
 /* String with pre-allocated buffer.
  * data_size is the maximum length. */
-#define PB_LTYPE_STRING 0x05
+#define PB_LTYPE_STRING 0x06
 
 /* Submessage
  * submsg_fields is pointer to field descriptions */
-#define PB_LTYPE_SUBMESSAGE 0x06
+#define PB_LTYPE_SUBMESSAGE 0x07
 
 /* Extension pseudo-field
  * The field contains a pointer to pb_extension_t */
-#define PB_LTYPE_EXTENSION 0x07
+#define PB_LTYPE_EXTENSION 0x08
 
 /* Number of declared LTYPES */
-#define PB_LTYPES_COUNT 8
+#define PB_LTYPES_COUNT 9
 #define PB_LTYPE_MASK 0x0F
 
 /**** Field repetition rules ****/
@@ -166,6 +175,7 @@ typedef uint8_t pb_type_t;
 /**** Field allocation types ****/
  
 #define PB_ATYPE_STATIC   0x00
+#define PB_ATYPE_POINTER  0x80
 #define PB_ATYPE_CALLBACK 0x40
 #define PB_ATYPE_MASK     0xC0
 
@@ -213,7 +223,11 @@ struct _pb_field_t {
 PB_PACKED_STRUCT_END
 
 /* Make sure that the standard integer types are of the expected sizes.
- * All kinds of things may break otherwise.. atleast all fixed* types. */
+ * All kinds of things may break otherwise.. atleast all fixed* types.
+ *
+ * If you get errors here, it probably means that your stdint.h is not
+ * correct for your platform.
+ */
 STATIC_ASSERT(sizeof(int8_t) == 1, INT8_T_WRONG_SIZE)
 STATIC_ASSERT(sizeof(uint8_t) == 1, UINT8_T_WRONG_SIZE)
 STATIC_ASSERT(sizeof(int16_t) == 2, INT16_T_WRONG_SIZE)
@@ -227,11 +241,13 @@ STATIC_ASSERT(sizeof(uint64_t) == 8, UINT64_T_WRONG_SIZE)
  * It has the number of bytes in the beginning, and after that an array.
  * Note that actual structs used will have a different length of bytes array.
  */
+#define PB_BYTES_ARRAY_T(n) struct { size_t size; uint8_t bytes[n]; }
+#define PB_BYTES_ARRAY_T_ALLOCSIZE(n) ((size_t)n + offsetof(pb_bytes_array_t, bytes))
+
 struct _pb_bytes_array_t {
     size_t size;
     uint8_t bytes[1];
 };
-
 typedef struct _pb_bytes_array_t pb_bytes_array_t;
 
 /* This structure is used for giving the callback function.
@@ -325,8 +341,23 @@ struct _pb_extension_t {
      * If this extension does not match a field, the next handler is
      * automatically called. */
     pb_extension_t *next;
+
+    /* The decoder sets this to true if the extension was found.
+     * Ignored for encoding. */
+    bool found;
 };
 
+/* Memory allocation functions to use. You can define pb_realloc and
+ * pb_free to custom functions if you want. */
+#ifdef PB_ENABLE_MALLOC
+#   ifndef pb_realloc
+#       define pb_realloc(ptr, size) realloc(ptr, size)
+#   endif
+#   ifndef pb_free
+#       define pb_free(ptr) free(ptr)
+#   endif
+#endif
+
 /* These macros are used to declare pb_field_t's in the constant array. */
 /* Size of a structure member, in bytes. */
 #define pb_membersize(st, m) (sizeof ((st*)0)->m)
@@ -370,6 +401,22 @@ struct _pb_extension_t {
     pb_membersize(st, m[0]), \
     pb_arraysize(st, m), ptr}
 
+/* Allocated fields carry the size of the actual data, not the pointer */
+#define PB_REQUIRED_POINTER(tag, st, m, fd, ltype, ptr) \
+    {tag, PB_ATYPE_POINTER | PB_HTYPE_REQUIRED | ltype, \
+    fd, 0, pb_membersize(st, m[0]), 0, ptr}
+
+/* Optional fields don't need a has_ variable, as information would be redundant */
+#define PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) \
+    {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \
+    fd, 0, pb_membersize(st, m[0]), 0, ptr}
+
+/* Repeated fields have a _count field and a pointer to array of pointers */
+#define PB_REPEATED_POINTER(tag, st, m, fd, ltype, ptr) \
+    {tag, PB_ATYPE_POINTER | PB_HTYPE_REPEATED | ltype, \
+    fd, pb_delta(st, m ## _count, m), \
+    pb_membersize(st, m[0]), 0, ptr}
+
 /* Callbacks are much like required fields except with special datatype. */
 #define PB_REQUIRED_CALLBACK(tag, st, m, fd, ltype, ptr) \
     {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REQUIRED | ltype, \
@@ -410,8 +457,8 @@ struct _pb_extension_t {
 #define PB_LTYPE_MAP_SINT32     PB_LTYPE_SVARINT
 #define PB_LTYPE_MAP_SINT64     PB_LTYPE_SVARINT
 #define PB_LTYPE_MAP_STRING     PB_LTYPE_STRING
-#define PB_LTYPE_MAP_UINT32     PB_LTYPE_VARINT
-#define PB_LTYPE_MAP_UINT64     PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_UINT32     PB_LTYPE_UVARINT
+#define PB_LTYPE_MAP_UINT64     PB_LTYPE_UVARINT
 #define PB_LTYPE_MAP_EXTENSION  PB_LTYPE_EXTENSION
 
 /* This is the actual macro used in field descriptions.
@@ -453,7 +500,11 @@ struct _pb_extension_t {
  * messages if not used.
  */
 #ifdef PB_NO_ERRMSG
-#define PB_RETURN_ERROR(stream,msg) return false
+#define PB_RETURN_ERROR(stream,msg) \
+    do {\
+        UNUSED(stream); \
+        return false; \
+    } while(0)
 #define PB_GET_ERROR(stream) "(errmsg disabled)"
 #else
 #define PB_RETURN_ERROR(stream,msg) \

pb_decode.c

@@ -13,7 +13,6 @@
     #define checkreturn __attribute__((warn_unused_result))
 #endif
 
-#define NANOPB_INTERNALS
 #include "pb.h"
 #include "pb_decode.h"
 
@@ -43,11 +42,13 @@ static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag);
 static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
 static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
 static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
+static void iter_from_extension(pb_field_iterator_t *iter, pb_extension_t *extension);
 static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);
 static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
 static bool checkreturn find_extension_field(pb_field_iterator_t *iter);
 static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct);
-static bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest);
 static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest);
 static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest);
 static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest);
@@ -57,11 +58,17 @@ static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t
 static bool checkreturn pb_skip_varint(pb_istream_t *stream);
 static bool checkreturn pb_skip_string(pb_istream_t *stream);
 
+#ifdef PB_ENABLE_MALLOC
+static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size);
+static void pb_release_single_field(const pb_field_iterator_t *iter);
+#endif
+
 /* --- Function pointers to field decoders ---
  * Order in the array must match pb_action_t LTYPE numbering.
  */
 static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = {
     &pb_dec_varint,
+    &pb_dec_uvarint,
     &pb_dec_svarint,
     &pb_dec_fixed32,
     &pb_dec_fixed64,
@@ -124,6 +131,26 @@ bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count)
     return true;
 }
 
+/* Read a single byte from input stream. buf may not be NULL.
+ * This is an optimization for the varint decoding. */
+static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf)
+{
+    if (stream->bytes_left == 0)
+        PB_RETURN_ERROR(stream, "end-of-stream");
+
+#ifndef PB_BUFFER_ONLY
+    if (!stream->callback(stream, buf, 1))
+        PB_RETURN_ERROR(stream, "io error");
+#else
+    *buf = *(uint8_t*)stream->state;
+    stream->state = (uint8_t*)stream->state + 1;
+#endif
+
+    stream->bytes_left--;
+    
+    return true;    
+}
+
 pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize)
 {
     pb_istream_t stream;
@@ -149,10 +176,10 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest)
     uint8_t byte;
     uint32_t result;
     
-    if (!pb_read(stream, &byte, 1))
+    if (!pb_readbyte(stream, &byte))
         return false;
     
-    if (!(byte & 0x80))
+    if ((byte & 0x80) == 0)
     {
         /* Quick case, 1 byte value */
         result = byte;
@@ -168,7 +195,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest)
             if (bitpos >= 32)
                 PB_RETURN_ERROR(stream, "varint overflow");
             
-            if (!pb_read(stream, &byte, 1))
+            if (!pb_readbyte(stream, &byte))
                 return false;
             
             result |= (uint32_t)(byte & 0x7F) << bitpos;
@@ -191,7 +218,7 @@ bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest)
         if (bitpos >= 64)
             PB_RETURN_ERROR(stream, "varint overflow");
         
-        if (!pb_read(stream, &byte, 1))
+        if (!pb_readbyte(stream, &byte))
             return false;
 
         result |= (uint64_t)(byte & 0x7F) << bitpos;
@@ -337,6 +364,10 @@ static bool pb_field_next(pb_field_iterator_t *iter)
     {
         prev_size *= iter->pos->array_size;
     }
+    else if (PB_ATYPE(iter->pos->type) == PB_ATYPE_POINTER)
+    {
+        prev_size = sizeof(void*);
+    }
     
     if (iter->pos->tag == 0)
         return false; /* Only happens with empty message types */
@@ -371,7 +402,7 @@ static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag)
         {
             return true;
         }
-        pb_field_next(iter);
+        (void)pb_field_next(iter);
     } while (iter->field_index != start);
     
     return false;
@@ -409,7 +440,7 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t
                 if (!pb_make_string_substream(stream, &substream))
                     return false;
                 
-                while (substream.bytes_left && *size < iter->pos->array_size)
+                while (substream.bytes_left > 0 && *size < iter->pos->array_size)
                 {
                     void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size);
                     if (!func(&substream, iter->pos, pItem))
@@ -443,6 +474,164 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t
     }
 }
 
+#ifdef PB_ENABLE_MALLOC
+/* Allocate storage for the field and store the pointer at iter->pData.
+ * array_size is the number of entries to reserve in an array.
+ * Zero size is not allowed, use pb_free() for releasing.
+ */
+static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size)
+{    
+    void *ptr = *(void**)pData;
+    
+    if (data_size == 0 || array_size == 0)
+        PB_RETURN_ERROR(stream, "invalid size");
+    
+    /* Check for multiplication overflows.
+     * This code avoids the costly division if the sizes are small enough.
+     * Multiplication is safe as long as only half of bits are set
+     * in either multiplicand.
+     */
+    {
+        const size_t check_limit = (size_t)1 << (sizeof(size_t) * 4);
+        if (data_size >= check_limit || array_size >= check_limit)
+        {
+            const size_t size_max = (size_t)-1;
+            if (size_max / array_size < data_size)
+            {
+                PB_RETURN_ERROR(stream, "size too large");
+            }
+        }
+    }
+    
+    /* Allocate new or expand previous allocation */
+    /* Note: on failure the old pointer will remain in the structure,
+     * the message must be freed by caller also on error return. */
+    ptr = pb_realloc(ptr, array_size * data_size);
+    if (ptr == NULL)
+        PB_RETURN_ERROR(stream, "realloc failed");
+    
+    *(void**)pData = ptr;
+    return true;
+}
+
+/* Clear a newly allocated item in case it contains a pointer, or is a submessage. */
+static void initialize_pointer_field(void *pItem, pb_field_iterator_t *iter)
+{
+    if (PB_LTYPE(iter->pos->type) == PB_LTYPE_STRING ||
+        PB_LTYPE(iter->pos->type) == PB_LTYPE_BYTES)
+    {
+        *(void**)pItem = NULL;
+    }
+    else if (PB_LTYPE(iter->pos->type) == PB_LTYPE_SUBMESSAGE)
+    {
+        pb_message_set_to_defaults((const pb_field_t *) iter->pos->ptr, pItem);
+    }
+}
+#endif
+
+static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
+{
+#ifndef PB_ENABLE_MALLOC
+    UNUSED(wire_type);
+    UNUSED(iter);
+    PB_RETURN_ERROR(stream, "no malloc support");
+#else
+    pb_type_t type;
+    pb_decoder_t func;
+    
+    type = iter->pos->type;
+    func = PB_DECODERS[PB_LTYPE(type)];
+    
+    switch (PB_HTYPE(type))
+    {
+        case PB_HTYPE_REQUIRED:
+        case PB_HTYPE_OPTIONAL:
+            if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE &&
+                *(void**)iter->pData != NULL)
+            {
+                /* Duplicate field, have to release the old allocation first. */
+                pb_release_single_field(iter);
+            }
+        
+            if (PB_LTYPE(type) == PB_LTYPE_STRING ||
+                PB_LTYPE(type) == PB_LTYPE_BYTES)
+            {
+                return func(stream, iter->pos, iter->pData);
+            }
+            else
+            {
+                if (!allocate_field(stream, iter->pData, iter->pos->data_size, 1))
+                    return false;
+                
+                initialize_pointer_field(*(void**)iter->pData, iter);
+                return func(stream, iter->pos, *(void**)iter->pData);
+            }
+    
+        case PB_HTYPE_REPEATED:
+            if (wire_type == PB_WT_STRING
+                && PB_LTYPE(type) <= PB_LTYPE_LAST_PACKABLE)
+            {
+                /* Packed array, multiple items come in at once. */
+                bool status = true;
+                size_t *size = (size_t*)iter->pSize;
+                size_t allocated_size = *size;
+                void *pItem;
+                pb_istream_t substream;
+                
+                if (!pb_make_string_substream(stream, &substream))
+                    return false;
+                
+                while (substream.bytes_left)
+                {
+                    if (*size + 1 > allocated_size)
+                    {
+                        /* Allocate more storage. This tries to guess the
+                         * number of remaining entries. Round the division
+                         * upwards. */
+                        allocated_size += (substream.bytes_left - 1) / iter->pos->data_size + 1;
+                        
+                        if (!allocate_field(&substream, iter->pData, iter->pos->data_size, allocated_size))
+                        {
+                            status = false;
+                            break;
+                        }
+                    }
+
+                    /* Decode the array entry */
+                    pItem = *(uint8_t**)iter->pData + iter->pos->data_size * (*size);
+                    initialize_pointer_field(pItem, iter);
+                    if (!func(&substream, iter->pos, pItem))
+                    {
+                        status = false;
+                        break;
+                    }
+                    (*size)++;
+                }
+                pb_close_string_substream(stream, &substream);
+                
+                return status;
+            }
+            else
+            {
+                /* Normal repeated field, i.e. only one item at a time. */
+                size_t *size = (size_t*)iter->pSize;
+                void *pItem;
+                
+                (*size)++;
+                if (!allocate_field(stream, iter->pData, iter->pos->data_size, *size))
+                    return false;
+            
+                pItem = *(uint8_t**)iter->pData + iter->pos->data_size * (*size - 1);
+                initialize_pointer_field(pItem, iter);
+                return func(stream, iter->pos, pItem);
+            }
+            
+        default:
+            PB_RETURN_ERROR(stream, "invalid field type");
+    }
+#endif
+}
+
 static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
 {
     pb_callback_t *pCallback = (pb_callback_t*)iter->pData;
@@ -497,6 +686,9 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
         case PB_ATYPE_STATIC:
             return decode_static_field(stream, wire_type, iter);
         
+        case PB_ATYPE_POINTER:
+            return decode_pointer_field(stream, wire_type, iter);
+        
         case PB_ATYPE_CALLBACK:
             return decode_callback_field(stream, wire_type, iter);
         
@@ -505,6 +697,19 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
     }
 }
 
+static void iter_from_extension(pb_field_iterator_t *iter, pb_extension_t *extension)
+{
+    const pb_field_t *field = (const pb_field_t*)extension->type->arg;
+    
+    iter->start = field;
+    iter->pos = field;
+    iter->field_index = 0;
+    iter->required_field_index = 0;
+    iter->dest_struct = extension->dest;
+    iter->pData = extension->dest;
+    iter->pSize = &extension->found;
+}
+
 /* Default handler for extension fields. Expects a pb_field_t structure
  * in extension->type->arg. */
 static bool checkreturn default_extension_decoder(pb_istream_t *stream,
@@ -512,19 +717,11 @@ static bool checkreturn default_extension_decoder(pb_istream_t *stream,
 {
     const pb_field_t *field = (const pb_field_t*)extension->type->arg;
     pb_field_iterator_t iter;
-    bool dummy;
     
     if (field->tag != tag)
         return true;
     
-    iter.start = field;
-    iter.pos = field;
-    iter.field_index = 0;
-    iter.required_field_index = 0;
-    iter.dest_struct = extension->dest;
-    iter.pData = extension->dest;
-    iter.pSize = &dummy;
-    
+    iter_from_extension(&iter, extension);
     return decode_field(stream, wire_type, &iter);
 }
 
@@ -536,7 +733,7 @@ static bool checkreturn decode_extension(pb_istream_t *stream,
     pb_extension_t *extension = *(pb_extension_t* const *)iter->pData;
     size_t pos = stream->bytes_left;
     
-    while (extension && pos == stream->bytes_left)
+    while (extension != NULL && pos == stream->bytes_left)
     {
         bool status;
         if (extension->type->decode)
@@ -563,7 +760,7 @@ static bool checkreturn find_extension_field(pb_field_iterator_t *iter)
     do {
         if (PB_LTYPE(iter->pos->type) == PB_LTYPE_EXTENSION)
             return true;
-        pb_field_next(iter);
+        (void)pb_field_next(iter);
     } while (iter->field_index != start);
     
     return false;
@@ -575,45 +772,60 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str
     pb_field_iterator_t iter;
     pb_field_init(&iter, fields, dest_struct);
     
-    /* Initialize size/has fields and apply default values */
     do
     {
         pb_type_t type;
         type = iter.pos->type;
     
+        /* Avoid crash on empty message types (zero fields) */
         if (iter.pos->tag == 0)
             continue;
         
         if (PB_ATYPE(type) == PB_ATYPE_STATIC)
         {
-            /* Initialize the size field for optional/repeated fields to 0. */
             if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL)
             {
+                /* Set has_field to false. Still initialize the optional field
+                 * itself also. */
                 *(bool*)iter.pSize = false;
             }
             else if (PB_HTYPE(type) == PB_HTYPE_REPEATED)
             {
+                /* Set array count to 0, no need to initialize contents. */
                 *(size_t*)iter.pSize = 0;
-                continue; /* Array is empty, no need to initialize contents */
+                continue;
             }
             
-            /* Initialize field contents to default value */
             if (PB_LTYPE(iter.pos->type) == PB_LTYPE_SUBMESSAGE)
             {
+                /* Initialize submessage to defaults */
                 pb_message_set_to_defaults((const pb_field_t *) iter.pos->ptr, iter.pData);
             }
             else if (iter.pos->ptr != NULL)
             {
+                /* Initialize to default value */
                 memcpy(iter.pData, iter.pos->ptr, iter.pos->data_size);
             }
             else
             {
+                /* Initialize to zeros */
                 memset(iter.pData, 0, iter.pos->data_size);
             }
         }
+        else if (PB_ATYPE(type) == PB_ATYPE_POINTER)
+        {
+            /* Initialize the pointer to NULL. */
+            *(void**)iter.pData = NULL;
+            
+            /* Initialize array count to 0. */
+            if (PB_HTYPE(type) == PB_HTYPE_REPEATED)
+            {
+                *(size_t*)iter.pSize = 0;
+            }
+        }
         else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK)
         {
-            continue; /* Don't overwrite callback */
+            /* Don't overwrite callback */
         }
     } while (pb_field_next(&iter));
 }
@@ -624,7 +836,7 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str
 
 bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct)
 {
-    uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0}; /* Used to check for required fields */
+    uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0, 0, 0, 0, 0, 0, 0, 0};
     uint32_t extension_range_start = 0;
     pb_field_iterator_t iter;
     
@@ -700,7 +912,7 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[
         } while (pb_field_next(&iter));
         
         /* Fixup if last field was also required. */
-        if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag)
+        if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag != 0)
             req_field_count++;
         
         /* Check the whole bytes */
@@ -720,8 +932,16 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[
 
 bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct)
 {
+    bool status;
     pb_message_set_to_defaults(fields, dest_struct);
-    return pb_decode_noinit(stream, fields, dest_struct);
+    status = pb_decode_noinit(stream, fields, dest_struct);
+    
+#ifdef PB_ENABLE_MALLOC
+    if (!status)
+        pb_release(fields, dest_struct);
+#endif
+    
+    return status;
 }
 
 bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct)
@@ -737,6 +957,96 @@ bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *
     return status;
 }
 
+#ifdef PB_ENABLE_MALLOC
+static void pb_release_single_field(const pb_field_iterator_t *iter)
+{
+    pb_type_t type;
+    type = iter->pos->type;
+
+    /* Release anything contained inside an extension or submsg.
+     * This has to be done even if the submsg itself is statically
+     * allocated. */
+    if (PB_LTYPE(type) == PB_LTYPE_EXTENSION)
+    {
+        /* Release fields from all extensions in the linked list */
+        pb_extension_t *ext = *(pb_extension_t**)iter->pData;
+        while (ext != NULL)
+        {
+            pb_field_iterator_t ext_iter;
+            iter_from_extension(&ext_iter, ext);
+            pb_release_single_field(&ext_iter);
+            ext = ext->next;
+        }
+    }
+    else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE)
+    {
+        /* Release fields in submessage or submsg array */
+        void *pItem = iter->pData;
+        pb_size_t count = 1;
+        
+        if (PB_ATYPE(type) == PB_ATYPE_POINTER)
+        {
+            pItem = *(void**)iter->pData;
+        }
+        
+        if (PB_HTYPE(type) == PB_HTYPE_REPEATED)
+        {
+            count = *(pb_size_t*)iter->pSize;
+        }
+        
+        if (pItem)
+        {
+            while (count--)
+            {
+                pb_release((const pb_field_t*)iter->pos->ptr, pItem);
+                pItem = (uint8_t*)pItem + iter->pos->data_size;
+            }
+        }
+    }
+    
+    if (PB_ATYPE(type) == PB_ATYPE_POINTER)
+    {
+        if (PB_HTYPE(type) == PB_HTYPE_REPEATED &&
+            (PB_LTYPE(type) == PB_LTYPE_STRING ||
+             PB_LTYPE(type) == PB_LTYPE_BYTES))
+        {
+            /* Release entries in repeated string or bytes array */
+            void **pItem = *(void***)iter->pData;
+            pb_size_t count = *(pb_size_t*)iter->pSize;
+            while (count--)
+            {
+                pb_free(*pItem);
+                *pItem++ = NULL;
+            }
+        }
+        
+        if (PB_HTYPE(type) == PB_HTYPE_REPEATED)
+        {
+            /* We are going to release the array, so set the size to 0 */
+            *(pb_size_t*)iter->pSize = 0;
+        }
+        
+        /* Release main item */
+        pb_free(*(void**)iter->pData);
+        *(void**)iter->pData = NULL;
+    }
+}
+
+void pb_release(const pb_field_t fields[], void *dest_struct)
+{
+    pb_field_iterator_t iter;
+    pb_field_init(&iter, fields, dest_struct);
+
+    if (iter.pos->tag == 0)
+        return; /* Empty message type */
+    
+    do
+    {
+        pb_release_single_field(&iter);
+    } while (pb_field_next(&iter));
+}
+#endif
+
 /* Field decoders */
 
 bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest)
@@ -795,7 +1105,25 @@ bool pb_decode_fixed64(pb_istream_t *stream, void *dest)
     #endif   
 }
 
-bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest)
+static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest)
+{
+    uint64_t value;
+    if (!pb_decode_varint(stream, &value))
+        return false;
+    
+    switch (field->data_size)
+    {
+        case 1: *(int8_t*)dest = (int8_t)value; break;
+        case 2: *(int16_t*)dest = (int16_t)value; break;
+        case 4: *(int32_t*)dest = (int32_t)value; break;
+        case 8: *(int64_t*)dest = (int64_t)value; break;
+        default: PB_RETURN_ERROR(stream, "invalid data_size");
+    }
+    
+    return true;
+}
+
+static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest)
 {
     uint64_t value;
     if (!pb_decode_varint(stream, &value))
@@ -803,8 +1131,6 @@ bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, vo
     
     switch (field->data_size)
     {
-        case 1: *(uint8_t*)dest = (uint8_t)value; break;
-        case 2: *(uint16_t*)dest = (uint16_t)value; break;
         case 4: *(uint32_t*)dest = (uint32_t)value; break;
         case 8: *(uint64_t*)dest = value; break;
         default: PB_RETURN_ERROR(stream, "invalid data_size");
@@ -813,7 +1139,7 @@ bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, vo
     return true;
 }
 
-bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest)
+static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest)
 {
     int64_t value;
     if (!pb_decode_svarint(stream, &value))
@@ -829,51 +1155,89 @@ bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, v
     return true;
 }
 
-bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest)
+static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest)
 {
     UNUSED(field);
     return pb_decode_fixed32(stream, dest);
 }
 
-bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest)
+static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest)
 {
     UNUSED(field);
     return pb_decode_fixed64(stream, dest);
 }
 
-bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest)
-{
-    pb_bytes_array_t *x = (pb_bytes_array_t*)dest;
-    
-    uint32_t temp;
-    if (!pb_decode_varint32(stream, &temp))
-        return false;
-    x->size = temp;
-    
-    /* Check length, noting the space taken by the size_t header. */
-    if (x->size > field->data_size - offsetof(pb_bytes_array_t, bytes))
-        PB_RETURN_ERROR(stream, "bytes overflow");
-    
-    return pb_read(stream, x->bytes, x->size);
-}
-
-bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest)
+static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest)
 {
     uint32_t size;
+    size_t alloc_size;
+    pb_bytes_array_t *bdest;
+    
+    if (!pb_decode_varint32(stream, &size))
+        return false;
+    
+    alloc_size = PB_BYTES_ARRAY_T_ALLOCSIZE(size);
+    if (size > alloc_size)
+        PB_RETURN_ERROR(stream, "size too large");
+    
+    if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
+    {
+#ifndef PB_ENABLE_MALLOC
+        PB_RETURN_ERROR(stream, "no malloc support");
+#else
+        if (!allocate_field(stream, dest, alloc_size, 1))
+            return false;
+        bdest = *(pb_bytes_array_t**)dest;
+#endif
+    }
+    else
+    {
+        if (alloc_size > field->data_size)
+            PB_RETURN_ERROR(stream, "bytes overflow");
+        bdest = (pb_bytes_array_t*)dest;
+    }
+    
+    bdest->size = size;
+
+    return pb_read(stream, bdest->bytes, size);
+}
+
+static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest)
+{
+    uint32_t size;
+    size_t alloc_size;
     bool status;
     if (!pb_decode_varint32(stream, &size))
         return false;
     
-    /* Check length, noting the null terminator */
-    if (size + 1 > field->data_size)
+    /* Space for null terminator */
+    alloc_size = size + 1;
+    
+    if (alloc_size < size)
+        PB_RETURN_ERROR(stream, "size too large");
+    
+    if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
+    {
+#ifndef PB_ENABLE_MALLOC
+        PB_RETURN_ERROR(stream, "no malloc support");
+#else
+        if (!allocate_field(stream, dest, alloc_size, 1))
+            return false;
+        dest = *(void**)dest;
+#endif
+    }
+    else
+    {
+        if (alloc_size > field->data_size)
             PB_RETURN_ERROR(stream, "string overflow");
+    }
     
     status = pb_read(stream, (uint8_t*)dest, size);
     *((uint8_t*)dest + size) = 0;
     return status;
 }
 
-bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest)
+static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest)
 {
     bool status;
     pb_istream_t substream;

pb_decode.h

@@ -73,6 +73,9 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc
  *
  * This can also be used for 'merging' two messages, i.e. update only the
  * fields that exist in the new message.
+ *
+ * Note: If this function returns with an error, it will not release any
+ * dynamically allocated fields. You will need to call pb_release() yourself.
  */
 bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct);
 
@@ -82,6 +85,14 @@ bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *des
  */
 bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct);
 
+#ifdef PB_ENABLE_MALLOC
+/* Release any allocated pointer fields. If you use dynamic allocation, you should
+ * call this for any successfully decoded message when you are done with it. If
+ * pb_decode() returns with an error, the message is already released.
+ */
+void pb_release(const pb_field_t fields[], void *dest_struct);
+#endif
+
 
 /**************************************
  * Functions for manipulating streams *

pb_encode.c

@@ -3,7 +3,6 @@
  * 2011 Petteri Aimonen <jpa@kapsi.fi>
  */
 
-#define NANOPB_INTERNALS
 #include "pb.h"
 #include "pb_encode.h"
 
@@ -28,6 +27,7 @@ static bool checkreturn encode_field(pb_ostream_t *stream, const pb_field_t *fie
 static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension);
 static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData);
 static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, const void *src);
 static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src);
 static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src);
 static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src);
@@ -40,6 +40,7 @@ static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t
  */
 static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = {
     &pb_enc_varint,
+    &pb_enc_uvarint,
     &pb_enc_svarint,
     &pb_enc_fixed32,
     &pb_enc_fixed64,
@@ -117,7 +118,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
     if (count == 0)
         return true;
 
-    if (count > field->array_size)
+    if (PB_ATYPE(field->type) != PB_ATYPE_POINTER && count > field->array_size)
         PB_RETURN_ERROR(stream, "array max size exceeded");
     
     /* We always pack arrays if the datatype allows it. */
@@ -170,8 +171,23 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
         {
             if (!pb_encode_tag_for_field(stream, field))
                 return false;
+
+            /* Normally the data is stored directly in the array entries, but
+             * for pointer-type string and bytes fields, the array entries are
+             * actually pointers themselves also. So we have to dereference once
+             * more to get to the actual data. */
+            if (PB_ATYPE(field->type) == PB_ATYPE_POINTER &&
+                (PB_LTYPE(field->type) == PB_LTYPE_STRING ||
+                 PB_LTYPE(field->type) == PB_LTYPE_BYTES))
+            {
+                if (!func(stream, field, *(const void* const*)p))
+                    return false;      
+            }
+            else
+            {
                 if (!func(stream, field, p))
                     return false;
+            }
             p = (const char*)p + field->data_size;
         }
     }
@@ -179,25 +195,38 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
     return true;
 }
 
-/* Encode a field with static allocation, i.e. one whose data is stored
- * in the structure itself. */
-static bool checkreturn encode_static_field(pb_ostream_t *stream,
+/* Encode a field with static or pointer allocation, i.e. one whose data
+ * is available to the encoder directly. */
+static bool checkreturn encode_basic_field(pb_ostream_t *stream,
     const pb_field_t *field, const void *pData)
 {
     pb_encoder_t func;
     const void *pSize;
-    bool dummy = true;
+    bool implicit_has = true;
     
     func = PB_ENCODERS[PB_LTYPE(field->type)];
     
     if (field->size_offset)
         pSize = (const char*)pData + field->size_offset;
     else
-        pSize = &dummy;
+        pSize = &implicit_has;
+
+    if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
+    {
+        /* pData is a pointer to the field, which contains pointer to
+         * the data. If the 2nd pointer is NULL, it is interpreted as if
+         * the has_field was false.
+         */
+        
+        pData = *(const void* const*)pData;
+        implicit_has = (pData != NULL);
+    }
 
     switch (PB_HTYPE(field->type))
     {
         case PB_HTYPE_REQUIRED:
+            if (!pData)
+                PB_RETURN_ERROR(stream, "missing required field");
             if (!pb_encode_tag_for_field(stream, field))
                 return false;
             if (!func(stream, field, pData))
@@ -255,7 +284,8 @@ static bool checkreturn encode_field(pb_ostream_t *stream,
     switch (PB_ATYPE(field->type))
     {
         case PB_ATYPE_STATIC:
-            return encode_static_field(stream, field, pData);
+        case PB_ATYPE_POINTER:
+            return encode_basic_field(stream, field, pData);
         
         case PB_ATYPE_CALLBACK:
             return encode_callback_field(stream, field, pData);
@@ -312,6 +342,9 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons
     while (field->tag != 0)
     {
         pData = (const char*)pData + prev_size + field->data_offset;
+        if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
+            prev_size = sizeof(const void*);
+        else
             prev_size = field->data_size;
         
         /* Special case for static arrays */
@@ -345,6 +378,17 @@ bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const
     return pb_encode_submessage(stream, fields, src_struct);
 }
 
+bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *src_struct)
+{
+    pb_ostream_t stream = PB_OSTREAM_SIZING;
+    
+    if (!pb_encode(&stream, fields, src_struct))
+        return false;
+    
+    *size = stream.bytes_written;
+    return true;
+}
+
 /********************
  * Helper functions *
  ********************/
@@ -371,9 +415,9 @@ bool checkreturn pb_encode_svarint(pb_ostream_t *stream, int64_t value)
 {
     uint64_t zigzagged;
     if (value < 0)
-        zigzagged = (uint64_t)(~(value << 1));
+        zigzagged = ~((uint64_t)value << 1);
     else
-        zigzagged = (uint64_t)(value << 1);
+        zigzagged = (uint64_t)value << 1;
     
     return pb_encode_varint(stream, zigzagged);
 }
@@ -414,7 +458,7 @@ bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value)
 
 bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number)
 {
-    uint64_t tag = wiretype | (field_number << 3);
+    uint64_t tag = ((uint64_t)field_number << 3) | wiretype;
     return pb_encode_varint(stream, tag);
 }
 
@@ -424,6 +468,7 @@ bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t
     switch (PB_LTYPE(field->type))
     {
         case PB_LTYPE_VARINT:
+        case PB_LTYPE_UVARINT:
         case PB_LTYPE_SVARINT:
             wiretype = PB_WT_VARINT;
             break;
@@ -465,7 +510,12 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie
     bool status;
     
     if (!pb_encode(&substream, fields, src_struct))
+    {
+#ifndef PB_NO_ERRMSG
+        stream->errmsg = substream.errmsg;
+#endif
         return false;
+    }
     
     size = substream.bytes_written;
     
@@ -504,14 +554,30 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie
 
 /* Field encoders */
 
-bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src)
+static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src)
+{
+    int64_t value = 0;
+    
+    /* Cases 1 and 2 are for compilers that have smaller types for bool
+     * or enums. */
+    switch (field->data_size)
+    {
+        case 1: value = *(const int8_t*)src; break;
+        case 2: value = *(const int16_t*)src; break;
+        case 4: value = *(const int32_t*)src; break;
+        case 8: value = *(const int64_t*)src; break;
+        default: PB_RETURN_ERROR(stream, "invalid data_size");
+    }
+    
+    return pb_encode_varint(stream, (uint64_t)value);
+}
+
+static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
     uint64_t value = 0;
     
     switch (field->data_size)
     {
-        case 1: value = *(const uint8_t*)src; break;
-        case 2: value = *(const uint16_t*)src; break;
         case 4: value = *(const uint32_t*)src; break;
         case 8: value = *(const uint64_t*)src; break;
         default: PB_RETURN_ERROR(stream, "invalid data_size");
@@ -520,7 +586,7 @@ bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, co
     return pb_encode_varint(stream, value);
 }
 
-bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src)
+static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
     int64_t value = 0;
     
@@ -534,43 +600,64 @@ bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, c
     return pb_encode_svarint(stream, value);
 }
 
-bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src)
+static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
     UNUSED(field);
     return pb_encode_fixed64(stream, src);
 }
 
-bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src)
+static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
     UNUSED(field);
     return pb_encode_fixed32(stream, src);
 }
 
-bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src)
+static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
     const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src;
     
-    if (bytes->size + offsetof(pb_bytes_array_t, bytes) > field->data_size)
+    if (src == NULL)
+    {
+        /* Threat null pointer as an empty bytes field */
+        return pb_encode_string(stream, NULL, 0);
+    }
+    
+    if (PB_ATYPE(field->type) == PB_ATYPE_STATIC &&
+        PB_BYTES_ARRAY_T_ALLOCSIZE(bytes->size) > field->data_size)
+    {
         PB_RETURN_ERROR(stream, "bytes size exceeded");
+    }
     
     return pb_encode_string(stream, bytes->bytes, bytes->size);
 }
 
-bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src)
+static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
-    /* strnlen() is not always available, so just use a for-loop */
+    /* strnlen() is not always available, so just use a loop */
     size_t size = 0;
+    size_t max_size = field->data_size;
     const char *p = (const char*)src;
-    while (size < field->data_size && *p != '\0')
+    
+    if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
+        max_size = (size_t)-1;
+
+    if (src == NULL)
+    {
+        size = 0; /* Threat null pointer as an empty string */
+    }
+    else
+    {
+        while (size < max_size && *p != '\0')
         {
             size++;
             p++;
         }
+    }
 
     return pb_encode_string(stream, (const uint8_t*)src, size);
 }
 
-bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src)
+static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
     if (field->ptr == NULL)
         PB_RETURN_ERROR(stream, "invalid field descriptor");

pb_encode.h

@@ -71,6 +71,10 @@ bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_
  */
 bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct);
 
+/* Encode the message to get the size of the encoded data, but do not store
+ * the data. */
+bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *src_struct);
+
 /**************************************
  * Functions for manipulating streams *
  **************************************/

tests/Makefile

@@ -4,3 +4,18 @@ all:
 clean:
 	scons -c
 
+coverage:
+	rm -rf build coverage
+
+	# LCOV does not like the newer gcov format
+	scons CC=gcc-4.6 CXX=gcc-4.6
+
+	# We are only interested in pb_encode.o and pb_decode.o
+	find build -name '*.gcda' -and \! \( -name '*pb_encode*' -or -name '*pb_decode*' \) -exec rm '{}' \;
+
+	# Collect the data
+	mkdir build/coverage
+	lcov --base-directory . --directory build/ --gcov-tool gcov-4.6 -c -o build/coverage/nanopb.info
+
+	# Generate HTML
+	genhtml -o build/coverage build/coverage/nanopb.info

tests/SConstruct

@@ -14,40 +14,56 @@ scons CC=clang CXX=clang++
 ''')
 
 import os
-env = Environment(ENV = os.environ)
+env = Environment(ENV = os.environ, tools = ['default', 'nanopb'])
 
 # Allow overriding the compiler with scons CC=???
 if 'CC' in ARGUMENTS: env.Replace(CC = ARGUMENTS['CC'])
 if 'CXX' in ARGUMENTS: env.Replace(CXX = ARGUMENTS['CXX'])
-if 'CFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CFLAGS'])
-if 'CXXFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CXXFLAGS'])
+if 'CCFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CCFLAGS'])
+if 'CXXFLAGS' in ARGUMENTS: env.Append(CXXFLAGS = ARGUMENTS['CXXFLAGS'])
 
 # Add the builders defined in site_init.py
 add_nanopb_builders(env)
 
 # Path to the files shared by tests, and to the nanopb core.
-env.Append(CPPPATH = ["#../", "#common"])
+env.Append(CPPPATH = ["#../", "$COMMON"])
 
 # Path for finding nanopb.proto
 env.Append(PROTOCPATH = '#../generator')
 
 # Check the compilation environment, unless we are just cleaning up.
 if not env.GetOption('clean'):
-    conf = Configure(env)
+    def check_ccflags(context, flags, linkflags = ''):
+        '''Check if given CCFLAGS are supported'''
+        context.Message('Checking support for CCFLAGS="%s"... ' % flags)
+        oldflags = context.env['CCFLAGS']
+        oldlinkflags = context.env['CCFLAGS']
+        context.env.Append(CCFLAGS = flags)
+        context.env.Append(LINKFLAGS = linkflags)
+        result = context.TryCompile("int main() {return 0;}", '.c')
+        context.env.Replace(CCFLAGS = oldflags)
+        context.env.Replace(LINKFLAGS = oldlinkflags)
+        context.Result(result)
+        return result
+    
+    conf = Configure(env, custom_tests = {'CheckCCFLAGS': check_ccflags})
 
     # If the platform doesn't support C99, use our own header file instead.
     stdbool = conf.CheckCHeader('stdbool.h')
     stdint = conf.CheckCHeader('stdint.h')
     stddef = conf.CheckCHeader('stddef.h')
     string = conf.CheckCHeader('string.h')
+    stdlib = conf.CheckCHeader('stdlib.h')
     if not stdbool or not stdint or not stddef or not string:
         conf.env.Append(CPPDEFINES = {'PB_SYSTEM_HEADER': '\\"pb_syshdr.h\\"'})
-        conf.env.Append(CPPPATH = "#../compat")
+        conf.env.Append(CPPPATH = "#../extra")
+	conf.env.Append(SYSHDR = '\\"pb_syshdr.h\\"')
         
         if stdbool: conf.env.Append(CPPDEFINES = {'HAVE_STDBOOL_H': 1})
         if stdint: conf.env.Append(CPPDEFINES = {'HAVE_STDINT_H': 1})
         if stddef: conf.env.Append(CPPDEFINES = {'HAVE_STDDEF_H': 1})
         if string: conf.env.Append(CPPDEFINES = {'HAVE_STRING_H': 1})
+        if stdlib: conf.env.Append(CPPDEFINES = {'HAVE_STDLIB_H': 1})
     
     # Check if we can use pkg-config to find protobuf include path
     status, output = conf.TryAction('pkg-config protobuf --variable=includedir > $TARGET')
@@ -60,7 +76,23 @@ if not env.GetOption('clean'):
     if 'gcc' in env['CC']:
         if conf.CheckLib('mudflap'):
             conf.env.Append(CCFLAGS = '-fmudflap')
-            conf.env.Append(LINKFLAGS = '-lmudflap -fmudflap')
+            conf.env.Append(LINKFLAGS = '-fmudflap')
+    
+    # Check if we can use extra strict warning flags (only with GCC)
+    extra = '-Wcast-qual -Wlogical-op -Wconversion'
+    extra += ' -fstrict-aliasing -Wstrict-aliasing=1'
+    extra += ' -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls'
+    extra += ' -Wstack-protector '
+    if 'gcc' in env['CC']:
+        if conf.CheckCCFLAGS(extra):
+            conf.env.Append(CORECFLAGS = extra)
+    
+    # Check if we can use undefined behaviour sanitizer (only with clang)
+    extra = '-fsanitize=undefined '
+    if 'clang' in env['CC']:
+        if conf.CheckCCFLAGS(extra, linkflags = extra):
+            conf.env.Append(CORECFLAGS = extra)
+            conf.env.Append(LINKFLAGS = extra)
     
     # End the config stuff
     env = conf.Finish()
@@ -70,19 +102,15 @@ if 'gcc' in env['CC']:
     # GNU Compiler Collection
     
     # Debug info, warnings as errors
-    env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror --coverage -fstack-protector-all')
-    env.Append(LINKFLAGS = '--coverage')
+    env.Append(CFLAGS = '-ansi -pedantic -g -Wall -Werror -fprofile-arcs -ftest-coverage ')
+    env.Append(CORECFLAGS = '-Wextra')
+    env.Append(LINKFLAGS = '-g --coverage')
     
     # We currently need uint64_t anyway, even though ANSI C90 otherwise..
     env.Append(CFLAGS = '-Wno-long-long')
-
-    # More strict checks on the nanopb core
-    env.Append(CORECFLAGS = '-Wextra -Wcast-qual -Wlogical-op -Wconversion')
-    env.Append(CORECFLAGS = ' -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls')
-    env.Append(CORECFLAGS = ' -Wstack-protector')
 elif 'clang' in env['CC']:
     # CLang
-    env.Append(CFLAGS = '-ansi -g -O0 -Wall -Werror')
+    env.Append(CFLAGS = '-ansi -g -Wall -Werror')
     env.Append(CORECFLAGS = ' -Wextra -Wcast-qual -Wconversion')
 elif 'cl' in env['CC']:
     # Microsoft Visual C++
@@ -102,13 +130,24 @@ elif 'tcc' in env['CC']:
 
 env.SetDefault(CORECFLAGS = '')
 
-if 'clang++' in env['CXX']:
-    env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers')
-elif 'g++' in env['CXX']:
-    env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers')
+if 'clang' in env['CXX']:
+    env.Append(CXXFLAGS = '-g -Wall -Werror -Wextra -Wno-missing-field-initializers')
+elif 'g++' in env['CXX'] or 'gcc' in env['CXX']:
+    env.Append(CXXFLAGS = '-g -Wall -Werror -Wextra -Wno-missing-field-initializers')
 elif 'cl' in env['CXX']:
     env.Append(CXXFLAGS = '/Zi /W2 /WX')
 
 # Now include the SConscript files from all subdirectories
-SConscript(Glob('*/SConscript'), exports = 'env')
+import os.path
+env['VARIANT_DIR'] = 'build'
+env['BUILD'] = '#' + env['VARIANT_DIR']
+env['COMMON'] = '#' + env['VARIANT_DIR'] + '/common'
+
+# Include common/SConscript first to make sure its exports are available
+# to other SConscripts.
+SConscript("common/SConscript", exports = 'env', variant_dir = env['VARIANT_DIR'] + '/common')
+
+for subdir in Glob('*/SConscript') + Glob('regression/*/SConscript'):
+    if str(subdir).startswith("common"): continue
+    SConscript(subdir, exports = 'env', variant_dir = env['VARIANT_DIR'] + '/' + os.path.dirname(str(subdir)))
 

tests/alltypes/SConscript

@@ -3,10 +3,33 @@
 
 Import("env")
 
-env.NanopbProto("alltypes")
-enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "#common/pb_encode.o"])
-dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "#common/pb_decode.o"])
+env.NanopbProto(["alltypes", "alltypes.options"])
+enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o"])
+dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o"])
 
+# Test the round-trip from nanopb encoder to nanopb decoder
 env.RunTest(enc)
 env.RunTest([dec, "encode_alltypes.output"])
 
+# Re-encode the data using protoc, and check that the results from nanopb
+# match byte-per-byte to the protoc output.
+env.Decode("encode_alltypes.output.decoded",
+           ["encode_alltypes.output", "alltypes.proto"],
+           MESSAGE='AllTypes')
+env.Encode("encode_alltypes.output.recoded",
+           ["encode_alltypes.output.decoded", "alltypes.proto"],
+           MESSAGE='AllTypes')
+env.Compare(["encode_alltypes.output", "encode_alltypes.output.recoded"])
+
+# Do the same checks with the optional fields present.
+env.RunTest("optionals.output", enc, ARGS = ['1'])
+env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1'])
+env.Decode("optionals.output.decoded",
+           ["optionals.output", "alltypes.proto"],
+           MESSAGE='AllTypes')
+env.Encode("optionals.output.recoded",
+           ["optionals.output.decoded", "alltypes.proto"],
+           MESSAGE='AllTypes')
+env.Compare(["optionals.output", "optionals.output.recoded"])
+
+

tests/alltypes/alltypes.proto

@@ -8,6 +8,24 @@ message EmptyMessage {
 
 }
 
+enum HugeEnum {
+    Negative = -2147483647; /* protoc doesn't accept -2147483648 here */
+    Positive =  2147483647;
+}
+
+message Limits {
+    required int32      int32_min  =  1 [default = 2147483647];
+    required int32      int32_max  =  2 [default = -2147483647];
+    required uint32     uint32_min =  3 [default = 4294967295];
+    required uint32     uint32_max =  4 [default = 0];
+    required int64      int64_min  =  5 [default = 9223372036854775807];
+    required int64      int64_max  =  6 [default = -9223372036854775807];
+    required uint64     uint64_min =  7 [default = 18446744073709551615];
+    required uint64     uint64_max =  8 [default = 0];
+    required HugeEnum   enum_min   =  9 [default = Positive];
+    required HugeEnum   enum_max   = 10 [default = Negative];
+}
+
 enum MyEnum {
     Zero = 0;
     First = 1;
@@ -39,26 +57,26 @@ message AllTypes {
     required EmptyMessage req_emptymsg = 18;
     
     
-    repeated int32      rep_int32   = 21;
-    repeated int64      rep_int64   = 22;
-    repeated uint32     rep_uint32  = 23;
-    repeated uint64     rep_uint64  = 24;
-    repeated sint32     rep_sint32  = 25;
-    repeated sint64     rep_sint64  = 26;
-    repeated bool       rep_bool    = 27;
+    repeated int32      rep_int32   = 21 [packed = true];
+    repeated int64      rep_int64   = 22 [packed = true];
+    repeated uint32     rep_uint32  = 23 [packed = true];
+    repeated uint64     rep_uint64  = 24 [packed = true];
+    repeated sint32     rep_sint32  = 25 [packed = true];
+    repeated sint64     rep_sint64  = 26 [packed = true];
+    repeated bool       rep_bool    = 27 [packed = true];
     
-    repeated fixed32    rep_fixed32 = 28;
-    repeated sfixed32   rep_sfixed32= 29;
-    repeated float      rep_float   = 30;
+    repeated fixed32    rep_fixed32 = 28 [packed = true];
+    repeated sfixed32   rep_sfixed32= 29 [packed = true];
+    repeated float      rep_float   = 30 [packed = true];
     
-    repeated fixed64    rep_fixed64 = 31;
-    repeated sfixed64   rep_sfixed64= 32;
-    repeated double     rep_double  = 33;
+    repeated fixed64    rep_fixed64 = 31 [packed = true];
+    repeated sfixed64   rep_sfixed64= 32 [packed = true];
+    repeated double     rep_double  = 33 [packed = true];
     
     repeated string     rep_string  = 34;
     repeated bytes      rep_bytes   = 35;
     repeated SubMessage rep_submsg  = 36;
-    repeated MyEnum     rep_enum    = 37;
+    repeated MyEnum     rep_enum    = 37 [packed = true];
     repeated EmptyMessage rep_emptymsg = 38;
     
     optional int32      opt_int32   = 41 [default = 4041];
@@ -83,6 +101,9 @@ message AllTypes {
     optional MyEnum     opt_enum    = 57 [default = Second];
     optional EmptyMessage opt_emptymsg = 58;
 
+    // Check that extreme integer values are handled correctly
+    required Limits     req_limits = 98;
+
     // Just to make sure that the size of the fields has been calculated
     // properly, i.e. otherwise a bug in last field might not be detected.
     required int32      end = 99;

tests/alltypes/decode_alltypes.c

@@ -19,10 +19,12 @@
    the decoding and checks the fields. */
 bool check_alltypes(pb_istream_t *stream, int mode)
 {
-    AllTypes alltypes;
+    /* Uses _init_default to just make sure that it works. */
+    AllTypes alltypes = AllTypes_init_default;
     
     /* Fill with garbage to better detect initialization errors */
     memset(&alltypes, 0xAA, sizeof(alltypes));
+    alltypes.extensions = 0;
     
     if (!pb_decode(stream, AllTypes_fields, &alltypes))
         return false;
@@ -170,6 +172,17 @@ bool check_alltypes(pb_istream_t *stream, int mode)
         TEST(alltypes.has_opt_emptymsg  == true);
     }
     
+    TEST(alltypes.req_limits.int32_min  == INT32_MIN);
+    TEST(alltypes.req_limits.int32_max  == INT32_MAX);
+    TEST(alltypes.req_limits.uint32_min == 0);
+    TEST(alltypes.req_limits.uint32_max == UINT32_MAX);
+    TEST(alltypes.req_limits.int64_min  == INT64_MIN);
+    TEST(alltypes.req_limits.int64_max  == INT64_MAX);
+    TEST(alltypes.req_limits.uint64_min == 0);
+    TEST(alltypes.req_limits.uint64_max == UINT64_MAX);
+    TEST(alltypes.req_limits.enum_min   == HugeEnum_Negative);
+    TEST(alltypes.req_limits.enum_max   == HugeEnum_Positive);
+    
     TEST(alltypes.end == 1099);
     
     return true;

tests/alltypes/encode_alltypes.c

@@ -13,7 +13,7 @@ int main(int argc, char **argv)
     int mode = (argc > 1) ? atoi(argv[1]) : 0;
     
     /* Initialize the structure with constants */
-    AllTypes alltypes = {0};
+    AllTypes alltypes = AllTypes_init_zero;
     
     alltypes.req_int32         = -1001;
     alltypes.req_int64         = -1002;
@@ -67,6 +67,17 @@ int main(int argc, char **argv)
     alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth;
     alltypes.rep_emptymsg_count = 5;
     
+    alltypes.req_limits.int32_min  = INT32_MIN;
+    alltypes.req_limits.int32_max  = INT32_MAX;
+    alltypes.req_limits.uint32_min = 0;
+    alltypes.req_limits.uint32_max = UINT32_MAX;
+    alltypes.req_limits.int64_min  = INT64_MIN;
+    alltypes.req_limits.int64_max  = INT64_MAX;
+    alltypes.req_limits.uint64_min = 0;
+    alltypes.req_limits.uint64_max = UINT64_MAX;
+    alltypes.req_limits.enum_min   = HugeEnum_Negative;
+    alltypes.req_limits.enum_max   = HugeEnum_Positive;
+    
     if (mode != 0)
     {
         /* Fill in values for optional fields */

tests/alltypes_callback/SConscript

@@ -0,0 +1,23 @@
+# Test the AllTypes encoding & decoding using callbacks for all fields.
+
+Import("env")
+
+c = Copy("$TARGET", "$SOURCE")
+env.Command("alltypes.proto", "#alltypes/alltypes.proto", c)
+
+env.NanopbProto(["alltypes", "alltypes.options"])
+enc = env.Program(["encode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_encode.o"])
+dec = env.Program(["decode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_decode.o"])
+
+refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX"
+
+# Encode and compare results
+env.RunTest(enc)
+env.RunTest("decode_alltypes.output", [refdec, "encode_alltypes_callback.output"])
+env.RunTest("decode_alltypes_callback.output", [dec, "encode_alltypes_callback.output"])
+
+# Do the same thing with the optional fields present
+env.RunTest("optionals.output", enc, ARGS = ['1'])
+env.RunTest("optionals.refdecout", [refdec, "optionals.output"], ARGS = ['1'])
+env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1'])
+

tests/alltypes_callback/alltypes.options

@@ -0,0 +1,3 @@
+# Generate all fields as callbacks.
+AllTypes.*              type:FT_CALLBACK
+SubMessage.substuff1    max_size:16

tests/alltypes_callback/decode_alltypes_callback.c

@@ -0,0 +1,424 @@
+/* Attempts to test all the datatypes supported by ProtoBuf when used as callback fields.
+ * Note that normally there would be no reason to use callback fields for this,
+ * because each encoder defined here only gives a single field.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <pb_decode.h>
+#include "alltypes.pb.h"
+#include "test_helpers.h"
+
+#define TEST(x) if (!(x)) { \
+    printf("Test " #x " failed (in field %d).\n", field->tag); \
+    return false; \
+    }
+
+static bool read_varint(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    uint64_t value;
+    if (!pb_decode_varint(stream, &value))
+        return false;
+    
+    TEST((int64_t)value == (long)*arg);
+    return true;
+}
+
+static bool read_svarint(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    int64_t value;
+    if (!pb_decode_svarint(stream, &value))
+        return false;
+    
+    TEST(value == (long)*arg);
+    return true;
+}
+
+static bool read_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    uint32_t value;
+    if (!pb_decode_fixed32(stream, &value))
+        return false;
+    
+    TEST(value == *(uint32_t*)*arg);
+    return true;
+}
+
+static bool read_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    uint64_t value;
+    if (!pb_decode_fixed64(stream, &value))
+        return false;
+    
+    TEST(value == *(uint64_t*)*arg);
+    return true;
+}
+
+static bool read_string(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    uint8_t buf[16] = {0};
+    size_t len = stream->bytes_left;
+    
+    if (len > sizeof(buf) - 1 || !pb_read(stream, buf, len))
+        return false;
+    
+    TEST(strcmp((char*)buf, *arg) == 0);
+    return true;
+}
+
+static bool read_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    SubMessage submsg = {""};
+    
+    if (!pb_decode(stream, SubMessage_fields, &submsg))
+        return false;
+    
+    TEST(memcmp(&submsg, *arg, sizeof(submsg)));
+    return true;
+}
+
+static bool read_emptymsg(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    EmptyMessage emptymsg = {0};
+    return pb_decode(stream, EmptyMessage_fields, &emptymsg);
+}
+
+static bool read_repeated_varint(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    int32_t** expected = (int32_t**)arg;
+    uint64_t value;
+    if (!pb_decode_varint(stream, &value))
+        return false;
+
+    TEST(*(*expected)++ == value);
+    return true;
+}
+
+static bool read_repeated_svarint(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    int32_t** expected = (int32_t**)arg;
+    int64_t value;
+    if (!pb_decode_svarint(stream, &value))
+        return false;
+
+    TEST(*(*expected)++ == value);
+    return true;
+}
+
+static bool read_repeated_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    uint32_t** expected = (uint32_t**)arg;
+    uint32_t value;
+    if (!pb_decode_fixed32(stream, &value))
+        return false;
+
+    TEST(*(*expected)++ == value);
+    return true;
+}
+
+static bool read_repeated_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    uint64_t** expected = (uint64_t**)arg;
+    uint64_t value;
+    if (!pb_decode_fixed64(stream, &value))
+        return false;
+
+    TEST(*(*expected)++ == value);
+    return true;
+}
+
+static bool read_repeated_string(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    uint8_t*** expected = (uint8_t***)arg;
+    uint8_t buf[16] = {0};
+    size_t len = stream->bytes_left;
+    
+    if (len > sizeof(buf) - 1 || !pb_read(stream, buf, len))
+        return false;
+    
+    TEST(strcmp((char*)*(*expected)++, (char*)buf) == 0);
+    return true;
+}
+
+static bool read_repeated_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    SubMessage** expected = (SubMessage**)arg;
+    SubMessage decoded = {""};
+    if (!pb_decode(stream, SubMessage_fields, &decoded))
+        return false;
+
+    TEST(memcmp((*expected)++, &decoded, sizeof(decoded)) == 0);
+    return true;
+}
+
+static bool read_limits(pb_istream_t *stream, const pb_field_t *field, void **arg)
+{
+    Limits decoded = {0};
+    if (!pb_decode(stream, Limits_fields, &decoded))
+        return false;
+
+    TEST(decoded.int32_min  == INT32_MIN);
+    TEST(decoded.int32_max  == INT32_MAX);
+    TEST(decoded.uint32_min == 0);
+    TEST(decoded.uint32_max == UINT32_MAX);
+    TEST(decoded.int64_min  == INT64_MIN);
+    TEST(decoded.int64_max  == INT64_MAX);
+    TEST(decoded.uint64_min == 0);
+    TEST(decoded.uint64_max == UINT64_MAX);
+    TEST(decoded.enum_min   == HugeEnum_Negative);
+    TEST(decoded.enum_max   == HugeEnum_Positive);
+    
+    return true;
+}
+
+/* This function is called once from main(), it handles
+   the decoding and checks the fields. */
+bool check_alltypes(pb_istream_t *stream, int mode)
+{
+    /* Values for use from callbacks through pointers. */
+    uint32_t    req_fixed32     = 1008;
+    int32_t     req_sfixed32    = -1009;
+    float       req_float       = 1010.0f;
+    uint64_t    req_fixed64     = 1011;
+    int64_t     req_sfixed64    = -1012;
+    double      req_double      = 1013.0;
+    SubMessage  req_submsg      = {"1016", 1016};
+    
+    int32_t     rep_int32[5]    = {0, 0, 0, 0, -2001};
+    int32_t     rep_int64[5]    = {0, 0, 0, 0, -2002};
+    int32_t     rep_uint32[5]   = {0, 0, 0, 0,  2003};
+    int32_t     rep_uint64[5]   = {0, 0, 0, 0,  2004};
+    int32_t     rep_sint32[5]   = {0, 0, 0, 0, -2005};
+    int32_t     rep_sint64[5]   = {0, 0, 0, 0, -2006};
+    int32_t     rep_bool[5]     = {false, false, false, false, true};
+    uint32_t    rep_fixed32[5]  = {0, 0, 0, 0,  2008};
+    int32_t     rep_sfixed32[5] = {0, 0, 0, 0, -2009};
+    float       rep_float[5]    = {0, 0, 0, 0,  2010.0f};
+    uint64_t    rep_fixed64[5]  = {0, 0, 0, 0,  2011};
+    int64_t     rep_sfixed64[5] = {0, 0, 0, 0, -2012};
+    double      rep_double[5]   = {0, 0, 0, 0,  2013.0};
+    char*       rep_string[5]   = {"", "", "", "", "2014"};
+    char*       rep_bytes[5]    = {"", "", "", "", "2015"};
+    SubMessage  rep_submsg[5]   = {{"", 0, 0, 3},
+                                   {"", 0, 0, 3},
+                                   {"", 0, 0, 3},
+                                   {"", 0, 0, 3},
+                                   {"2016", 2016, true, 2016}};
+    int32_t     rep_enum[5]     = {0, 0, 0, 0, MyEnum_Truth};
+    
+    uint32_t    opt_fixed32     = 3048;
+    int32_t     opt_sfixed32    = 3049;
+    float       opt_float       = 3050.0f;
+    uint64_t    opt_fixed64     = 3051;
+    int64_t     opt_sfixed64    = 3052;
+    double      opt_double      = 3053.0f;
+    SubMessage  opt_submsg      = {"3056", 3056};
+    
+    /* Bind callbacks for required fields */
+    AllTypes alltypes;
+    
+    /* Fill with garbage to better detect initialization errors */
+    memset(&alltypes, 0xAA, sizeof(alltypes));
+    alltypes.extensions = 0;
+    
+    alltypes.req_int32.funcs.decode = &read_varint;
+    alltypes.req_int32.arg = (void*)-1001;
+    
+    alltypes.req_int64.funcs.decode = &read_varint;
+    alltypes.req_int64.arg = (void*)-1002;
+    
+    alltypes.req_uint32.funcs.decode = &read_varint;
+    alltypes.req_uint32.arg = (void*)1003;
+
+    alltypes.req_uint32.funcs.decode = &read_varint;
+    alltypes.req_uint32.arg = (void*)1003;    
+    
+    alltypes.req_uint64.funcs.decode = &read_varint;
+    alltypes.req_uint64.arg = (void*)1004;
+    
+    alltypes.req_sint32.funcs.decode = &read_svarint;
+    alltypes.req_sint32.arg = (void*)-1005;   
+    
+    alltypes.req_sint64.funcs.decode = &read_svarint;
+    alltypes.req_sint64.arg = (void*)-1006;   
+    
+    alltypes.req_bool.funcs.decode = &read_varint;
+    alltypes.req_bool.arg = (void*)true;   
+    
+    alltypes.req_fixed32.funcs.decode = &read_fixed32;
+    alltypes.req_fixed32.arg = &req_fixed32;
+    
+    alltypes.req_sfixed32.funcs.decode = &read_fixed32;
+    alltypes.req_sfixed32.arg = &req_sfixed32;
+    
+    alltypes.req_float.funcs.decode = &read_fixed32;
+    alltypes.req_float.arg = &req_float;
+    
+    alltypes.req_fixed64.funcs.decode = &read_fixed64;
+    alltypes.req_fixed64.arg = &req_fixed64;
+    
+    alltypes.req_sfixed64.funcs.decode = &read_fixed64;
+    alltypes.req_sfixed64.arg = &req_sfixed64;
+    
+    alltypes.req_double.funcs.decode = &read_fixed64;
+    alltypes.req_double.arg = &req_double;
+    
+    alltypes.req_string.funcs.decode = &read_string;
+    alltypes.req_string.arg = "1014";
+    
+    alltypes.req_bytes.funcs.decode = &read_string;
+    alltypes.req_bytes.arg = "1015";
+    
+    alltypes.req_submsg.funcs.decode = &read_submsg;
+    alltypes.req_submsg.arg = &req_submsg;
+    
+    alltypes.req_enum.funcs.decode = &read_varint;
+    alltypes.req_enum.arg = (void*)MyEnum_Truth;
+    
+    alltypes.req_emptymsg.funcs.decode = &read_emptymsg;
+    
+    /* Bind callbacks for repeated fields */
+    alltypes.rep_int32.funcs.decode = &read_repeated_varint;
+    alltypes.rep_int32.arg = rep_int32;
+    
+    alltypes.rep_int64.funcs.decode = &read_repeated_varint;
+    alltypes.rep_int64.arg = rep_int64;
+    
+    alltypes.rep_uint32.funcs.decode = &read_repeated_varint;
+    alltypes.rep_uint32.arg = rep_uint32;
+    
+    alltypes.rep_uint64.funcs.decode = &read_repeated_varint;
+    alltypes.rep_uint64.arg = rep_uint64;
+    
+    alltypes.rep_sint32.funcs.decode = &read_repeated_svarint;
+    alltypes.rep_sint32.arg = rep_sint32;
+    
+    alltypes.rep_sint64.funcs.decode = &read_repeated_svarint;
+    alltypes.rep_sint64.arg = rep_sint64;
+    
+    alltypes.rep_bool.funcs.decode = &read_repeated_varint;
+    alltypes.rep_bool.arg = rep_bool;
+    
+    alltypes.rep_fixed32.funcs.decode = &read_repeated_fixed32;
+    alltypes.rep_fixed32.arg = rep_fixed32;
+    
+    alltypes.rep_sfixed32.funcs.decode = &read_repeated_fixed32;
+    alltypes.rep_sfixed32.arg = rep_sfixed32;
+    
+    alltypes.rep_float.funcs.decode = &read_repeated_fixed32;
+    alltypes.rep_float.arg = rep_float;
+    
+    alltypes.rep_fixed64.funcs.decode = &read_repeated_fixed64;
+    alltypes.rep_fixed64.arg = rep_fixed64;
+    
+    alltypes.rep_sfixed64.funcs.decode = &read_repeated_fixed64;
+    alltypes.rep_sfixed64.arg = rep_sfixed64;
+    
+    alltypes.rep_double.funcs.decode = &read_repeated_fixed64;
+    alltypes.rep_double.arg = rep_double;
+    
+    alltypes.rep_string.funcs.decode = &read_repeated_string;
+    alltypes.rep_string.arg = rep_string;
+    
+    alltypes.rep_bytes.funcs.decode = &read_repeated_string;
+    alltypes.rep_bytes.arg = rep_bytes;
+    
+    alltypes.rep_submsg.funcs.decode = &read_repeated_submsg;
+    alltypes.rep_submsg.arg = rep_submsg;
+    
+    alltypes.rep_enum.funcs.decode = &read_repeated_varint;
+    alltypes.rep_enum.arg = rep_enum;
+    
+    alltypes.rep_emptymsg.funcs.decode = &read_emptymsg;
+    
+    alltypes.req_limits.funcs.decode = &read_limits;
+    
+    alltypes.end.funcs.decode = &read_varint;
+    alltypes.end.arg = (void*)1099;
+    
+    /* Bind callbacks for optional fields */
+    if (mode == 1)
+    {
+        alltypes.opt_int32.funcs.decode = &read_varint;
+        alltypes.opt_int32.arg = (void*)3041;
+        
+        alltypes.opt_int64.funcs.decode = &read_varint;
+        alltypes.opt_int64.arg = (void*)3042;
+        
+        alltypes.opt_uint32.funcs.decode = &read_varint;
+        alltypes.opt_uint32.arg = (void*)3043;
+        
+        alltypes.opt_uint64.funcs.decode = &read_varint;
+        alltypes.opt_uint64.arg = (void*)3044;
+        
+        alltypes.opt_sint32.funcs.decode = &read_svarint;
+        alltypes.opt_sint32.arg = (void*)3045;
+        
+        alltypes.opt_sint64.funcs.decode = &read_svarint;
+        alltypes.opt_sint64.arg = (void*)3046;
+        
+        alltypes.opt_bool.funcs.decode = &read_varint;
+        alltypes.opt_bool.arg = (void*)true;
+
+        alltypes.opt_fixed32.funcs.decode = &read_fixed32;
+        alltypes.opt_fixed32.arg = &opt_fixed32;
+        
+        alltypes.opt_sfixed32.funcs.decode = &read_fixed32;
+        alltypes.opt_sfixed32.arg = &opt_sfixed32;
+        
+        alltypes.opt_float.funcs.decode = &read_fixed32;
+        alltypes.opt_float.arg = &opt_float;
+        
+        alltypes.opt_fixed64.funcs.decode = &read_fixed64;
+        alltypes.opt_fixed64.arg = &opt_fixed64;
+        
+        alltypes.opt_sfixed64.funcs.decode = &read_fixed64;
+        alltypes.opt_sfixed64.arg = &opt_sfixed64;
+        
+        alltypes.opt_double.funcs.decode = &read_fixed64;
+        alltypes.opt_double.arg = &opt_double;
+        
+        alltypes.opt_string.funcs.decode = &read_string;
+        alltypes.opt_string.arg = "3054";
+        
+        alltypes.opt_bytes.funcs.decode = &read_string;
+        alltypes.opt_bytes.arg = "3055";
+        
+        alltypes.opt_submsg.funcs.decode = &read_submsg;
+        alltypes.opt_submsg.arg = &opt_submsg;
+        
+        alltypes.opt_enum.funcs.decode = &read_varint;
+        alltypes.opt_enum.arg = (void*)MyEnum_Truth;
+        
+        alltypes.opt_emptymsg.funcs.decode = &read_emptymsg;
+    }
+    
+    return pb_decode(stream, AllTypes_fields, &alltypes);
+}
+
+int main(int argc, char **argv)
+{
+    uint8_t buffer[1024];
+    size_t count;
+    pb_istream_t stream;
+
+    /* Whether to expect the optional values or the default values. */
+    int mode = (argc > 1) ? atoi(argv[1]) : 0;
+    
+    /* Read the data into buffer */
+    SET_BINARY_MODE(stdin);
+    count = fread(buffer, 1, sizeof(buffer), stdin);
+    
+    /* Construct a pb_istream_t for reading from the buffer */
+    stream = pb_istream_from_buffer(buffer, count);
+    
+    /* Decode and print out the stuff */
+    if (!check_alltypes(&stream, mode))
+    {
+        printf("Parsing failed: %s\n", PB_GET_ERROR(&stream));
+        return 1;
+    } else {
+        return 0;
+    }
+}

tests/alltypes_callback/encode_alltypes_callback.c

@@ -0,0 +1,397 @@
+/* Attempts to test all the datatypes supported by ProtoBuf when used as callback fields.
+ * Note that normally there would be no reason to use callback fields for this,
+ * because each encoder defined here only gives a single field.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pb_encode.h>
+#include "alltypes.pb.h"
+#include "test_helpers.h"
+
+static bool write_varint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_varint(stream, (long)*arg);
+}
+
+static bool write_svarint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_svarint(stream, (long)*arg);
+}
+
+static bool write_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_fixed32(stream, *arg);
+}
+
+static bool write_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_fixed64(stream, *arg);
+}
+
+static bool write_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_string(stream, *arg, strlen(*arg));
+}
+
+static bool write_submsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+   
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, SubMessage_fields, *arg);
+}
+
+static bool write_emptymsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    EmptyMessage emptymsg = {0};
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg);
+}
+
+static bool write_repeated_varint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_varint(stream, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_varint(stream, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_varint(stream, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_varint(stream, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_varint(stream, (long)*arg);
+}
+
+static bool write_repeated_svarint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_svarint(stream, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_svarint(stream, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_svarint(stream, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_svarint(stream, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_svarint(stream, (long)*arg);
+}
+
+static bool write_repeated_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    uint32_t dummy = 0;
+
+    /* Make it a packed field */
+    return pb_encode_tag(stream, PB_WT_STRING, field->tag) &&
+           pb_encode_varint(stream, 5 * 4) && /* Number of bytes */
+           pb_encode_fixed32(stream, &dummy) &&
+           pb_encode_fixed32(stream, &dummy) &&
+           pb_encode_fixed32(stream, &dummy) &&
+           pb_encode_fixed32(stream, &dummy) &&
+           pb_encode_fixed32(stream, *arg);
+}
+
+static bool write_repeated_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    uint64_t dummy = 0;
+
+    /* Make it a packed field */
+    return pb_encode_tag(stream, PB_WT_STRING, field->tag) &&
+           pb_encode_varint(stream, 5 * 8) && /* Number of bytes */
+           pb_encode_fixed64(stream, &dummy) &&
+           pb_encode_fixed64(stream, &dummy) &&
+           pb_encode_fixed64(stream, &dummy) &&
+           pb_encode_fixed64(stream, &dummy) &&
+           pb_encode_fixed64(stream, *arg);
+}
+
+static bool write_repeated_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_string(stream, 0, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_string(stream, 0, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_string(stream, 0, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_string(stream, 0, 0) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_string(stream, *arg, strlen(*arg));
+}
+
+static bool write_repeated_submsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    SubMessage dummy = {""};
+
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, SubMessage_fields, &dummy) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, SubMessage_fields, &dummy) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, SubMessage_fields, &dummy) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, SubMessage_fields, &dummy) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, SubMessage_fields, *arg);
+}
+
+static bool write_limits(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    Limits limits = {0};
+    limits.int32_min  = INT32_MIN;
+    limits.int32_max  = INT32_MAX;
+    limits.uint32_min = 0;
+    limits.uint32_max = UINT32_MAX;
+    limits.int64_min  = INT64_MIN;
+    limits.int64_max  = INT64_MAX;
+    limits.uint64_min = 0;
+    limits.uint64_max = UINT64_MAX;
+    limits.enum_min   = HugeEnum_Negative;
+    limits.enum_max   = HugeEnum_Positive;
+   
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, Limits_fields, &limits);
+}
+
+static bool write_repeated_emptymsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    EmptyMessage emptymsg = {0};
+    return pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg);
+}
+
+int main(int argc, char **argv)
+{
+    int mode = (argc > 1) ? atoi(argv[1]) : 0;
+
+    /* Values for use from callbacks through pointers. */
+    uint32_t    req_fixed32     = 1008;
+    int32_t     req_sfixed32    = -1009;
+    float       req_float       = 1010.0f;
+    uint64_t    req_fixed64     = 1011;
+    int64_t     req_sfixed64    = -1012;
+    double      req_double      = 1013.0;
+    SubMessage  req_submsg      = {"1016", 1016};
+    
+    uint32_t    rep_fixed32     = 2008;
+    int32_t     rep_sfixed32    = -2009;
+    float       rep_float       = 2010.0f;
+    uint64_t    rep_fixed64     = 2011;
+    int64_t     rep_sfixed64    = -2012;
+    double      rep_double      = 2013.0;
+    SubMessage  rep_submsg      = {"2016", 2016, true, 2016};
+    
+    uint32_t    opt_fixed32     = 3048;
+    int32_t     opt_sfixed32    = 3049;
+    float       opt_float       = 3050.0f;
+    uint64_t    opt_fixed64     = 3051;
+    int64_t     opt_sfixed64    = 3052;
+    double      opt_double      = 3053.0f;
+    SubMessage  opt_submsg      = {"3056", 3056};
+    
+    /* Bind callbacks for required fields */
+    AllTypes alltypes = {{{0}}};
+    
+    alltypes.req_int32.funcs.encode = &write_varint;
+    alltypes.req_int32.arg = (void*)-1001;
+    
+    alltypes.req_int64.funcs.encode = &write_varint;
+    alltypes.req_int64.arg = (void*)-1002;
+    
+    alltypes.req_uint32.funcs.encode = &write_varint;
+    alltypes.req_uint32.arg = (void*)1003;
+
+    alltypes.req_uint32.funcs.encode = &write_varint;
+    alltypes.req_uint32.arg = (void*)1003;    
+    
+    alltypes.req_uint64.funcs.encode = &write_varint;
+    alltypes.req_uint64.arg = (void*)1004;
+    
+    alltypes.req_sint32.funcs.encode = &write_svarint;
+    alltypes.req_sint32.arg = (void*)-1005;   
+    
+    alltypes.req_sint64.funcs.encode = &write_svarint;
+    alltypes.req_sint64.arg = (void*)-1006;   
+    
+    alltypes.req_bool.funcs.encode = &write_varint;
+    alltypes.req_bool.arg = (void*)true;   
+    
+    alltypes.req_fixed32.funcs.encode = &write_fixed32;
+    alltypes.req_fixed32.arg = &req_fixed32;
+    
+    alltypes.req_sfixed32.funcs.encode = &write_fixed32;
+    alltypes.req_sfixed32.arg = &req_sfixed32;
+    
+    alltypes.req_float.funcs.encode = &write_fixed32;
+    alltypes.req_float.arg = &req_float;
+    
+    alltypes.req_fixed64.funcs.encode = &write_fixed64;
+    alltypes.req_fixed64.arg = &req_fixed64;
+    
+    alltypes.req_sfixed64.funcs.encode = &write_fixed64;
+    alltypes.req_sfixed64.arg = &req_sfixed64;
+    
+    alltypes.req_double.funcs.encode = &write_fixed64;
+    alltypes.req_double.arg = &req_double;
+    
+    alltypes.req_string.funcs.encode = &write_string;
+    alltypes.req_string.arg = "1014";
+    
+    alltypes.req_bytes.funcs.encode = &write_string;
+    alltypes.req_bytes.arg = "1015";
+    
+    alltypes.req_submsg.funcs.encode = &write_submsg;
+    alltypes.req_submsg.arg = &req_submsg;
+    
+    alltypes.req_enum.funcs.encode = &write_varint;
+    alltypes.req_enum.arg = (void*)MyEnum_Truth;
+    
+    alltypes.req_emptymsg.funcs.encode = &write_emptymsg;
+    
+    /* Bind callbacks for repeated fields */
+    alltypes.rep_int32.funcs.encode = &write_repeated_varint;
+    alltypes.rep_int32.arg = (void*)-2001;
+    
+    alltypes.rep_int64.funcs.encode = &write_repeated_varint;
+    alltypes.rep_int64.arg = (void*)-2002;
+    
+    alltypes.rep_uint32.funcs.encode = &write_repeated_varint;
+    alltypes.rep_uint32.arg = (void*)2003;
+    
+    alltypes.rep_uint64.funcs.encode = &write_repeated_varint;
+    alltypes.rep_uint64.arg = (void*)2004;
+    
+    alltypes.rep_sint32.funcs.encode = &write_repeated_svarint;
+    alltypes.rep_sint32.arg = (void*)-2005;
+    
+    alltypes.rep_sint64.funcs.encode = &write_repeated_svarint;
+    alltypes.rep_sint64.arg = (void*)-2006;
+    
+    alltypes.rep_bool.funcs.encode = &write_repeated_varint;
+    alltypes.rep_bool.arg = (void*)true;
+    
+    alltypes.rep_fixed32.funcs.encode = &write_repeated_fixed32;
+    alltypes.rep_fixed32.arg = &rep_fixed32;
+
+    alltypes.rep_sfixed32.funcs.encode = &write_repeated_fixed32;
+    alltypes.rep_sfixed32.arg = &rep_sfixed32;
+    
+    alltypes.rep_float.funcs.encode = &write_repeated_fixed32;
+    alltypes.rep_float.arg = &rep_float;
+    
+    alltypes.rep_fixed64.funcs.encode = &write_repeated_fixed64;
+    alltypes.rep_fixed64.arg = &rep_fixed64;
+
+    alltypes.rep_sfixed64.funcs.encode = &write_repeated_fixed64;
+    alltypes.rep_sfixed64.arg = &rep_sfixed64;
+    
+    alltypes.rep_double.funcs.encode = &write_repeated_fixed64;
+    alltypes.rep_double.arg = &rep_double;
+    
+    alltypes.rep_string.funcs.encode = &write_repeated_string;
+    alltypes.rep_string.arg = "2014";
+    
+    alltypes.rep_bytes.funcs.encode = &write_repeated_string;
+    alltypes.rep_bytes.arg = "2015";
+    
+    alltypes.rep_submsg.funcs.encode = &write_repeated_submsg;
+    alltypes.rep_submsg.arg = &rep_submsg;
+    
+    alltypes.rep_enum.funcs.encode = &write_repeated_varint;
+    alltypes.rep_enum.arg = (void*)MyEnum_Truth;
+    
+    alltypes.rep_emptymsg.funcs.encode = &write_repeated_emptymsg;
+    
+    alltypes.req_limits.funcs.encode = &write_limits;
+    
+    /* Bind callbacks for optional fields */
+    if (mode != 0)
+    {
+        alltypes.opt_int32.funcs.encode = &write_varint;
+        alltypes.opt_int32.arg = (void*)3041;
+        
+        alltypes.opt_int64.funcs.encode = &write_varint;
+        alltypes.opt_int64.arg = (void*)3042;
+        
+        alltypes.opt_uint32.funcs.encode = &write_varint;
+        alltypes.opt_uint32.arg = (void*)3043;
+        
+        alltypes.opt_uint64.funcs.encode = &write_varint;
+        alltypes.opt_uint64.arg = (void*)3044;
+        
+        alltypes.opt_sint32.funcs.encode = &write_svarint;
+        alltypes.opt_sint32.arg = (void*)3045;
+        
+        alltypes.opt_sint64.funcs.encode = &write_svarint;
+        alltypes.opt_sint64.arg = (void*)3046;
+        
+        alltypes.opt_bool.funcs.encode = &write_varint;
+        alltypes.opt_bool.arg = (void*)true;
+
+        alltypes.opt_fixed32.funcs.encode = &write_fixed32;
+        alltypes.opt_fixed32.arg = &opt_fixed32;
+        
+        alltypes.opt_sfixed32.funcs.encode = &write_fixed32;
+        alltypes.opt_sfixed32.arg = &opt_sfixed32;
+        
+        alltypes.opt_float.funcs.encode = &write_fixed32;
+        alltypes.opt_float.arg = &opt_float;
+        
+        alltypes.opt_fixed64.funcs.encode = &write_fixed64;
+        alltypes.opt_fixed64.arg = &opt_fixed64;
+        
+        alltypes.opt_sfixed64.funcs.encode = &write_fixed64;
+        alltypes.opt_sfixed64.arg = &opt_sfixed64;
+        
+        alltypes.opt_double.funcs.encode = &write_fixed64;
+        alltypes.opt_double.arg = &opt_double;
+        
+        alltypes.opt_string.funcs.encode = &write_string;
+        alltypes.opt_string.arg = "3054";
+        
+        alltypes.opt_bytes.funcs.encode = &write_string;
+        alltypes.opt_bytes.arg = "3055";
+        
+        alltypes.opt_submsg.funcs.encode = &write_submsg;
+        alltypes.opt_submsg.arg = &opt_submsg;
+        
+        alltypes.opt_enum.funcs.encode = &write_varint;
+        alltypes.opt_enum.arg = (void*)MyEnum_Truth;
+        
+        alltypes.opt_emptymsg.funcs.encode = &write_emptymsg;
+    }
+    
+    alltypes.end.funcs.encode = &write_varint;
+    alltypes.end.arg = (void*)1099;
+    
+    {
+        uint8_t buffer[2048];
+        pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        
+        /* Now encode it and check if we succeeded. */
+        if (pb_encode(&stream, AllTypes_fields, &alltypes))
+        {
+            SET_BINARY_MODE(stdout);
+            fwrite(buffer, 1, stream.bytes_written, stdout);
+            return 0; /* Success */
+        }
+        else
+        {
+            fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
+            return 1; /* Failure */
+        }
+    }
+}

tests/alltypes_pointer/SConscript

@@ -0,0 +1,38 @@
+# Encode the AllTypes message using pointers for all fields, and verify the
+# output against the normal AllTypes test case.
+
+Import("env", "malloc_env")
+
+c = Copy("$TARGET", "$SOURCE")
+env.Command("alltypes.proto", "#alltypes/alltypes.proto", c)
+
+env.NanopbProto(["alltypes", "alltypes.options"])
+enc = malloc_env.Program(["encode_alltypes_pointer.c",
+                          "alltypes.pb.c",
+                          "$COMMON/pb_encode_with_malloc.o",
+                          "$COMMON/malloc_wrappers.o"])
+dec = malloc_env.Program(["decode_alltypes_pointer.c",
+                          "alltypes.pb.c",
+                          "$COMMON/pb_decode_with_malloc.o",
+                          "$COMMON/malloc_wrappers.o"])
+
+# Encode and compare results to non-pointer alltypes test case
+env.RunTest(enc)
+env.Compare(["encode_alltypes_pointer.output", "$BUILD/alltypes/encode_alltypes.output"])
+
+# Decode (under valgrind if available)
+valgrind = env.WhereIs('valgrind')
+kwargs = {}
+if valgrind:
+    kwargs['COMMAND'] = valgrind
+    kwargs['ARGS'] = ["-q", dec[0].abspath]
+
+env.RunTest("decode_alltypes.output", [dec, "encode_alltypes_pointer.output"], **kwargs)
+
+# Do the same thing with the optional fields present
+env.RunTest("optionals.output", enc, ARGS = ['1'])
+env.Compare(["optionals.output", "$BUILD/alltypes/optionals.output"])
+
+kwargs['ARGS'] = kwargs.get('ARGS', []) + ['1']
+env.RunTest("optionals.decout", [dec, "optionals.output"], **kwargs)
+

tests/alltypes_pointer/alltypes.options

@@ -0,0 +1,3 @@
+# Generate all fields as pointers.
+* type:FT_POINTER
+

tests/alltypes_pointer/decode_alltypes_pointer.c

@@ -0,0 +1,174 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <pb_decode.h>
+#include "alltypes.pb.h"
+#include "test_helpers.h"
+
+#define TEST(x) if (!(x)) { \
+    fprintf(stderr, "Test " #x " failed.\n"); \
+    status = false; \
+    }
+
+/* This function is called once from main(), it handles
+   the decoding and checks the fields. */
+bool check_alltypes(pb_istream_t *stream, int mode)
+{
+    bool status = true;
+    AllTypes alltypes;
+    
+    /* Fill with garbage to better detect initialization errors */
+    memset(&alltypes, 0xAA, sizeof(alltypes));
+    alltypes.extensions = 0;
+    
+    if (!pb_decode(stream, AllTypes_fields, &alltypes))
+        return false;
+    
+    TEST(alltypes.req_int32     && *alltypes.req_int32         == -1001);
+    TEST(alltypes.req_int64     && *alltypes.req_int64         == -1002);
+    TEST(alltypes.req_uint32    && *alltypes.req_uint32        == 1003);
+    TEST(alltypes.req_uint64    && *alltypes.req_uint64        == 1004);
+    TEST(alltypes.req_sint32    && *alltypes.req_sint32        == -1005);
+    TEST(alltypes.req_sint64    && *alltypes.req_sint64        == -1006);
+    TEST(alltypes.req_bool      && *alltypes.req_bool          == true);
+    
+    TEST(alltypes.req_fixed32   && *alltypes.req_fixed32       == 1008);
+    TEST(alltypes.req_sfixed32  && *alltypes.req_sfixed32      == -1009);
+    TEST(alltypes.req_float     && *alltypes.req_float         == 1010.0f);
+    
+    TEST(alltypes.req_fixed64   && *alltypes.req_fixed64       == 1011);
+    TEST(alltypes.req_sfixed64  && *alltypes.req_sfixed64      == -1012);
+    TEST(alltypes.req_double    && *alltypes.req_double        == 1013.0f);
+    
+    TEST(alltypes.req_string    && strcmp(alltypes.req_string, "1014") == 0);
+    TEST(alltypes.req_bytes     && alltypes.req_bytes->size == 4);
+    TEST(alltypes.req_bytes     && memcmp(&alltypes.req_bytes->bytes, "1015", 4) == 0);
+    TEST(alltypes.req_submsg    && alltypes.req_submsg->substuff1
+                                && strcmp(alltypes.req_submsg->substuff1, "1016") == 0);
+    TEST(alltypes.req_submsg    && alltypes.req_submsg->substuff2
+                                && *alltypes.req_submsg->substuff2 == 1016);
+    TEST(*alltypes.req_enum == MyEnum_Truth);
+
+    TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0);
+    TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0);
+    TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0);
+    TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0);
+    TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0);
+    TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0);
+    TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false);
+    
+    TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0);
+    TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0);
+    TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f);
+    
+    TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0);
+    TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0);
+    TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0);
+    
+    TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0');
+    TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4]->size == 4 && alltypes.rep_bytes[0]->size == 0);
+    TEST(memcmp(&alltypes.rep_bytes[4]->bytes, "2015", 4) == 0);
+
+    TEST(alltypes.rep_submsg_count == 5);
+    TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0');
+    TEST(*alltypes.rep_submsg[4].substuff2 == 2016 && *alltypes.rep_submsg[0].substuff2 == 0);
+    TEST(*alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == NULL);
+    
+    TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero);
+    TEST(alltypes.rep_emptymsg_count == 5);
+
+    if (mode == 0)
+    {
+        /* Expect that optional values are not present */
+        TEST(alltypes.opt_int32         == NULL);
+        TEST(alltypes.opt_int64         == NULL);
+        TEST(alltypes.opt_uint32        == NULL);
+        TEST(alltypes.opt_uint64        == NULL);
+        TEST(alltypes.opt_sint32        == NULL);
+        TEST(alltypes.opt_sint64        == NULL);
+        TEST(alltypes.opt_bool          == NULL);
+        
+        TEST(alltypes.opt_fixed32       == NULL);
+        TEST(alltypes.opt_sfixed32      == NULL);
+        TEST(alltypes.opt_float         == NULL);
+        TEST(alltypes.opt_fixed64       == NULL);
+        TEST(alltypes.opt_sfixed64      == NULL);
+        TEST(alltypes.opt_double        == NULL);
+        
+        TEST(alltypes.opt_string        == NULL);
+        TEST(alltypes.opt_bytes         == NULL);
+        TEST(alltypes.opt_submsg        == NULL);
+        TEST(alltypes.opt_enum          == NULL);
+    }
+    else
+    {
+        /* Expect filled-in values */
+        TEST(alltypes.opt_int32 && *alltypes.opt_int32      == 3041);
+        TEST(alltypes.opt_int64 && *alltypes.opt_int64      == 3042);
+        TEST(alltypes.opt_uint32 && *alltypes.opt_uint32    == 3043);
+        TEST(alltypes.opt_uint64 && *alltypes.opt_uint64    == 3044);
+        TEST(alltypes.opt_sint32 && *alltypes.opt_sint32    == 3045);
+        TEST(alltypes.opt_sint64 && *alltypes.opt_sint64    == 3046);
+        TEST(alltypes.opt_bool && *alltypes.opt_bool        == true);
+        
+        TEST(alltypes.opt_fixed32 && *alltypes.opt_fixed32  == 3048);
+        TEST(alltypes.opt_sfixed32 && *alltypes.opt_sfixed32== 3049);
+        TEST(alltypes.opt_float && *alltypes.opt_float      == 3050.0f);
+        TEST(alltypes.opt_fixed64 && *alltypes.opt_fixed64  == 3051);
+        TEST(alltypes.opt_sfixed64 && *alltypes.opt_sfixed64== 3052);
+        TEST(alltypes.opt_double && *alltypes.opt_double    == 3053.0);
+        
+        TEST(alltypes.opt_string && strcmp(alltypes.opt_string, "3054") == 0);
+        TEST(alltypes.opt_bytes && alltypes.opt_bytes->size == 4);
+        TEST(alltypes.opt_bytes && memcmp(&alltypes.opt_bytes->bytes, "3055", 4) == 0);
+        TEST(alltypes.opt_submsg && strcmp(alltypes.opt_submsg->substuff1, "3056") == 0);
+        TEST(alltypes.opt_submsg && *alltypes.opt_submsg->substuff2 == 3056);
+        TEST(alltypes.opt_enum && *alltypes.opt_enum == MyEnum_Truth);
+        TEST(alltypes.opt_emptymsg);
+    }
+    
+    TEST(alltypes.req_limits->int32_min && *alltypes.req_limits->int32_min   == INT32_MIN);
+    TEST(alltypes.req_limits->int32_max && *alltypes.req_limits->int32_max   == INT32_MAX);
+    TEST(alltypes.req_limits->uint32_min && *alltypes.req_limits->uint32_min == 0);
+    TEST(alltypes.req_limits->uint32_max && *alltypes.req_limits->uint32_max == UINT32_MAX);
+    TEST(alltypes.req_limits->int64_min && *alltypes.req_limits->int64_min   == INT64_MIN);
+    TEST(alltypes.req_limits->int64_max && *alltypes.req_limits->int64_max   == INT64_MAX);
+    TEST(alltypes.req_limits->uint64_min && *alltypes.req_limits->uint64_min == 0);
+    TEST(alltypes.req_limits->uint64_max && *alltypes.req_limits->uint64_max == UINT64_MAX);
+    TEST(alltypes.req_limits->enum_min && *alltypes.req_limits->enum_min     == HugeEnum_Negative);
+    TEST(alltypes.req_limits->enum_max && *alltypes.req_limits->enum_max     == HugeEnum_Positive);
+    
+    TEST(alltypes.end && *alltypes.end == 1099);
+
+    pb_release(AllTypes_fields, &alltypes);
+
+    return status;
+}
+
+int main(int argc, char **argv)
+{
+    uint8_t buffer[1024];
+    size_t count;
+    pb_istream_t stream;
+    
+    /* Whether to expect the optional values or the default values. */
+    int mode = (argc > 1) ? atoi(argv[1]) : 0;
+    
+    /* Read the data into buffer */
+    SET_BINARY_MODE(stdin);
+    count = fread(buffer, 1, sizeof(buffer), stdin);
+    
+    /* Construct a pb_istream_t for reading from the buffer */
+    stream = pb_istream_from_buffer(buffer, count);
+    
+    /* Decode and verify the message */
+    if (!check_alltypes(&stream, mode))
+    {
+        fprintf(stderr, "Test failed: %s\n", PB_GET_ERROR(&stream));
+        return 1;
+    }
+    else
+    {
+        return 0;
+    }
+}

tests/alltypes_pointer/encode_alltypes_pointer.c

@@ -0,0 +1,188 @@
+/* Attempts to test all the datatypes supported by ProtoBuf.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pb_encode.h>
+#include "alltypes.pb.h"
+#include "test_helpers.h"
+
+int main(int argc, char **argv)
+{
+    int mode = (argc > 1) ? atoi(argv[1]) : 0;
+    
+    /* Values for required fields */
+    int32_t     req_int32         = -1001;
+    int64_t     req_int64         = -1002;
+    uint32_t    req_uint32        = 1003;
+    uint64_t    req_uint64        = 1004;
+    int32_t     req_sint32        = -1005;
+    int64_t     req_sint64        = -1006;
+    bool        req_bool          = true;
+    uint32_t    req_fixed32       = 1008;
+    int32_t     req_sfixed32      = -1009;
+    float       req_float         = 1010.0f;
+    uint64_t    req_fixed64       = 1011;
+    int64_t     req_sfixed64      = -1012;
+    double      req_double        = 1013.0;
+    char*       req_string        = "1014";
+    PB_BYTES_ARRAY_T(4) req_bytes = {4, {'1', '0', '1', '5'}};
+    static int32_t req_substuff   = 1016;
+    SubMessage  req_submsg        = {"1016", &req_substuff};
+    MyEnum      req_enum          = MyEnum_Truth;
+    EmptyMessage req_emptymsg     = {0};
+    
+    int32_t     end               = 1099;
+
+    /* Values for repeated fields */
+    int32_t     rep_int32[5]      = {0, 0, 0, 0, -2001};
+    int64_t     rep_int64[5]      = {0, 0, 0, 0, -2002};
+    uint32_t    rep_uint32[5]     = {0, 0, 0, 0, 2003};
+    uint64_t    rep_uint64[5]     = {0, 0, 0, 0, 2004};
+    int32_t     rep_sint32[5]     = {0, 0, 0, 0, -2005};
+    int64_t     rep_sint64[5]     = {0, 0, 0, 0, -2006};
+    bool        rep_bool[5]       = {false, false, false, false, true};
+    uint32_t    rep_fixed32[5]    = {0, 0, 0, 0, 2008};
+    int32_t     rep_sfixed32[5]   = {0, 0, 0, 0, -2009};
+    float       rep_float[5]      = {0, 0, 0, 0, 2010.0f};
+    uint64_t    rep_fixed64[5]    = {0, 0, 0, 0, 2011};
+    int64_t     rep_sfixed64[5]   = {0, 0, 0, 0, -2012};
+    double      rep_double[5]     = {0, 0, 0, 0, 2013.0f};
+    char*       rep_string[5]     = {"", "", "", "", "2014"};
+    static PB_BYTES_ARRAY_T(4) rep_bytes_4 = {4, {'2', '0', '1', '5'}};
+    pb_bytes_array_t *rep_bytes[5]= {NULL, NULL, NULL, NULL, (pb_bytes_array_t*)&rep_bytes_4};
+    static int32_t rep_sub2zero   = 0;
+    static int32_t rep_substuff2  = 2016;
+    static uint32_t rep_substuff3 = 2016;
+    SubMessage  rep_submsg[5]     = {{"", &rep_sub2zero},
+                                     {"", &rep_sub2zero},
+                                     {"", &rep_sub2zero},
+                                     {"", &rep_sub2zero},
+                                     {"2016", &rep_substuff2, &rep_substuff3}};
+    MyEnum      rep_enum[5]       = {0, 0, 0, 0, MyEnum_Truth};
+    EmptyMessage rep_emptymsg[5]  = {{0}, {0}, {0}, {0}, {0}};
+
+    /* Values for optional fields */
+    int32_t     opt_int32         = 3041;
+    int64_t     opt_int64         = 3042;
+    uint32_t    opt_uint32        = 3043;
+    uint64_t    opt_uint64        = 3044;
+    int32_t     opt_sint32        = 3045;
+    int64_t     opt_sint64        = 3046;
+    bool        opt_bool          = true;
+    uint32_t    opt_fixed32       = 3048;
+    int32_t     opt_sfixed32      = 3049;
+    float       opt_float         = 3050.0f;
+    uint64_t    opt_fixed64       = 3051;
+    int64_t     opt_sfixed64      = 3052;
+    double      opt_double        = 3053.0;
+    char*       opt_string        = "3054";
+    PB_BYTES_ARRAY_T(4) opt_bytes = {4, {'3', '0', '5', '5'}};
+    static int32_t opt_substuff   = 3056;
+    SubMessage  opt_submsg        = {"3056", &opt_substuff};
+    MyEnum      opt_enum          = MyEnum_Truth;
+    EmptyMessage opt_emptymsg     = {0};
+
+    /* Values for the Limits message. */
+    static int32_t  int32_min  = INT32_MIN;
+    static int32_t  int32_max  = INT32_MAX;
+    static uint32_t uint32_min = 0;
+    static uint32_t uint32_max = UINT32_MAX;
+    static int64_t  int64_min  = INT64_MIN;
+    static int64_t  int64_max  = INT64_MAX;
+    static uint64_t uint64_min = 0;
+    static uint64_t uint64_max = UINT64_MAX;
+    static HugeEnum enum_min   = HugeEnum_Negative;
+    static HugeEnum enum_max   = HugeEnum_Positive;
+    Limits req_limits = {&int32_min,    &int32_max,
+                         &uint32_min,   &uint32_max,
+                         &int64_min,    &int64_max,
+                         &uint64_min,   &uint64_max,
+                         &enum_min,     &enum_max};
+
+    /* Initialize the message struct with pointers to the fields. */
+    AllTypes alltypes = {0};
+
+    alltypes.req_int32         = &req_int32;
+    alltypes.req_int64         = &req_int64;
+    alltypes.req_uint32        = &req_uint32;
+    alltypes.req_uint64        = &req_uint64;
+    alltypes.req_sint32        = &req_sint32;
+    alltypes.req_sint64        = &req_sint64;
+    alltypes.req_bool          = &req_bool;
+    alltypes.req_fixed32       = &req_fixed32;
+    alltypes.req_sfixed32      = &req_sfixed32;
+    alltypes.req_float         = &req_float;
+    alltypes.req_fixed64       = &req_fixed64;
+    alltypes.req_sfixed64      = &req_sfixed64;
+    alltypes.req_double        = &req_double;
+    alltypes.req_string        = req_string;
+    alltypes.req_bytes         = (pb_bytes_array_t*)&req_bytes;
+    alltypes.req_submsg        = &req_submsg;
+    alltypes.req_enum          = &req_enum;
+    alltypes.req_emptymsg      = &req_emptymsg;
+    alltypes.req_limits        = &req_limits;
+    
+    alltypes.rep_int32_count    = 5; alltypes.rep_int32     = rep_int32;
+    alltypes.rep_int64_count    = 5; alltypes.rep_int64     = rep_int64;
+    alltypes.rep_uint32_count   = 5; alltypes.rep_uint32    = rep_uint32;
+    alltypes.rep_uint64_count   = 5; alltypes.rep_uint64    = rep_uint64;
+    alltypes.rep_sint32_count   = 5; alltypes.rep_sint32    = rep_sint32;
+    alltypes.rep_sint64_count   = 5; alltypes.rep_sint64    = rep_sint64;
+    alltypes.rep_bool_count     = 5; alltypes.rep_bool      = rep_bool;
+    alltypes.rep_fixed32_count  = 5; alltypes.rep_fixed32   = rep_fixed32;
+    alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32  = rep_sfixed32;
+    alltypes.rep_float_count    = 5; alltypes.rep_float     = rep_float;
+    alltypes.rep_fixed64_count  = 5; alltypes.rep_fixed64   = rep_fixed64;
+    alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64  = rep_sfixed64;
+    alltypes.rep_double_count   = 5; alltypes.rep_double    = rep_double;
+    alltypes.rep_string_count   = 5; alltypes.rep_string    = rep_string;
+    alltypes.rep_bytes_count    = 5; alltypes.rep_bytes     = rep_bytes;
+    alltypes.rep_submsg_count   = 5; alltypes.rep_submsg    = rep_submsg;
+    alltypes.rep_enum_count     = 5; alltypes.rep_enum      = rep_enum;
+    alltypes.rep_emptymsg_count = 5; alltypes.rep_emptymsg  = rep_emptymsg;
+    
+    if (mode != 0)
+    {
+        /* Fill in values for optional fields */
+        alltypes.opt_int32         = &opt_int32;
+        alltypes.opt_int64         = &opt_int64;
+        alltypes.opt_uint32        = &opt_uint32;
+        alltypes.opt_uint64        = &opt_uint64;
+        alltypes.opt_sint32        = &opt_sint32;
+        alltypes.opt_sint64        = &opt_sint64;
+        alltypes.opt_bool          = &opt_bool;
+        alltypes.opt_fixed32       = &opt_fixed32;
+        alltypes.opt_sfixed32      = &opt_sfixed32;
+        alltypes.opt_float         = &opt_float;
+        alltypes.opt_fixed64       = &opt_fixed64;
+        alltypes.opt_sfixed64      = &opt_sfixed64;
+        alltypes.opt_double        = &opt_double;
+        alltypes.opt_string        = opt_string;
+        alltypes.opt_bytes         = (pb_bytes_array_t*)&opt_bytes;
+        alltypes.opt_submsg        = &opt_submsg;
+        alltypes.opt_enum          = &opt_enum;
+        alltypes.opt_emptymsg      = &opt_emptymsg;
+    }
+    
+    alltypes.end = &end;
+    
+    {
+        uint8_t buffer[4096];
+        pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        
+        /* Now encode it and check if we succeeded. */
+        if (pb_encode(&stream, AllTypes_fields, &alltypes))
+        {
+            SET_BINARY_MODE(stdout);
+            fwrite(buffer, 1, stream.bytes_written, stdout);
+            return 0; /* Success */
+        }
+        else
+        {
+            fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
+            return 1; /* Failure */
+        }
+    }
+}

tests/backwards_compatibility/SConscript

@@ -3,8 +3,8 @@
 
 Import("env")
 
-enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "#common/pb_encode.o"])
-dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "#common/pb_decode.o"])
+enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_encode.o"])
+dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_decode.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_legacy.output"])

tests/backwards_compatibility/decode_legacy.c

@@ -11,6 +11,7 @@
 #include <stdlib.h>
 #include <pb_decode.h>
 #include "alltypes_legacy.h"
+#include "test_helpers.h"
 
 #define TEST(x) if (!(x)) { \
     printf("Test " #x " failed.\n"); \
@@ -176,15 +177,19 @@ bool check_alltypes(pb_istream_t *stream, int mode)
 
 int main(int argc, char **argv)
 {
+    uint8_t buffer[1024];
+    size_t count;
+    pb_istream_t stream;
+
     /* Whether to expect the optional values or the default values. */
     int mode = (argc > 1) ? atoi(argv[1]) : 0;
     
     /* Read the data into buffer */
-    uint8_t buffer[1024];
-    size_t count = fread(buffer, 1, sizeof(buffer), stdin);
+    SET_BINARY_MODE(stdin);
+    count = fread(buffer, 1, sizeof(buffer), stdin);
     
     /* Construct a pb_istream_t for reading from the buffer */
-    pb_istream_t stream = pb_istream_from_buffer(buffer, count);
+    stream = pb_istream_from_buffer(buffer, count);
     
     /* Decode and print out the stuff */
     if (!check_alltypes(&stream, mode))

tests/basic_buffer/SConscript

@@ -2,11 +2,11 @@
 
 Import("env")
 
-enc = env.Program(["encode_buffer.c", "#common/person.pb.c", "#common/pb_encode.o"])
-dec = env.Program(["decode_buffer.c", "#common/person.pb.c", "#common/pb_decode.o"])
+enc = env.Program(["encode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"])
+dec = env.Program(["decode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_buffer.output"])
-env.Decode(["encode_buffer.output", "#common/person.proto"], MESSAGE = "Person")
+env.Decode(["encode_buffer.output", "$COMMON/person.proto"], MESSAGE = "Person")
 env.Compare(["decode_buffer.output", "encode_buffer.decoded"])
 

tests/basic_buffer/decode_buffer.c

@@ -16,7 +16,7 @@
 bool print_person(pb_istream_t *stream)
 {
     int i;
-    Person person;
+    Person person = Person_init_zero;
     
     if (!pb_decode(stream, Person_fields, &person))
         return false;

tests/basic_stream/SConscript

@@ -2,11 +2,11 @@
 
 Import("env")
 
-enc = env.Program(["encode_stream.c", "#common/person.pb.c", "#common/pb_encode.o"])
-dec = env.Program(["decode_stream.c", "#common/person.pb.c", "#common/pb_decode.o"])
+enc = env.Program(["encode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"])
+dec = env.Program(["decode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_stream.output"])
-env.Decode(["encode_stream.output", "#common/person.proto"], MESSAGE = "Person")
+env.Decode(["encode_stream.output", "$COMMON/person.proto"], MESSAGE = "Person")
 env.Compare(["decode_stream.output", "encode_stream.decoded"])
 

tests/basic_stream/decode_stream.c

@@ -12,7 +12,7 @@
 bool print_person(pb_istream_t *stream)
 {
     int i;
-    Person person;
+    Person person = Person_init_zero;
     
     if (!pb_decode(stream, Person_fields, &person))
         return false;

tests/buffer_only/SConscript

@@ -4,20 +4,24 @@ Import("env")
 
 # Take copy of the files for custom build.
 c = Copy("$TARGET", "$SOURCE")
-env.Command("pb_encode.c", "#../pb_encode.c", c)
-env.Command("pb_decode.c", "#../pb_decode.c", c)
-env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c)
-env.Command("alltypes.pb.c", "#alltypes/alltypes.pb.c", c)
-env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
-env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c)
+env.Command("alltypes.pb.c", "$BUILD/alltypes/alltypes.pb.c", c)
+env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c)
 
 # Define the compilation options
 opts = env.Clone()
 opts.Append(CPPDEFINES = {'PB_BUFFER_ONLY': 1})
 
+# Build new version of core
+strict = opts.Clone()
+strict.Append(CFLAGS = strict['CORECFLAGS'])
+strict.Object("pb_decode_bufonly.o", "$NANOPB/pb_decode.c")
+strict.Object("pb_encode_bufonly.o", "$NANOPB/pb_encode.c")
+
 # Now build and run the test normally.
-enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"])
-dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"])
+enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_bufonly.o"])
+dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_bufonly.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_alltypes.output"])

tests/callbacks/SConscript

@@ -3,8 +3,8 @@
 Import("env")
 
 env.NanopbProto("callbacks")
-enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "#common/pb_encode.o"])
-dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "#common/pb_decode.o"])
+enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_encode.o"])
+dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_decode.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_callbacks.output"])

tests/common/SConscript

@@ -8,10 +8,39 @@ env.NanopbProto("unittestproto")
 # Protocol definitions for basic_buffer/stream tests
 env.NanopbProto("person")
 
+#--------------------------------------------
 # Binaries of the pb_decode.c and pb_encode.c
 # These are built using more strict warning flags.
 strict = env.Clone()
 strict.Append(CFLAGS = strict['CORECFLAGS'])
-strict.Object("pb_decode.o", "#../pb_decode.c")
-strict.Object("pb_encode.o", "#../pb_encode.c")
+strict.Object("pb_decode.o", "$NANOPB/pb_decode.c")
+strict.Object("pb_encode.o", "$NANOPB/pb_encode.c")
+
+#-----------------------------------------------
+# Binaries of pb_decode etc. with malloc support
+# Uses malloc_wrappers.c to count allocations.
+malloc_env = env.Clone()
+malloc_env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1,
+                                'PB_SYSTEM_HEADER': '\\"malloc_wrappers_syshdr.h\\"'})
+malloc_env.Append(CPPPATH = ["$COMMON"])
+
+if 'SYSHDR' in malloc_env:
+    malloc_env.Append(CPPDEFINES = {'PB_OLD_SYSHDR': malloc_env['SYSHDR']})
+
+# Disable libmudflap, because it will confuse valgrind
+# and other memory leak detection tools.
+if '-fmudflap' in env["CCFLAGS"]:
+    malloc_env["CCFLAGS"].remove("-fmudflap")
+    malloc_env["LINKFLAGS"].remove("-fmudflap")
+    malloc_env["LIBS"].remove("mudflap")
+
+malloc_strict = malloc_env.Clone()
+malloc_strict.Append(CFLAGS = malloc_strict['CORECFLAGS'])
+malloc_strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c")
+malloc_strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c")
+
+malloc_env.Object("malloc_wrappers.o", "malloc_wrappers.c")
+malloc_env.Depends("$NANOPB/pb.h", ["malloc_wrappers_syshdr.h", "malloc_wrappers.h"])
+
+Export("malloc_env")
 

tests/common/malloc_wrappers.c

@@ -0,0 +1,54 @@
+#include "malloc_wrappers.h"
+#include <stdint.h>
+#include <assert.h>
+#include <string.h>
+
+static size_t alloc_count = 0;
+
+/* Allocate memory and place check values before and after. */
+void* malloc_with_check(size_t size)
+{
+    size_t size32 = (size + 3) / 4 + 3;
+    uint32_t *buf = malloc(size32 * sizeof(uint32_t));
+    buf[0] = size32;
+    buf[1] = 0xDEADBEEF;
+    buf[size32 - 1] = 0xBADBAD;
+    return buf + 2;
+}
+
+/* Free memory allocated with malloc_with_check() and do the checks. */
+void free_with_check(void *mem)
+{
+    uint32_t *buf = (uint32_t*)mem - 2;
+    assert(buf[1] == 0xDEADBEEF);
+    assert(buf[buf[0] - 1] == 0xBADBAD);
+    free(buf);
+}
+
+/* Track memory usage */
+void* counting_realloc(void *ptr, size_t size)
+{
+    /* Don't allocate crazy amounts of RAM when fuzzing */
+    if (size > 1000000)
+        return NULL;
+
+    if (!ptr && size)
+        alloc_count++;
+    
+    return realloc(ptr, size);
+}
+
+void counting_free(void *ptr)
+{
+    if (ptr)
+    {
+        assert(alloc_count > 0);
+        alloc_count--;
+        free(ptr);
+    }
+}
+
+size_t get_alloc_count()
+{
+    return alloc_count;
+}

tests/common/malloc_wrappers.h

@@ -0,0 +1,7 @@
+#include <stdlib.h>
+
+void* malloc_with_check(size_t size);
+void free_with_check(void *mem);
+void* counting_realloc(void *ptr, size_t size);
+void counting_free(void *ptr);
+size_t get_alloc_count();

tests/common/malloc_wrappers_syshdr.h

@@ -0,0 +1,15 @@
+/* This is just a wrapper in order to get our own malloc wrappers into nanopb core. */
+
+#define pb_realloc(ptr,size) counting_realloc(ptr,size)
+#define pb_free(ptr) counting_free(ptr)
+
+#ifdef PB_OLD_SYSHDR
+#include PB_OLD_SYSHDR
+#else
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+#include <string.h>
+#endif
+
+#include <malloc_wrappers.h>

tests/cxx_main_program/SConscript

@@ -3,14 +3,18 @@
 
 Import("env")
 
+# This is needed to get INT32_MIN etc. macros defined
+env = env.Clone()
+env.Append(CPPDEFINES = ['__STDC_LIMIT_MACROS'])
+
 # Copy the files to .cxx extension in order to force C++ build.
 c = Copy("$TARGET", "$SOURCE")
 env.Command("pb_encode.cxx", "#../pb_encode.c", c)
 env.Command("pb_decode.cxx", "#../pb_decode.c", c)
-env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c)
-env.Command("alltypes.pb.cxx", "#alltypes/alltypes.pb.c", c)
-env.Command("encode_alltypes.cxx", "#alltypes/encode_alltypes.c", c)
-env.Command("decode_alltypes.cxx", "#alltypes/decode_alltypes.c", c)
+env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c)
+env.Command("alltypes.pb.cxx", "$BUILD/alltypes/alltypes.pb.c", c)
+env.Command("encode_alltypes.cxx", "$BUILD/alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.cxx", "$BUILD/alltypes/decode_alltypes.c", c)
 
 # Now build and run the test normally.
 enc = env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx"])

tests/decode_unittests/SConscript

@@ -1,4 +1,4 @@
 Import('env')
-p = env.Program(["decode_unittests.c", "#common/unittestproto.pb.c"])
+p = env.Program(["decode_unittests.c", "$COMMON/unittestproto.pb.c"])
 env.RunTest(p)
 

tests/decode_unittests/decode_unittests.c

@@ -1,4 +1,5 @@
 /* This includes the whole .c file to get access to static functions. */
+#define PB_ENABLE_MALLOC
 #include "pb_decode.c"
 
 #include <stdio.h>
@@ -299,6 +300,28 @@ int main()
               dest.submsg.data_count == 5)
     }
     
+    {
+        pb_istream_t s = {0};
+        void *data = NULL;
+        
+        COMMENT("Testing allocate_field")
+        TEST(allocate_field(&s, &data, 10, 10) && data != NULL);
+        TEST(allocate_field(&s, &data, 10, 20) && data != NULL);
+        
+        {
+            void *oldvalue = data;
+            size_t very_big = (size_t)-1;
+            size_t somewhat_big = very_big / 2 + 1;
+            size_t not_so_big = (size_t)1 << (4 * sizeof(size_t));
+        
+            TEST(!allocate_field(&s, &data, very_big, 2) && data == oldvalue);
+            TEST(!allocate_field(&s, &data, somewhat_big, 2) && data == oldvalue);
+            TEST(!allocate_field(&s, &data, not_so_big, not_so_big) && data == oldvalue);
+        }
+        
+        pb_free(data);
+    }
+    
     if (status != 0)
         fprintf(stdout, "\n\nSome tests FAILED!\n");
     

tests/encode_unittests/SConscript

@@ -1,5 +1,5 @@
 # Build and run the stand-alone unit tests for the nanopb encoder part.
 
 Import('env')
-p = env.Program(["encode_unittests.c", "#common/unittestproto.pb.c"])
+p = env.Program(["encode_unittests.c", "$COMMON/unittestproto.pb.c"])
 env.RunTest(p)

tests/encode_unittests/encode_unittests.c

@@ -282,6 +282,15 @@ int main()
                     "\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05"))
     }
 
+    {
+        IntegerContainer msg = {{5, {1,2,3,4,5}}};
+        size_t size;
+        
+        COMMENT("Test pb_get_encoded_size.")
+        TEST(pb_get_encoded_size(&size, IntegerContainer_fields, &msg) &&
+             size == 9);
+    }
+    
     {
         uint8_t buffer[10];
         pb_ostream_t s;

tests/extensions/SConscript

@@ -4,12 +4,12 @@ Import("env")
 
 # We use the files from the alltypes test case
 incpath = env.Clone()
-incpath.Append(PROTOCPATH = '#alltypes')
-incpath.Append(CPPPATH = '#alltypes')
+incpath.Append(PROTOCPATH = '$BUILD/alltypes')
+incpath.Append(CPPPATH = '$BUILD/alltypes')
 
-incpath.NanopbProto("extensions")
-enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb$OBJSUFFIX", "#common/pb_encode.o"])
-dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb$OBJSUFFIX", "#common/pb_decode.o"])
+incpath.NanopbProto(["extensions", "extensions.options"])
+enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_encode.o"])
+dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_decode.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_extensions.output"])

tests/extensions/decode_extensions.c

@@ -49,7 +49,9 @@ int main(int argc, char **argv)
     }
 
     /* Check that the extensions decoded properly */
+    TEST(ext1.found)
     TEST(extensionfield1 == 12345)
+    TEST(ext2.found)
     TEST(strcmp(extensionfield2.test1, "test") == 0)
     TEST(extensionfield2.test2 == 54321)
     

tests/extensions/extensions.proto

@@ -1,13 +1,12 @@
 import 'alltypes.proto';
 
 extend AllTypes {
-    optional int32 AllTypes_extensionfield1 = 255;
+    optional int32 AllTypes_extensionfield1 = 255 [default = 5];
 }
 
 message ExtensionMessage {
     extend AllTypes {
         optional ExtensionMessage AllTypes_extensionfield2 = 254;
-	required ExtensionMessage AllTypes_extensionfield3 = 253;
 	repeated ExtensionMessage AllTypes_extensionfield4 = 252;
     }
     

tests/extra_fields/SConscript

@@ -2,13 +2,15 @@
 
 Import("env")
 
-dec = env.GetBuildPath('#basic_buffer/${PROGPREFIX}decode_buffer${PROGSUFFIX}')
+dec = env.GetBuildPath('$BUILD/basic_buffer/${PROGPREFIX}decode_buffer${PROGSUFFIX}')
 env.RunTest('person_with_extra_field.output', [dec, "person_with_extra_field.pb"])
 env.Compare(["person_with_extra_field.output", "person_with_extra_field.expected"])
 
-dec = env.GetBuildPath('#basic_stream/${PROGPREFIX}decode_stream${PROGSUFFIX}')
+dec = env.GetBuildPath('$BUILD/basic_stream/${PROGPREFIX}decode_stream${PROGSUFFIX}')
 env.RunTest('person_with_extra_field_stream.output', [dec, "person_with_extra_field.pb"])
 env.Compare(["person_with_extra_field_stream.output", "person_with_extra_field.expected"])
 
-dec2 = env.GetBuildPath('#alltypes/${PROGPREFIX}decode_alltypes${PROGSUFFIX}')
+# This uses the backwards compatibility alltypes test, so that
+# alltypes_with_extra_fields.pb doesn't have to be remade so often.
+dec2 = env.GetBuildPath('$BUILD/backwards_compatibility/${PROGPREFIX}decode_legacy${PROGSUFFIX}')
 env.RunTest('alltypes_with_extra_fields.output', [dec2, 'alltypes_with_extra_fields.pb'])

tests/field_size_16/SConscript

@@ -5,20 +5,24 @@ Import("env")
 
 # Take copy of the files for custom build.
 c = Copy("$TARGET", "$SOURCE")
-env.Command("pb_encode.c", "#../pb_encode.c", c)
-env.Command("pb_decode.c", "#../pb_decode.c", c)
-env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
-env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c)
 
-env.NanopbProto("alltypes")
+env.NanopbProto(["alltypes", "alltypes.options"])
 
 # Define the compilation options
 opts = env.Clone()
 opts.Append(CPPDEFINES = {'PB_FIELD_16BIT': 1})
 
+# Build new version of core
+strict = opts.Clone()
+strict.Append(CFLAGS = strict['CORECFLAGS'])
+strict.Object("pb_decode_fields16.o", "$NANOPB/pb_decode.c")
+strict.Object("pb_encode_fields16.o", "$NANOPB/pb_encode.c")
+
 # Now build and run the test normally.
-enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"])
-dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"])
+enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields16.o"])
+dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields16.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_alltypes.output"])

tests/field_size_16/alltypes.proto

@@ -8,6 +8,24 @@ message EmptyMessage {
 
 }
 
+enum HugeEnum {
+    Negative = -2147483647; /* protoc doesn't accept -2147483648 here */
+    Positive =  2147483647;
+}
+
+message Limits {
+    required int32      int32_min  =  1;
+    required int32      int32_max  =  2;
+    required uint32     uint32_min =  3;
+    required uint32     uint32_max =  4;
+    required int64      int64_min  =  5;
+    required int64      int64_max  =  6;
+    required uint64     uint64_min =  7;
+    required uint64     uint64_max =  8;
+    required HugeEnum   enum_min   =  9;
+    required HugeEnum   enum_max   = 10;
+}
+
 enum MyEnum {
     Zero = 0;
     First = 1;
@@ -83,8 +101,13 @@ message AllTypes {
     optional MyEnum     opt_enum    = 10057 [default = Second];
     optional EmptyMessage opt_emptymsg = 10058;
 
+    // Check that extreme integer values are handled correctly
+    required Limits     req_limits = 98;
+
     // Just to make sure that the size of the fields has been calculated
     // properly, i.e. otherwise a bug in last field might not be detected.
     required int32      end = 10099;
+
+    extensions 200 to 255;
 }
 

tests/field_size_32/SConscript

@@ -5,20 +5,24 @@ Import("env")
 
 # Take copy of the files for custom build.
 c = Copy("$TARGET", "$SOURCE")
-env.Command("pb_encode.c", "#../pb_encode.c", c)
-env.Command("pb_decode.c", "#../pb_decode.c", c)
-env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
-env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c)
 
-env.NanopbProto("alltypes")
+env.NanopbProto(["alltypes", "alltypes.options"])
 
 # Define the compilation options
 opts = env.Clone()
 opts.Append(CPPDEFINES = {'PB_FIELD_32BIT': 1})
 
+# Build new version of core
+strict = opts.Clone()
+strict.Append(CFLAGS = strict['CORECFLAGS'])
+strict.Object("pb_decode_fields32.o", "$NANOPB/pb_decode.c")
+strict.Object("pb_encode_fields32.o", "$NANOPB/pb_encode.c")
+
 # Now build and run the test normally.
-enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"])
-dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"])
+enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields32.o"])
+dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields32.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_alltypes.output"])

tests/field_size_32/alltypes.proto

@@ -8,6 +8,24 @@ message EmptyMessage {
 
 }
 
+enum HugeEnum {
+    Negative = -2147483647; /* protoc doesn't accept -2147483648 here */
+    Positive =  2147483647;
+}
+
+message Limits {
+    required int32      int32_min  =  1;
+    required int32      int32_max  =  2;
+    required uint32     uint32_min =  3;
+    required uint32     uint32_max =  4;
+    required int64      int64_min  =  5;
+    required int64      int64_max  =  6;
+    required uint64     uint64_min =  7;
+    required uint64     uint64_max =  8;
+    required HugeEnum   enum_min   =  9;
+    required HugeEnum   enum_max   = 10;
+}
+
 enum MyEnum {
     Zero = 0;
     First = 1;
@@ -83,8 +101,13 @@ message AllTypes {
     optional MyEnum     opt_enum    = 10057 [default = Second];
     optional EmptyMessage opt_emptymsg = 10058;
 
+    // Check that extreme integer values are handled correctly
+    required Limits     req_limits = 98;
+
     // Just to make sure that the size of the fields has been calculated
     // properly, i.e. otherwise a bug in last field might not be detected.
     required int32      end = 13432099;
+
+    extensions 200 to 255;
 }
 

tests/fuzztest/SConscript

@@ -0,0 +1,34 @@
+# Run a fuzz test to verify robustness against corrupted/malicious data.
+
+Import("env", "malloc_env")
+
+# We want both pointer and static versions of the AllTypes message
+# Prefix them with package name.
+env.Command("alltypes_static.proto", "#alltypes/alltypes.proto",
+            lambda target, source, env:
+                open(str(target[0]), 'w').write("package alltypes_static;\n"
+                                                + open(str(source[0])).read()))
+env.Command("alltypes_pointer.proto", "#alltypes/alltypes.proto",
+            lambda target, source, env:
+                open(str(target[0]), 'w').write("package alltypes_pointer;\n"
+                                                + open(str(source[0])).read()))
+
+p1 = env.NanopbProto(["alltypes_pointer", "alltypes_pointer.options"])
+p2 = env.NanopbProto(["alltypes_static", "alltypes_static.options"])
+fuzz = malloc_env.Program(["fuzztest.c",
+                    "alltypes_pointer.pb.c",
+                    "alltypes_static.pb.c",
+                    "$COMMON/pb_encode_with_malloc.o",
+                    "$COMMON/pb_decode_with_malloc.o",
+                    "$COMMON/malloc_wrappers.o"])
+
+env.RunTest(fuzz)
+
+fuzzstub = malloc_env.Program(["fuzzstub.c",
+                    "alltypes_pointer.pb.c",
+                    "alltypes_static.pb.c",
+                    "$COMMON/pb_encode_with_malloc.o",
+                    "$COMMON/pb_decode_with_malloc.o",
+                    "$COMMON/malloc_wrappers.o"])
+
+

tests/fuzztest/alltypes_pointer.options

@@ -0,0 +1,3 @@
+# Generate all fields as pointers.
+* type:FT_POINTER
+

tests/fuzztest/alltypes_static.options

@@ -0,0 +1,3 @@
+* max_size:32
+* max_count:8
+*.extensions type:FT_IGNORE

tests/fuzztest/fuzzstub.c

@@ -0,0 +1,189 @@
+/* Fuzz testing for the nanopb core.
+ * This can be used with external fuzzers, e.g. radamsa.
+ * It performs most of the same checks as fuzztest, but does not feature data generation.
+ */
+
+#include <pb_decode.h>
+#include <pb_encode.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <time.h>
+#include <malloc_wrappers.h>
+#include "alltypes_static.pb.h"
+#include "alltypes_pointer.pb.h"
+
+#define BUFSIZE 4096
+
+static bool do_static_decode(uint8_t *buffer, size_t msglen, bool assert_success)
+{
+    pb_istream_t stream;
+    bool status;
+    
+    alltypes_static_AllTypes *msg = malloc_with_check(sizeof(alltypes_static_AllTypes));
+    stream = pb_istream_from_buffer(buffer, msglen);
+    status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg);
+    
+    if (!status && assert_success)
+    {
+        /* Anything that was successfully encoded, should be decodeable.
+         * One exception: strings without null terminator are encoded up
+         * to end of buffer, but refused on decode because the terminator
+         * would not fit. */
+        if (strcmp(stream.errmsg, "string overflow") != 0)
+            assert(status);
+    }
+    
+    free_with_check(msg);
+    return status;
+}
+
+static bool do_pointer_decode(uint8_t *buffer, size_t msglen, bool assert_success)
+{
+    pb_istream_t stream;
+    bool status;
+    alltypes_pointer_AllTypes *msg;
+    
+    msg = malloc_with_check(sizeof(alltypes_pointer_AllTypes));
+    memset(msg, 0, sizeof(alltypes_pointer_AllTypes));
+    stream = pb_istream_from_buffer(buffer, msglen);
+
+    assert(get_alloc_count() == 0);
+    status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg);
+    
+    if (assert_success)
+        assert(status);
+    
+    pb_release(alltypes_pointer_AllTypes_fields, msg);    
+    assert(get_alloc_count() == 0);
+    
+    free_with_check(msg);
+
+    return status;
+}
+
+/* Do a decode -> encode -> decode -> encode roundtrip */
+static void do_static_roundtrip(uint8_t *buffer, size_t msglen)
+{
+    bool status;
+    uint8_t *buf2 = malloc_with_check(BUFSIZE);
+    uint8_t *buf3 = malloc_with_check(BUFSIZE);
+    size_t msglen2, msglen3;
+    alltypes_static_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_static_AllTypes));
+    alltypes_static_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_static_AllTypes));
+    memset(msg1, 0, sizeof(alltypes_static_AllTypes));
+    memset(msg2, 0, sizeof(alltypes_static_AllTypes));
+    
+    {
+        pb_istream_t stream = pb_istream_from_buffer(buffer, msglen);
+        status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg1);
+        assert(status);
+    }
+    
+    {
+        pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE);
+        status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg1);
+        assert(status);
+        msglen2 = stream.bytes_written;
+    }
+    
+    {
+        pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2);
+        status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg2);
+        assert(status);
+    }
+    
+    {
+        pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE);
+        status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg2);
+        assert(status);
+        msglen3 = stream.bytes_written;
+    }
+    
+    assert(msglen2 == msglen3);
+    assert(memcmp(buf2, buf3, msglen2) == 0);
+    
+    free_with_check(msg1);
+    free_with_check(msg2);
+    free_with_check(buf2);
+    free_with_check(buf3);
+}
+
+/* Do decode -> encode -> decode -> encode roundtrip */
+static void do_pointer_roundtrip(uint8_t *buffer, size_t msglen)
+{
+    bool status;
+    uint8_t *buf2 = malloc_with_check(BUFSIZE);
+    uint8_t *buf3 = malloc_with_check(BUFSIZE);
+    size_t msglen2, msglen3;
+    alltypes_pointer_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_pointer_AllTypes));
+    alltypes_pointer_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_pointer_AllTypes));
+    memset(msg1, 0, sizeof(alltypes_pointer_AllTypes));
+    memset(msg2, 0, sizeof(alltypes_pointer_AllTypes));
+    
+    {
+        pb_istream_t stream = pb_istream_from_buffer(buffer, msglen);
+        status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg1);
+        assert(status);
+    }
+    
+    {
+        pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE);
+        status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg1);
+        assert(status);
+        msglen2 = stream.bytes_written;
+    }
+    
+    {
+        pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2);
+        status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg2);
+        assert(status);
+    }
+    
+    {
+        pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE);
+        status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg2);
+        assert(status);
+        msglen3 = stream.bytes_written;
+    }
+    
+    assert(msglen2 == msglen3);
+    assert(memcmp(buf2, buf3, msglen2) == 0);
+    
+    pb_release(alltypes_pointer_AllTypes_fields, msg1);
+    pb_release(alltypes_pointer_AllTypes_fields, msg2);
+    free_with_check(msg1);
+    free_with_check(msg2);
+    free_with_check(buf2);
+    free_with_check(buf3);
+}
+
+static void run_iteration()
+{
+    uint8_t *buffer = malloc_with_check(BUFSIZE);
+    size_t msglen;
+    bool status;
+    
+    msglen = fread(buffer, BUFSIZE, 1, stdin);
+
+    status = do_static_decode(buffer, msglen, false);
+    
+    if (status)
+        do_static_roundtrip(buffer, msglen);
+    
+    status = do_pointer_decode(buffer, msglen, false);
+    
+    if (status)
+        do_pointer_roundtrip(buffer, msglen);
+    
+    free_with_check(buffer);
+}
+
+int main(int argc, char **argv)
+{
+    run_iteration();
+    
+    return 0;
+}
+

tests/fuzztest/fuzztest.c

@@ -0,0 +1,431 @@
+/* Fuzz testing for the nanopb core.
+ * Attempts to verify all the properties defined in the security model document.
+ */
+
+#include <pb_decode.h>
+#include <pb_encode.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <time.h>
+#include <malloc_wrappers.h>
+#include "alltypes_static.pb.h"
+#include "alltypes_pointer.pb.h"
+
+static uint64_t random_seed;
+
+/* Uses xorshift64 here instead of rand() for both speed and
+ * reproducibility across platforms. */
+static uint32_t rand_word()
+{
+    random_seed ^= random_seed >> 12;
+    random_seed ^= random_seed << 25;
+    random_seed ^= random_seed >> 27;
+    return random_seed * 2685821657736338717ULL;
+}
+
+/* Get a random integer in range, with approximately flat distribution. */
+static int rand_int(int min, int max)
+{
+    return rand_word() % (max + 1 - min) + min;
+}
+
+static bool rand_bool()
+{
+    return rand_word() & 1;
+}
+
+/* Get a random byte, with skewed distribution.
+ * Important corner cases like 0xFF, 0x00 and 0xFE occur more
+ * often than other values. */
+static uint8_t rand_byte()
+{
+    uint32_t w = rand_word();
+    uint8_t b = w & 0xFF;
+    if (w & 0x100000)
+        b >>= (w >> 8) & 7;
+    if (w & 0x200000)
+        b <<= (w >> 12) & 7;
+    if (w & 0x400000)
+        b ^= 0xFF;
+    return b;
+}
+
+/* Get a random length, with skewed distribution.
+ * Favors the shorter lengths, but always atleast 1. */
+static size_t rand_len(size_t max)
+{
+    uint32_t w = rand_word();
+    size_t s;
+    if (w & 0x800000)
+        w &= 3;
+    else if (w & 0x400000)
+        w &= 15;
+    else if (w & 0x200000)
+        w &= 255;
+
+    s = (w % max);
+    if (s == 0)
+        s = 1;
+    
+    return s;
+}
+
+/* Fills a buffer with random data with skewed distribution. */
+static void rand_fill(uint8_t *buf, size_t count)
+{
+    while (count--)
+        *buf++ = rand_byte();
+}
+
+/* Fill with random protobuf-like data */
+static size_t rand_fill_protobuf(uint8_t *buf, size_t min_bytes, size_t max_bytes, int min_tag)
+{
+    pb_ostream_t stream = pb_ostream_from_buffer(buf, max_bytes);
+
+    while(stream.bytes_written < min_bytes)
+    {
+        pb_wire_type_t wt = rand_int(0, 3);
+        if (wt == 3) wt = 5; /* Gap in values */
+        
+        if (!pb_encode_tag(&stream, wt, rand_int(min_tag, min_tag + 512)))
+            break;
+    
+        if (wt == PB_WT_VARINT)
+        {
+            uint64_t value;
+            rand_fill((uint8_t*)&value, sizeof(value));
+            pb_encode_varint(&stream, value);
+        }
+        else if (wt == PB_WT_64BIT)
+        {
+            uint64_t value;
+            rand_fill((uint8_t*)&value, sizeof(value));
+            pb_encode_fixed64(&stream, &value);
+        }
+        else if (wt == PB_WT_32BIT)
+        {
+            uint32_t value;
+            rand_fill((uint8_t*)&value, sizeof(value));
+            pb_encode_fixed32(&stream, &value);
+        }
+        else if (wt == PB_WT_STRING)
+        {
+            size_t len;
+            uint8_t *buf;
+            
+            if (min_bytes > stream.bytes_written)
+                len = rand_len(min_bytes - stream.bytes_written);
+            else
+                len = 0;
+            
+            buf = malloc(len);
+            pb_encode_varint(&stream, len);
+            rand_fill(buf, len);
+            pb_write(&stream, buf, len);
+            free(buf);
+        }
+    }
+    
+    return stream.bytes_written;
+}
+
+/* Given a buffer of data, mess it up a bit */
+static void rand_mess(uint8_t *buf, size_t count)
+{
+    int m = rand_int(0, 3);
+    
+    if (m == 0)
+    {
+        /* Replace random substring */
+        int s = rand_int(0, count - 1);
+        int l = rand_len(count - s);
+        rand_fill(buf + s, l);
+    }
+    else if (m == 1)
+    {
+        /* Swap random bytes */
+        int a = rand_int(0, count - 1);
+        int b = rand_int(0, count - 1);
+        int x = buf[a];
+        buf[a] = buf[b];
+        buf[b] = x;
+    }
+    else if (m == 2)
+    {
+        /* Duplicate substring */
+        int s = rand_int(0, count - 2);
+        int l = rand_len((count - s) / 2);
+        memcpy(buf + s + l, buf + s, l);
+    }
+    else if (m == 3)
+    {
+        /* Add random protobuf noise */
+        int s = rand_int(0, count - 1);
+        int l = rand_len(count - s);
+        rand_fill_protobuf(buf + s, l, count - s, 1);
+    }
+}
+
+/* Some default data to put in the message */
+static const alltypes_static_AllTypes initval = alltypes_static_AllTypes_init_default;
+
+#define BUFSIZE 4096
+
+static bool do_static_encode(uint8_t *buffer, size_t *msglen)
+{
+    pb_ostream_t stream;
+    bool status;
+
+    /* Allocate a message and fill it with defaults */
+    alltypes_static_AllTypes *msg = malloc_with_check(sizeof(alltypes_static_AllTypes));
+    memcpy(msg, &initval, sizeof(initval));
+
+    /* Apply randomness to the data before encoding */
+    while (rand_int(0, 7))
+        rand_mess((uint8_t*)msg, sizeof(alltypes_static_AllTypes));
+
+    stream = pb_ostream_from_buffer(buffer, BUFSIZE);
+    status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg);
+    assert(stream.bytes_written <= BUFSIZE);
+    assert(stream.bytes_written <= alltypes_static_AllTypes_size);
+    
+    *msglen = stream.bytes_written;
+    free_with_check(msg);
+    
+    return status;
+}
+
+/* Append or prepend protobuf noise */
+static void do_protobuf_noise(uint8_t *buffer, size_t *msglen)
+{
+    int m = rand_int(0, 2);
+    size_t max_size = BUFSIZE - 32 - *msglen;
+    if (m == 1)
+    {
+        /* Prepend */
+        uint8_t *tmp = malloc_with_check(BUFSIZE);
+        size_t s = rand_fill_protobuf(tmp, rand_len(max_size), BUFSIZE - *msglen, 512);
+        memmove(buffer + s, buffer, *msglen);
+        memcpy(buffer, tmp, s);
+        free_with_check(tmp);
+        *msglen += s;
+    }
+    else if (m == 2)
+    {
+        /* Append */
+        size_t s = rand_fill_protobuf(buffer + *msglen, rand_len(max_size), BUFSIZE - *msglen, 512);
+        *msglen += s;
+    }
+}
+
+static bool do_static_decode(uint8_t *buffer, size_t msglen, bool assert_success)
+{
+    pb_istream_t stream;
+    bool status;
+    
+    alltypes_static_AllTypes *msg = malloc_with_check(sizeof(alltypes_static_AllTypes));
+    rand_fill((uint8_t*)msg, sizeof(alltypes_static_AllTypes));
+    stream = pb_istream_from_buffer(buffer, msglen);
+    status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg);
+    
+    if (!status && assert_success)
+    {
+        /* Anything that was successfully encoded, should be decodeable.
+         * One exception: strings without null terminator are encoded up
+         * to end of buffer, but refused on decode because the terminator
+         * would not fit. */
+        if (strcmp(stream.errmsg, "string overflow") != 0)
+            assert(status);
+    }
+    
+    free_with_check(msg);
+    return status;
+}
+
+static bool do_pointer_decode(uint8_t *buffer, size_t msglen, bool assert_success)
+{
+    pb_istream_t stream;
+    bool status;
+    alltypes_pointer_AllTypes *msg;
+    
+    msg = malloc_with_check(sizeof(alltypes_pointer_AllTypes));
+    memset(msg, 0, sizeof(alltypes_pointer_AllTypes));
+    stream = pb_istream_from_buffer(buffer, msglen);
+
+    assert(get_alloc_count() == 0);
+    status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg);
+    
+    if (assert_success)
+        assert(status);
+    
+    pb_release(alltypes_pointer_AllTypes_fields, msg);    
+    assert(get_alloc_count() == 0);
+    
+    free_with_check(msg);
+
+    return status;
+}
+
+/* Do a decode -> encode -> decode -> encode roundtrip */
+static void do_static_roundtrip(uint8_t *buffer, size_t msglen)
+{
+    bool status;
+    uint8_t *buf2 = malloc_with_check(BUFSIZE);
+    uint8_t *buf3 = malloc_with_check(BUFSIZE);
+    size_t msglen2, msglen3;
+    alltypes_static_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_static_AllTypes));
+    alltypes_static_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_static_AllTypes));
+    memset(msg1, 0, sizeof(alltypes_static_AllTypes));
+    memset(msg2, 0, sizeof(alltypes_static_AllTypes));
+    
+    {
+        pb_istream_t stream = pb_istream_from_buffer(buffer, msglen);
+        status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg1);
+        assert(status);
+    }
+    
+    {
+        pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE);
+        status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg1);
+        assert(status);
+        msglen2 = stream.bytes_written;
+    }
+    
+    {
+        pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2);
+        status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg2);
+        assert(status);
+    }
+    
+    {
+        pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE);
+        status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg2);
+        assert(status);
+        msglen3 = stream.bytes_written;
+    }
+    
+    assert(msglen2 == msglen3);
+    assert(memcmp(buf2, buf3, msglen2) == 0);
+    
+    free_with_check(msg1);
+    free_with_check(msg2);
+    free_with_check(buf2);
+    free_with_check(buf3);
+}
+
+/* Do decode -> encode -> decode -> encode roundtrip */
+static void do_pointer_roundtrip(uint8_t *buffer, size_t msglen)
+{
+    bool status;
+    uint8_t *buf2 = malloc_with_check(BUFSIZE);
+    uint8_t *buf3 = malloc_with_check(BUFSIZE);
+    size_t msglen2, msglen3;
+    alltypes_pointer_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_pointer_AllTypes));
+    alltypes_pointer_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_pointer_AllTypes));
+    memset(msg1, 0, sizeof(alltypes_pointer_AllTypes));
+    memset(msg2, 0, sizeof(alltypes_pointer_AllTypes));
+    
+    {
+        pb_istream_t stream = pb_istream_from_buffer(buffer, msglen);
+        status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg1);
+        assert(status);
+    }
+    
+    {
+        pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE);
+        status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg1);
+        assert(status);
+        msglen2 = stream.bytes_written;
+    }
+    
+    {
+        pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2);
+        status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg2);
+        assert(status);
+    }
+    
+    {
+        pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE);
+        status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg2);
+        assert(status);
+        msglen3 = stream.bytes_written;
+    }
+    
+    assert(msglen2 == msglen3);
+    assert(memcmp(buf2, buf3, msglen2) == 0);
+    
+    pb_release(alltypes_pointer_AllTypes_fields, msg1);
+    pb_release(alltypes_pointer_AllTypes_fields, msg2);
+    free_with_check(msg1);
+    free_with_check(msg2);
+    free_with_check(buf2);
+    free_with_check(buf3);
+}
+
+static void run_iteration()
+{
+    uint8_t *buffer = malloc_with_check(BUFSIZE);
+    size_t msglen;
+    bool status;
+    
+    rand_fill(buffer, BUFSIZE);
+
+    if (do_static_encode(buffer, &msglen))
+    {
+        do_protobuf_noise(buffer, &msglen);
+    
+        status = do_static_decode(buffer, msglen, true);
+        
+        if (status)
+            do_static_roundtrip(buffer, msglen);
+        
+        status = do_pointer_decode(buffer, msglen, true);
+        
+        if (status)
+            do_pointer_roundtrip(buffer, msglen);
+        
+        /* Apply randomness to the encoded data */
+        while (rand_bool())
+            rand_mess(buffer, BUFSIZE);
+        
+        /* Apply randomness to encoded data length */
+        if (rand_bool())
+            msglen = rand_int(0, BUFSIZE);
+        
+        status = do_static_decode(buffer, msglen, false);
+        do_pointer_decode(buffer, msglen, status);
+        
+        if (status)
+        {
+            do_static_roundtrip(buffer, msglen);
+            do_pointer_roundtrip(buffer, msglen);
+        }
+    }
+    
+    free_with_check(buffer);
+}
+
+int main(int argc, char **argv)
+{
+    int i;
+    if (argc > 1)
+    {
+        random_seed = atol(argv[1]);
+    }
+    else
+    {
+        random_seed = time(NULL);
+    }
+    
+    fprintf(stderr, "Random seed: %llu\n", (long long unsigned)random_seed);
+    
+    for (i = 0; i < 10000; i++)
+    {
+        run_iteration();
+    }
+    
+    return 0;
+}
+

tests/fuzztest/run_radamsa.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+TMP=`tempfile`
+
+echo $TMP
+while true
+do
+   radamsa sample_data/* > $TMP
+   $1 < $TMP
+   test $? -gt 127 && break
+done
+ 

tests/mem_release/SConscript

@@ -0,0 +1,12 @@
+Import("env", "malloc_env")
+
+env.NanopbProto("mem_release.proto")
+
+test = malloc_env.Program(["mem_release.c",
+                    "mem_release.pb.c",
+                    "$COMMON/pb_encode_with_malloc.o",
+                    "$COMMON/pb_decode_with_malloc.o",
+                    "$COMMON/malloc_wrappers.o"])
+
+env.RunTest(test)
+

tests/mem_release/mem_release.c

@@ -0,0 +1,106 @@
+/* Make sure that all fields are freed in various scenarios. */
+
+#include <pb_decode.h>
+#include <pb_encode.h>
+#include <malloc_wrappers.h>
+#include <stdio.h>
+#include <test_helpers.h>
+#include "mem_release.pb.h"
+
+#define TEST(x) if (!(x)) { \
+    fprintf(stderr, "Test " #x " on line %d failed.\n", __LINE__); \
+    return false; \
+    }
+
+static char *test_str_arr[] = {"1", "2", ""};
+static SubMessage test_msg_arr[] = {SubMessage_init_zero, SubMessage_init_zero};
+
+static bool do_test()
+{
+    uint8_t buffer[256];
+    size_t msgsize;
+    
+    /* Construct a message with various fields filled in */
+    {
+        TestMessage msg = TestMessage_init_zero;
+        pb_extension_t ext2;
+        pb_ostream_t stream;
+        msg.static_req_submsg.dynamic_str = "12345";
+        msg.static_req_submsg.dynamic_str_arr_count = 3;
+        msg.static_req_submsg.dynamic_str_arr = test_str_arr;
+        msg.static_req_submsg.dynamic_submsg_count = 2;
+        msg.static_req_submsg.dynamic_submsg = test_msg_arr;
+        msg.static_req_submsg.dynamic_submsg[1].dynamic_str = "abc";
+        msg.static_opt_submsg.dynamic_str = "abc";
+        msg.has_static_opt_submsg = true;
+        msg.dynamic_submsg = &msg.static_req_submsg;
+        
+        msg.extensions = &ext2;
+        ext2.type = &static_ext;
+        ext2.dest = &msg.static_req_submsg;
+        ext2.next = NULL;
+        
+        stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        if (!pb_encode(&stream, TestMessage_fields, &msg))
+        {
+            fprintf(stderr, "Encode failed: %s\n", PB_GET_ERROR(&stream));
+            return false;
+        }
+        msgsize = stream.bytes_written;
+    }
+    
+    /* Output encoded message for debug */
+    SET_BINARY_MODE(stdout);
+    fwrite(buffer, 1, msgsize, stdout);
+    
+    /* Decode memory using dynamic allocation */
+    {
+        TestMessage msg = TestMessage_init_zero;
+        pb_istream_t stream;
+        SubMessage ext2_dest = SubMessage_init_zero;
+        pb_extension_t ext2;
+        
+        msg.extensions = &ext2;
+        ext2.type = &static_ext;
+        ext2.dest = &ext2_dest;
+        ext2.next = NULL;
+        
+        stream = pb_istream_from_buffer(buffer, msgsize);
+        if (!pb_decode(&stream, TestMessage_fields, &msg))
+        {
+            fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&stream));
+            return false;
+        }
+        
+        /* Make sure it encodes back to same data */
+        {
+            uint8_t buffer2[256];
+            pb_ostream_t ostream = pb_ostream_from_buffer(buffer2, sizeof(buffer2));
+            TEST(pb_encode(&ostream, TestMessage_fields, &msg));
+            TEST(ostream.bytes_written == msgsize);
+            TEST(memcmp(buffer, buffer2, msgsize) == 0);
+        }
+        
+        /* Make sure that malloc counters work */
+        TEST(get_alloc_count() > 0);
+        
+        /* Make sure that pb_release releases everything */
+        pb_release(TestMessage_fields, &msg);
+        TEST(get_alloc_count() == 0);
+        
+        /* Check that double-free is a no-op */
+        pb_release(TestMessage_fields, &msg);
+        TEST(get_alloc_count() == 0);
+    }
+    
+    return true;
+}
+
+int main()
+{
+    if (do_test())
+        return 0;
+    else
+        return 1;
+}
+

tests/mem_release/mem_release.proto

@@ -0,0 +1,23 @@
+syntax = "proto2";
+import "nanopb.proto";
+
+message SubMessage
+{
+    optional string dynamic_str = 1 [(nanopb).type = FT_POINTER];
+    repeated string dynamic_str_arr = 2 [(nanopb).type = FT_POINTER];
+    repeated SubMessage dynamic_submsg = 3 [(nanopb).type = FT_POINTER];
+}
+
+message TestMessage
+{
+    required SubMessage static_req_submsg = 1 [(nanopb).type = FT_STATIC];
+    optional SubMessage dynamic_submsg = 2 [(nanopb).type = FT_POINTER];
+    optional SubMessage static_opt_submsg = 3 [(nanopb).type = FT_STATIC];
+    extensions 100 to 200;
+}
+
+extend TestMessage
+{
+    optional SubMessage static_ext = 101 [(nanopb).type = FT_STATIC];
+}
+

tests/missing_fields/SConscript

@@ -3,6 +3,6 @@
 Import("env")
 
 env.NanopbProto("missing_fields")
-test = env.Program(["missing_fields.c", "missing_fields.pb.c", "#common/pb_encode.o", "#common/pb_decode.o"])
+test = env.Program(["missing_fields.c", "missing_fields.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_decode.o"])
 env.RunTest(test)
 

tests/missing_fields/missing_fields.c

@@ -8,6 +8,7 @@
 int main()
 {
     uint8_t buffer[512];
+    size_t size;
     
     /* Create a message with one missing field */
     {
@@ -19,12 +20,14 @@ int main()
             printf("Encode failed.\n");
             return 1;
         }
+
+        size = stream.bytes_written;
     }
 
     /* Test that it decodes properly if we don't require that field */
     {
         MissingField msg = {0};
-        pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer));
+        pb_istream_t stream = pb_istream_from_buffer(buffer, size);
         
         if (!pb_decode(&stream, MissingField_fields, &msg))
         {
@@ -36,7 +39,7 @@ int main()
     /* Test that it does *not* decode properly if we require the field */
     {
         AllFields msg = {0};
-        pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer));
+        pb_istream_t stream = pb_istream_from_buffer(buffer, size);
         
         if (pb_decode(&stream, AllFields_fields, &msg))
         {

tests/no_errmsg/SConscript

@@ -4,20 +4,24 @@ Import("env")
 
 # Take copy of the files for custom build.
 c = Copy("$TARGET", "$SOURCE")
-env.Command("pb_encode.c", "#../pb_encode.c", c)
-env.Command("pb_decode.c", "#../pb_decode.c", c)
-env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c)
-env.Command("alltypes.pb.c", "#alltypes/alltypes.pb.c", c)
-env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
-env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c)
+env.Command("alltypes.pb.c", "$BUILD/alltypes/alltypes.pb.c", c)
+env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c)
 
 # Define the compilation options
 opts = env.Clone()
 opts.Append(CPPDEFINES = {'PB_NO_ERRMSG': 1})
 
+# Build new version of core
+strict = opts.Clone()
+strict.Append(CFLAGS = strict['CORECFLAGS'])
+strict.Object("pb_decode_noerr.o", "$NANOPB/pb_decode.c")
+strict.Object("pb_encode_noerr.o", "$NANOPB/pb_encode.c")
+
 # Now build and run the test normally.
-enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"])
-dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"])
+enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_noerr.o"])
+dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_noerr.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_alltypes.output"])

tests/options/options.expected

@@ -5,3 +5,6 @@ pb_callback_t int32_callback;
 \sEnumValue1 = 1
 Message5_EnumValue1
 } pb_packed my_packed_struct;
+! skipped_field
+! SkippedMessage
+

tests/options/options.proto

@@ -63,11 +63,15 @@ message my_packed_struct
 }
 
 // Message with ignored field
-// Note: doesn't really test if the field is missing in the output,
-// but atleast tests that the output compiles.
 message Message6
 {
     required int32 field1 = 1;
-    optional int32 field2 = 2 [(nanopb).type = FT_IGNORE];
+    optional int32 skipped_field = 2 [(nanopb).type = FT_IGNORE];
 }
 
+// Message that is skipped
+message SkippedMessage
+{
+    option (nanopb_msgopt).skip_message = true;
+    required int32 foo = 1;
+}

tests/package_name/SConscript

@@ -0,0 +1,36 @@
+# Check that alltypes test case works also when the .proto file defines
+# a package name.
+
+Import("env")
+
+# Build a modified alltypes.proto
+def modify_proto(target, source, env):
+    '''Add a "package test.package;" directive to the beginning of the .proto file.'''
+    data = open(str(source[0]), 'r').read()
+    open(str(target[0]), 'w').write("package test.package;\n\n" + data)
+    return 0
+
+env.Command("alltypes.proto", "#alltypes/alltypes.proto", modify_proto)
+env.Command("alltypes.options", "#alltypes/alltypes.options", Copy("$TARGET", "$SOURCE"))
+env.NanopbProto(["alltypes", "alltypes.options"])
+
+# Build a modified encode_alltypes.c
+def modify_c(target, source, env):
+    '''Add package name to type names in .c file.'''
+
+    data = open(str(source[0]), 'r').read()
+    
+    type_names = ['AllTypes', 'MyEnum', 'HugeEnum']
+    for name in type_names:
+        data = data.replace(name, 'test_package_' + name)
+    
+    open(str(target[0]), 'w').write(data)
+    return 0
+env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", modify_c)
+
+# Encode and compare results to original alltypes testcase
+enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o"])
+refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX"
+env.RunTest(enc)
+env.Compare(["encode_alltypes.output", "$BUILD/alltypes/encode_alltypes.output"])
+

tests/regression/issue_125/SConscript

@@ -0,0 +1,9 @@
+# Regression test for Issue 125: Wrong identifier name for extension fields
+
+Import("env")
+
+env.NanopbProto(["extensionbug", "extensionbug.options"])
+env.Object('extensionbug.pb.c')
+
+env.Match(['extensionbug.pb.h', 'extensionbug.expected'])
+

tests/regression/issue_125/extensionbug.expected

@@ -0,0 +1,3 @@
+pb_extension_type_t Message2_extras
+uint32_t field2
+

tests/regression/issue_125/extensionbug.options

@@ -0,0 +1,4 @@
+* type:FT_IGNORE
+
+Message2.extras type:FT_STATIC
+Message2.field2 type:FT_STATIC

tests/regression/issue_125/extensionbug.proto

@@ -0,0 +1,16 @@
+message Message1
+{
+	optional uint32	fieldA = 1;
+	extensions 30 to max;
+}
+
+message Message2
+{
+	extend Message1
+	{
+		optional Message2 extras = 30;
+	}
+	
+	optional uint32	field1 = 1;
+	optional uint32 field2 = 2;
+}

tests/site_scons/site_init.py

@@ -12,42 +12,6 @@ except ImportError:
 def add_nanopb_builders(env):
     '''Add the necessary builder commands for nanopb tests.'''
 
-    # Build command for building .pb from .proto using protoc
-    def proto_actions(source, target, env, for_signature):
-        esc = env['ESCAPE']
-        dirs = ' '.join(['-I' + esc(env.GetBuildPath(d)) for d in env['PROTOCPATH']])
-        return '$PROTOC $PROTOCFLAGS %s -o%s %s' % (dirs, esc(str(target[0])), esc(str(source[0])))
-
-    proto_file_builder = Builder(generator = proto_actions,
-                                 suffix = '.pb',
-                                 src_suffix = '.proto')
-    env.Append(BUILDERS = {'Proto': proto_file_builder})
-    env.SetDefault(PROTOC = 'protoc')
-    env.SetDefault(PROTOCPATH = ['.'])
-
-    # Build command for running nanopb generator
-    import os.path
-    def nanopb_targets(target, source, env):
-        basename = os.path.splitext(str(source[0]))[0]
-        target.append(basename + '.pb.h')
-        return target, source
-
-    nanopb_file_builder = Builder(action = '$NANOPB_GENERATOR $NANOPB_FLAGS $SOURCE',
-                                  suffix = '.pb.c',
-                                  src_suffix = '.pb',
-                                  emitter = nanopb_targets)
-    env.Append(BUILDERS = {'Nanopb': nanopb_file_builder})
-    gen_path = env['ESCAPE'](env.GetBuildPath("#../generator/nanopb_generator.py"))
-    env.SetDefault(NANOPB_GENERATOR = 'python ' + gen_path)
-    env.SetDefault(NANOPB_FLAGS = '-q')
-
-    # Combined method to run both protoc and nanopb generator
-    def run_protoc_and_nanopb(env, source):
-        b1 = env.Proto(source)
-        b2 = env.Nanopb(source)
-        return b1 + b2
-    env.AddMethod(run_protoc_and_nanopb, "NanopbProto")
-
     # Build command that runs a test program and saves the output
     def run_test(target, source, env):
         if len(source) > 1:
@@ -55,15 +19,24 @@ def add_nanopb_builders(env):
         else:
             infile = None
         
-        pipe = subprocess.Popen(str(source[0]),
+        if env.has_key("COMMAND"):
+            args = [env["COMMAND"]]
+        else:
+            args = [str(source[0])]
+        
+        if env.has_key('ARGS'):
+            args.extend(env['ARGS'])
+        
+        print 'Command line: ' + str(args)
+        pipe = subprocess.Popen(args,
                                 stdin = infile,
                                 stdout = open(str(target[0]), 'w'),
                                 stderr = sys.stderr)
         result = pipe.wait()
         if result == 0:
-            print '\033[32m[ OK ]\033[0m   Ran ' + str(source[0])
+            print '\033[32m[ OK ]\033[0m   Ran ' + args[0]
         else:
-            print '\033[31m[FAIL]\033[0m   Program ' + str(source[0]) + ' returned ' + str(result)
+            print '\033[31m[FAIL]\033[0m   Program ' + args[0] + ' returned ' + str(result)
         return result
         
     run_test_builder = Builder(action = run_test,
@@ -81,6 +54,17 @@ def add_nanopb_builders(env):
                              suffix = '.decoded')
     env.Append(BUILDERS = {'Decode': decode_builder})    
 
+    # Build command that encodes a message using protoc
+    def encode_actions(source, target, env, for_signature):
+        esc = env['ESCAPE']
+        dirs = ' '.join(['-I' + esc(env.GetBuildPath(d)) for d in env['PROTOCPATH']])
+        return '$PROTOC $PROTOCFLAGS %s --encode=%s %s <%s >%s' % (
+            dirs, env['MESSAGE'], esc(str(source[1])), esc(str(source[0])), esc(str(target[0])))
+
+    encode_builder = Builder(generator = encode_actions,
+                             suffix = '.encoded')
+    env.Append(BUILDERS = {'Encode': encode_builder})    
+
     # Build command that asserts that two files be equal
     def compare_files(target, source, env):
         data1 = open(str(source[0]), 'rb').read()
@@ -101,9 +85,20 @@ def add_nanopb_builders(env):
         data = open(str(source[0]), 'rU').read()
         patterns = open(str(source[1]))
         for pattern in patterns:
-            if pattern.strip() and not re.search(pattern.strip(), data, re.MULTILINE):
+            if pattern.strip():
+                invert = False
+                if pattern.startswith('! '):
+                    invert = True
+                    pattern = pattern[2:]
+                
+                status = re.search(pattern.strip(), data, re.MULTILINE)
+                
+                if not status and not invert:
                     print '\033[31m[FAIL]\033[0m   Pattern not found in ' + str(source[0]) + ': ' + pattern
                     return 1
+                elif status and invert:
+                    print '\033[31m[FAIL]\033[0m   Pattern should not exist, but does in ' + str(source[0]) + ': ' + pattern
+                    return 1
         else:
             print '\033[32m[ OK ]\033[0m   All patterns found in ' + str(source[0])
             return 0

tests/site_scons/site_tools/nanopb.py

@@ -0,0 +1,126 @@
+'''
+Scons Builder for nanopb .proto definitions.
+
+This tool will locate the nanopb generator and use it to generate .pb.c and
+.pb.h files from the .proto files.
+
+Basic example
+-------------
+# Build myproto.pb.c and myproto.pb.h from myproto.proto
+myproto = env.NanopbProto("myproto")
+
+# Link nanopb core to the program
+env.Append(CPPPATH = "$NANOB")
+myprog = env.Program(["myprog.c", myproto, "$NANOPB/pb_encode.c", "$NANOPB/pb_decode.c"])
+
+Configuration options
+---------------------
+Normally, this script is used in the test environment of nanopb and it locates
+the nanopb generator by a relative path. If this script is used in another
+application, the path to nanopb root directory has to be defined:
+
+env.SetDefault(NANOPB = "path/to/nanopb")
+
+Additionally, the path to protoc and the options to give to protoc can be
+defined manually:
+
+env.SetDefault(PROTOC = "path/to/protoc")
+env.SetDefault(PROTOCFLAGS = "--plugin=protoc-gen-nanopb=path/to/protoc-gen-nanopb")
+'''
+
+import SCons.Action
+import SCons.Builder
+import SCons.Util
+import os.path
+
+class NanopbWarning(SCons.Warnings.Warning):
+    pass
+SCons.Warnings.enableWarningClass(NanopbWarning)
+
+def _detect_nanopb(env):
+    '''Find the path to nanopb root directory.'''
+    if env.has_key('NANOPB'):
+        # Use nanopb dir given by user
+        return env['NANOPB']
+    
+    p = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
+    if os.path.isdir(p) and os.path.isfile(os.path.join(p, 'pb.h')):
+        # Assume we are running under tests/site_scons/site_tools
+        return p
+    
+    raise SCons.Errors.StopError(NanopbWarning,
+        "Could not find the nanopb root directory")
+
+def _detect_protoc(env):
+    '''Find the path to the protoc compiler.'''
+    if env.has_key('PROTOC'):
+        # Use protoc defined by user
+        return env['PROTOC']
+    
+    n = _detect_nanopb(env)
+    p1 = os.path.join(n, 'generator-bin', 'protoc' + env['PROGSUFFIX'])
+    if os.path.exists(p1):
+        # Use protoc bundled with binary package
+        return env['ESCAPE'](p1)
+    
+    p = env.WhereIs('protoc')
+    if p:
+        # Use protoc from path
+        return env['ESCAPE'](p)
+    
+    raise SCons.Errors.StopError(NanopbWarning,
+        "Could not find the protoc compiler")
+
+def _detect_protocflags(env):
+    '''Find the options to use for protoc.'''
+    if env.has_key('PROTOCFLAGS'):
+        return env['PROTOCFLAGS']
+    
+    p = _detect_protoc(env)
+    n = _detect_nanopb(env)
+    p1 = os.path.join(n, 'generator-bin', 'protoc' + env['PROGSUFFIX'])
+    if p == env['ESCAPE'](p1):
+        # Using the bundled protoc, no options needed
+        return ''
+    
+    e = env['ESCAPE']
+    if env['PLATFORM'] == 'win32':
+        return e('--plugin=protoc-gen-nanopb=' + os.path.join(n, 'generator', 'protoc-gen-nanopb.bat'))
+    else:
+        return e('--plugin=protoc-gen-nanopb=' + os.path.join(n, 'generator', 'protoc-gen-nanopb'))
+
+def _nanopb_proto_actions(source, target, env, for_signature):
+    esc = env['ESCAPE']
+    dirs = ' '.join(['-I' + esc(env.GetBuildPath(d)) for d in env['PROTOCPATH']])
+    return '$PROTOC $PROTOCFLAGS %s --nanopb_out=. %s' % (dirs, esc(str(source[0])))
+
+def _nanopb_proto_emitter(target, source, env):
+    basename = os.path.splitext(str(source[0]))[0]
+    target.append(basename + '.pb.h')
+    
+    if os.path.exists(basename + '.options'):
+        source.append(basename + '.options')
+    
+    return target, source
+
+_nanopb_proto_builder = SCons.Builder.Builder(
+    generator = _nanopb_proto_actions,
+    suffix = '.pb.c',
+    src_suffix = '.proto',
+    emitter = _nanopb_proto_emitter)
+       
+def generate(env):
+    '''Add Builder for nanopb protos.'''
+    
+    env['NANOPB'] = _detect_nanopb(env)
+    env['PROTOC'] = _detect_protoc(env)
+    env['PROTOCFLAGS'] = _detect_protocflags(env)
+    
+    env.SetDefault(PROTOCPATH = ['.', os.path.join(env['NANOPB'], 'generator', 'proto')])
+    
+    env.SetDefault(NANOPB_PROTO_CMD = '$PROTOC $PROTOC_OPTS --nanopb_out=. $SOURCE')
+    env['BUILDERS']['NanopbProto'] = _nanopb_proto_builder
+    
+def exists(env):
+    return _detect_protoc(env) and _detect_protoc_opts(env)
+

tests/special_characters/SConscript

@@ -2,6 +2,5 @@
 
 Import('env')
 
-env.Proto("funny-proto+name has.characters.proto")
-env.Nanopb("funny-proto+name has.characters.pb.c", "funny-proto+name has.characters.pb")
+env.NanopbProto("funny-proto+name has.characters.proto")
 env.Object("funny-proto+name has.characters.pb.c")

tests/splint/SConscript

@@ -0,0 +1,13 @@
+# Check the nanopb core using splint
+
+Import('env')
+
+p = env.WhereIs('splint')
+
+if p:
+    env.Command('pb_decode.splint', '$NANOPB/pb_decode.c',
+        'splint -f splint/splint.rc $SOURCE 2> $TARGET')
+
+    env.Command('pb_encode.splint', '$NANOPB/pb_encode.c',
+        'splint -f splint/splint.rc $SOURCE 2> $TARGET')
+