Remove the PB_ENCODERS array.

Reduces code size by 2% and is overall cleaner.

.gitignore

@@ -5,35 +5,22 @@
 *.pb.c
 *.pb.h
 *.pb
+*.pyc
 *~
 *.tar.gz
+.sconsign.dblite
+config.log
+.sconf_temp
+tests/build
 julkaisu.txt
 docs/*.html
 docs/generator_flow.png
-example/client
+examples/simple/simple
-example/server
+examples/network_server/client
-example_avr_double/decode_double
+examples/network_server/server
-example_avr_double/encode_double
+examples/using_double_on_avr/decode_double
-example_avr_double/test_conversions
+examples/using_double_on_avr/encode_double
-example_unions/decode
+examples/using_double_on_avr/test_conversions
-example_unions/encode
+examples/using_union_messages/decode
+examples/using_union_messages/encode
 generator/nanopb_pb2.pyc
-tests/decode_unittests
-tests/encode_unittests
-tests/test_compiles
-tests/test_decode1
-tests/test_decode2
-tests/test_decode3
-tests/test_decode3_buf
-tests/test_decode_callbacks
-tests/test_encode1
-tests/test_encode2
-tests/test_encode3
-tests/test_encode3_buf
-tests/test_encode_callbacks
-tests/test_missing_fields
-tests/test_multiple_files
-tests/bc_decode
-tests/bc_encode
-tests/breakpoints
-

CHANGELOG

@@ -1,4 +1,31 @@
-nanopb-0.2.1
+nanopb-0.2.4 (2013-11-07)
+ Remove the deprecated NANOPB_INTERNALS functions from public API.
+ Document the security model.
+ Check array and bytes max sizes when encoding (issue 90)
+ Add #defines for maximum encoded message size (issue 89)
+ Add #define tags for extension fields (issue 93)
+ Fix MISRA C violations (issue 91)
+ Clean up pb_field_t definition with typedefs.
+
+nanopb-0.2.3 (2013-09-18)
+ Improve compatibility by removing ternary operator from initializations (issue 88)
+ Fix build error on Visual C++ (issue 84, patch by Markus Schwarzenberg)
+ Don't stop on unsupported extension fields (issue 83)
+ Add an example pb_syshdr.h file for non-C99 compilers
+ Reorganize tests and examples into subfolders (issue 63)
+ Switch from Makefiles to scons for building the tests
+ Make the tests buildable on Windows
+
+nanopb-0.2.2 (2013-08-18)
+ Add support for extension fields (issue 17)
+ Fix unknown fields in empty message (issue 78)
+ Include the field tags in the generated .pb.h file.
+ Add pb_decode_delimited and pb_encode_delimited wrapper functions (issue 74)
+ Add a section in top of pb.h for changing compilation settings (issue 76)
+ Documentation improvements (issues 12, 77 and others)
+ Improved tests
+
+nanopb-0.2.1 (2013-04-14)
  NOTE: The default callback function signature has changed.
        If you don't want to update your code, define PB_OLD_CALLBACK_STYLE.
  
@@ -15,7 +42,7 @@ nanopb-0.2.1
  Various new generator options
  Improved tests
 
-nanopb-0.2.0
+nanopb-0.2.0 (2013-03-02)
  NOTE: This release requires you to regenerate all .pb.c
        files. Files generated by older versions will not
        compile anymore.
@@ -28,7 +55,7 @@ nanopb-0.2.0
  Add option to give file extension to generator (by Michael Haberler)
  Documentation updates
 
-nanopb-0.1.9
+nanopb-0.1.9 (2013-02-13)
  Fixed error message bugs (issues 52, 56)
  Sanitize #ifndef filename (issue 50)
  Performance improvements
@@ -39,13 +66,13 @@ nanopb-0.1.9
  Added generator option to make message structs packed. (issue 49)
  Add more test cases.
 
-nanopb-0.1.8
+nanopb-0.1.8 (2012-12-13)
  Fix bugs in the enum short names introduced in 0.1.7 (issues 42, 43)
  Fix STATIC_ASSERT macro when using multiple .proto files. (issue 41)
  Fix missing initialization of istream.errmsg
  Make tests/Makefile work for non-gcc compilers (issue 40)
 
-nanopb-0.1.7
+nanopb-0.1.7 (2012-11-11)
  Remove "skip" mode from pb_istream_t callbacks. Example implementation had a bug. (issue 37)
  Add option to use shorter names for enum values (issue 38)
  Improve options support in generator (issues 12, 30)
@@ -55,19 +82,19 @@ nanopb-0.1.7
  Add buffer size check in example (issue 34)
  Fix build warnings on MS compilers (issue 33)
 
-nanopb-0.1.6
+nanopb-0.1.6 (2012-09-02)
  Reorganize the field decoder interface (issue 2)
  Improve performance in submessage decoding (issue 28)
  Implement error messages in the decoder side (issue 7)
  Extended testcases (alltypes test is now complete).
  Fix some compiler warnings (issues 25, 26, 27, 32).
 
-nanopb-0.1.5
+nanopb-0.1.5 (2012-08-04)
  Fix bug in decoder with packed arrays (issue 23).
  Extended testcases.
  Fix some compiler warnings.
 
-nanopb-0.1.4
+nanopb-0.1.4 (2012-07-05)
  Add compile-time options for easy-to-use >255 field support.
  Improve the detection of missing required fields.
  Added example on how to handle union messages.
@@ -75,20 +102,20 @@ nanopb-0.1.4
  Fix problems that stopped the code from compiling with some compilers.
  Fix some compiler warnings.
 
-nanopb-0.1.3
+nanopb-0.1.3 (2012-06-12)
  Refactor the field encoder interface.
  Improve generator error messages (issue 5)
  Add descriptor.proto into the #include exclusion list
  Fix some compiler warnings.
 
-nanopb-0.1.2
+nanopb-0.1.2 (2012-02-15)
  Make the generator to generate include for other .proto files (issue 4).
  Fixed generator not working on Windows (issue 3)
 
-nanopb-0.1.1
+nanopb-0.1.1 (2012-01-14)
  Fixed bug in encoder with 'bytes' fields (issue 1).
  Fixed a bug in the generator that caused a compiler error on sfixed32 and sfixed64 fields.
  Extended testcases.
 
-nanopb-0.1.0
+nanopb-0.1.0 (2012-01-06)
  First stable release.

cmake/FindNanopb.cmake

@@ -148,7 +148,7 @@ function(NANOPB_GENERATE_CPP SRCS HDRS)
              "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
       COMMAND python
       ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb
-      DEPENDS ${FIL_WE}.pb
+      DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb"
       COMMENT "Running nanopb generator on ${FIL_WE}.pb"
       VERBATIM )
   endforeach()

compat/pb_syshdr.h

@@ -0,0 +1,94 @@
+/* This is an example of a header file for platforms/compilers that do
+ * not come with stdint.h/stddef.h/stdbool.h/string.h. To use it, define
+ * PB_SYSTEM_HEADER as "pb_syshdr.h", including the quotes, and add the
+ * compat folder to your include path.
+ *
+ * It is very likely that you will need to customize this file to suit
+ * your platform. For any compiler that supports C99, this file should
+ * not be necessary.
+ */
+
+#ifndef _PB_SYSHDR_H_
+#define _PB_SYSHDR_H_
+
+/* stdint.h subset */
+#ifdef HAVE_STDINT_H
+#include <stdint.h>
+#else
+/* You will need to modify these to match the word size of your platform. */
+typedef signed char int8_t;
+typedef unsigned char uint8_t;
+typedef signed short int16_t;
+typedef unsigned short uint16_t;
+typedef signed int int32_t;
+typedef unsigned int uint32_t;
+typedef signed long long int64_t;
+typedef unsigned long long uint64_t;
+#endif
+
+/* stddef.h subset */
+#ifdef HAVE_STDDEF_H
+#include <stddef.h>
+#else
+
+typedef uint32_t size_t;
+#define offsetof(st, m) ((size_t)(&((st *)0)->m))
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+#endif
+
+/* stdbool.h subset */
+#ifdef HAVE_STDBOOL_H
+#include <stdbool.h>
+#else
+
+#ifndef __cplusplus
+typedef int bool;
+#define false 0
+#define true 1
+#endif
+
+#endif
+
+/* string.h subset */
+#ifdef HAVE_STRING_H
+#include <string.h>
+#else
+
+/* Implementations are from the Public Domain C Library (PDCLib). */
+static size_t strlen( const char * s )
+{
+    size_t rc = 0;
+    while ( s[rc] )
+    {
+        ++rc;
+    }
+    return rc;
+}
+
+static void * memcpy( void *s1, const void *s2, size_t n )
+{
+    char * dest = (char *) s1;
+    const char * src = (const char *) s2;
+    while ( n-- )
+    {
+        *dest++ = *src++;
+    }
+    return s1;
+}
+
+static void * memset( void * s, int c, size_t n )
+{
+    unsigned char * p = (unsigned char *) s;
+    while ( n-- )
+    {
+        *p++ = (unsigned char) c;
+    }
+    return s;
+}
+#endif
+
+#endif

docs/Makefile

@@ -1,4 +1,4 @@
-all: index.html concepts.html reference.html \
+all: index.html concepts.html reference.html security.html \
 	generator_flow.png
 
 %.png: %.svg

docs/concepts.rst

@@ -256,6 +256,50 @@ generates this field description array for the structure *Person_PhoneNumber*::
  };
 
 
+Extension fields
+================
+Protocol Buffers supports a concept of `extension fields`_, which are
+additional fields to a message, but defined outside the actual message.
+The definition can even be in a completely separate .proto file.
+
+The base message is declared as extensible by keyword *extensions* in
+the .proto file::
+
+ message MyMessage {
+     .. fields ..
+     extensions 100 to 199;
+ }
+
+For each extensible message, *nanopb_generator.py* declares an additional
+callback field called *extensions*. The field and associated datatype
+*pb_extension_t* forms a linked list of handlers. When an unknown field is
+encountered, the decoder calls each handler in turn until either one of them
+handles the field, or the list is exhausted.
+
+The actual extensions are declared using the *extend* keyword in the .proto,
+and are in the global namespace::
+
+ extend MyMessage {
+     optional int32 myextension = 100;
+ }
+
+For each extension, *nanopb_generator.py* creates a constant of type
+*pb_extension_type_t*. To link together the base message and the extension,
+you have to:
+
+1. Allocate storage for your field, matching the datatype in the .proto.
+   For example, for a *int32* field, you need a *int32_t* variable to store
+   the value.
+2. Create a *pb_extension_t* constant, with pointers to your variable and
+   to the generated *pb_extension_type_t*.
+3. Set the *message.extensions* pointer to point to the *pb_extension_t*.
+
+An example of this is available in *tests/test_encode_extensions.c* and
+*tests/test_decode_extensions.c*.
+
+.. _`extension fields`: https://developers.google.com/protocol-buffers/docs/proto#extensions
+
+
 Return values and error handling
 ================================
 

docs/index.rst

@@ -41,7 +41,7 @@ Features and limitations
 #) Allows specifying maximum size for strings and arrays, so that they can be allocated statically.
 #) No malloc needed: everything can be allocated statically or on the stack.
 #) You can use either encoder or decoder alone to cut the code size in half.
-#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, packed arrays.
+#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, packed arrays, extension fields.
 #) Callback mechanism for handling messages larger than can fit in available RAM.
 #) Extensive set of tests.
 
@@ -90,20 +90,37 @@ After that, buffer will contain the encoded message.
 The number of bytes in the message is stored in *stream.bytes_written*.
 You can feed the message to *protoc --decode=Example message.proto* to verify its validity.
 
-For complete examples of the simple cases, see *tests/test_decode1.c* and *tests/test_encode1.c*. For an example with network interface, see the *example* subdirectory.
+For a complete example of the simple case, see *example/simple.c*.
+For a more complex example with network interface, see the *example/network_server* subdirectory.
 
 Compiler requirements
 =====================
-Nanopb should compile with most ansi-C compatible compilers. It however requires a few header files to be available:
+Nanopb should compile with most ansi-C compatible compilers. It however
+requires a few header files to be available:
 
 #) *string.h*, with these functions: *strlen*, *memcpy*, *memset*
 #) *stdint.h*, for definitions of *int32_t* etc.
 #) *stddef.h*, for definition of *size_t*
 #) *stdbool.h*, for definition of *bool*
 
-If these header files do not come with your compiler, you should be able to find suitable replacements online. Mostly the requirements are very simple, just a few basic functions and typedefs.
+If these header files do not come with your compiler, you can use the
+file *compat/pb_syshdr.h* instead. It contains an example of how to provide
+the dependencies. You may have to edit it a bit to suit your custom platform.
 
-Debugging and testing
+To use the pb_syshdr.h, define *PB_SYSTEM_HEADER* to be the name of your custom
-=====================
+header file. It should provide all the dependencies listed above.
-Extensive unittests are included under the *tests* folder. Just type *make* there to run the tests.
+
+Running the test cases
+======================
+Extensive unittests and test cases are included under the *tests* folder.
+
+To build the tests, you will need the `scons`__ build system. The tests should
+be runnable on most platforms. Windows and Linux builds are regularly tested.
+
+__ http://www.scons.org/
+
+In addition to the build system, you will also need a working Google Protocol
+Buffers *protoc* compiler, and the Python bindings for Protocol Buffers. On
+Debian-based systems, install the following packages: *protobuf-compiler*,
+*python-protobuf* and *libprotobuf-dev*.
 

docs/menu.rst

@@ -3,8 +3,9 @@
     1) `Overview`_
     2) `Concepts`_
     3) `API reference`_
+    4) `Security model`_
     
 .. _`Overview`: index.html
 .. _`Concepts`: concepts.html
 .. _`API reference`: reference.html
-
+.. _`Security model`: security.html

docs/reference.rst

@@ -25,7 +25,9 @@ __BIG_ENDIAN__                 Set this if your platform stores integers and
                                systems (different layout for ints and floats)
                                are currently not supported.
 NANOPB_INTERNALS               Set this to expose the field encoder functions
-                               that are hidden since nanopb-0.1.3.
+                               that are hidden since nanopb-0.1.3. Starting
+                               with nanopb-0.2.4, this flag does nothing. Use
+                               the newer functions that have better interface.
 PB_MAX_REQUIRED_FIELDS         Maximum number of required fields to check for
                                presence. Default value is 64. Increases stack
                                usage 1 byte per every 8 fields. Compiler
@@ -49,6 +51,11 @@ PB_BUFFER_ONLY                 Disables the support for custom streams. Only
 PB_OLD_CALLBACK_STYLE          Use the old function signature (void\* instead
                                of void\*\*) for callback fields. This was the
                                default until nanopb-0.2.1.
+PB_SYSTEM_HEADER               Replace the standard header files with a single
+                               header file. It should define all the required
+                               functions and typedefs listed on the
+                               `overview page`_. Value must include quotes,
+                               for example *#define PB_SYSTEM_HEADER "foo.h"*.
 ============================  ================================================
 
 The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow
@@ -56,7 +63,7 @@ raising some datatype limits to suit larger messages. Their need is recognized
 automatically by C-preprocessor #if-directives in the generated .pb.h files.
 The default setting is to use the smallest datatypes (least resources used).
 
-
+.. _`overview page`: index.html#compiler-requirements
 
 
 Proto file options
@@ -299,6 +306,41 @@ Protocol Buffers wire types. These are used with `pb_encode_tag`_. ::
         PB_WT_32BIT  = 5
     } pb_wire_type_t;
 
+pb_extension_type_t
+-------------------
+Defines the handler functions and auxiliary data for a field that extends
+another message. Usually autogenerated by *nanopb_generator.py*::
+
+    typedef struct {
+        bool (*decode)(pb_istream_t *stream, pb_extension_t *extension,
+                   uint32_t tag, pb_wire_type_t wire_type);
+        bool (*encode)(pb_ostream_t *stream, const pb_extension_t *extension);
+        const void *arg;
+    } pb_extension_type_t;
+
+In the normal case, the function pointers are *NULL* and the decoder and
+encoder use their internal implementations. The internal implementations
+assume that *arg* points to a *pb_field_t* that describes the field in question.
+
+To implement custom processing of unknown fields, you can provide pointers
+to your own functions. Their functionality is mostly the same as for normal
+callback fields, except that they get called for any unknown field when decoding.
+
+pb_extension_t
+--------------
+Ties together the extension field type and the storage for the field value::
+
+    typedef struct {
+        const pb_extension_type_t *type;
+        void *dest;
+        pb_extension_t *next;
+    } pb_extension_t;
+
+:type:      Pointer to the structure that defines the callback functions.
+:dest:      Pointer to the variable that stores the field value
+            (as used by the default extension callback functions.)
+:next:      Pointer to the next extension handler, or *NULL*.
+
 PB_GET_ERROR
 ------------
 Get the current error message from a stream, or a placeholder string if

docs/security.rst

@@ -0,0 +1,79 @@
+======================
+Nanopb: Security model
+======================
+
+.. include :: menu.rst
+
+.. contents ::
+
+
+
+Importance of security in a Protocol Buffers library
+====================================================
+In the context of protocol buffers, security comes into play when decoding
+untrusted data. Naturally, if the attacker can modify the contents of a
+protocol buffers message, he can feed the application any values possible.
+Therefore the application itself must be prepared to receive untrusted values.
+
+Where nanopb plays a part is preventing the attacker from running arbitrary
+code on the target system. Mostly this means that there must not be any
+possibility to cause buffer overruns, memory corruption or invalid pointers
+by the means of crafting a malicious message.
+
+Division of trusted and untrusted data
+======================================
+The following data is regarded as **trusted**. It must be under the control of
+the application writer. Malicious data in these structures could cause
+security issues, such as execution of arbitrary code:
+
+1. Callback and extension fields in message structures given to pb_encode()
+   and pb_decode(). These fields are memory pointers, and are generated
+   depending on the .proto file.
+2. The automatically generated field definitions, i.e. *pb_field_t* lists.
+3. Contents of the *pb_istream_t* and *pb_ostream_t* structures (this does not
+   mean the contents of the stream itself, just the stream definition).
+
+The following data is regarded as **untrusted**. Invalid/malicious data in
+these will cause "garbage in, garbage out" behaviour. It will not cause
+buffer overflows, information disclosure or other security problems:
+
+1. All data read from *pb_istream_t*.
+2. All fields in message structures, except callbacks and extensions.
+   (Beginning with nanopb-0.2.4, in earlier versions the field sizes are partially unchecked.)
+
+Invariants
+==========
+The following invariants are maintained during operation, even if the
+untrusted data has been maliciously crafted:
+
+1. Nanopb will never read more than *bytes_left* bytes from *pb_istream_t*.
+2. Nanopb will never write more than *max_size* bytes to *pb_ostream_t*.
+3. Nanopb will never access memory out of bounds of the message structure.
+4. After pb_decode() returns successfully, the message structure will be
+   internally consistent:
+
+   - The *count* fields of arrays will not exceed the array size.
+   - The *size* field of bytes will not exceed the allocated size.
+   - All string fields will have null terminator.
+
+5. After pb_encode() returns successfully, the resulting message is a valid
+   protocol buffers message. (Except if user-defined callbacks write incorrect
+   data.)
+
+Further considerations
+======================
+Even if the nanopb library is free of any security issues, there are still
+several possible attack vectors that the application author must consider.
+The following list is not comprehensive:
+
+1. Stack usage may depend on the contents of the message. The message
+   definition places an upper bound on how much stack will be used. Tests
+   should be run with all fields present, to record the maximum possible
+   stack usage.
+2. Callbacks can do anything. The code for the callbacks must be carefully
+   checked if they are used with untrusted data.
+3. If using stream input, a maximum size should be set in *pb_istream_t* to
+   stop a denial of service attack from using an infinite message.
+4. If using network sockets as streams, a timeout should be set to stop
+   denial of service attacks.
+

example/Makefile

@@ -1,14 +0,0 @@
-CFLAGS=-ansi -Wall -Werror -I .. -g -O0
-DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h
-
-all: server client
-
-clean:
-	rm -f server client fileproto.pb.c fileproto.pb.h
-
-%: %.c $(DEPS) fileproto.pb.h fileproto.pb.c
-	$(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c fileproto.pb.c common.c
-
-fileproto.pb.c fileproto.pb.h: fileproto.proto ../generator/nanopb_generator.py
-	protoc -I. -I../generator -I/usr/include -ofileproto.pb $<
-	python ../generator/nanopb_generator.py fileproto.pb

example_avr_double/Makefile

@@ -1,22 +0,0 @@
-CFLAGS=-Wall -Werror -I .. -g -O0
-DEPS=double_conversion.c ../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h
-
-all: run_tests
-
-clean:
-	rm -f test_conversions encode_double decode_double doubleproto.pb.c doubleproto.pb.h
-
-test_conversions: test_conversions.c double_conversion.c
-	$(CC) $(CFLAGS) -o $@ $^
-
-%: %.c $(DEPS) doubleproto.pb.h doubleproto.pb.c
-	$(CC) $(CFLAGS) -o $@ $< double_conversion.c ../pb_decode.c ../pb_encode.c doubleproto.pb.c
-
-doubleproto.pb.c doubleproto.pb.h: doubleproto.proto ../generator/nanopb_generator.py
-	protoc -I. -I../generator -I/usr/include -odoubleproto.pb $<
-	python ../generator/nanopb_generator.py doubleproto.pb
-
-run_tests: test_conversions encode_double decode_double
-	./test_conversions
-	./encode_double | ./decode_double
-    

example_unions/Makefile

@@ -1,17 +0,0 @@
-CFLAGS=-ansi -Wall -Werror -I .. -g -O0
-DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h
-
-all: encode decode
-	./encode 1 | ./decode
-	./encode 2 | ./decode
-	./encode 3 | ./decode
-
-clean:
-	rm -f encode unionproto.pb.h unionproto.pb.c
-
-%: %.c $(DEPS) unionproto.pb.h unionproto.pb.c
-	$(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c unionproto.pb.c
-
-unionproto.pb.h unionproto.pb.c: unionproto.proto ../generator/nanopb_generator.py
-	protoc -I. -I../generator -I/usr/include -ounionproto.pb $<
-	python ../generator/nanopb_generator.py unionproto.pb

examples/network_server/Makefile

@@ -0,0 +1,19 @@
+CFLAGS = -ansi -Wall -Werror -g -O0
+
+# Path to the nanopb root folder
+NANOPB_DIR = ../..
+DEPS = $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \
+       $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h $(NANOPB_DIR)/pb.h
+CFLAGS += -I$(NANOPB_DIR)
+
+all: server client
+
+clean:
+	rm -f server client fileproto.pb.c fileproto.pb.h
+
+%: %.c $(DEPS) fileproto.pb.h fileproto.pb.c
+	$(CC) $(CFLAGS) -o $@ $< $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c fileproto.pb.c common.c
+
+fileproto.pb.c fileproto.pb.h: fileproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py
+	protoc -ofileproto.pb $<
+	python $(NANOPB_DIR)/generator/nanopb_generator.py fileproto.pb

examples/network_server/README

@@ -0,0 +1,60 @@
+Nanopb example "network_server"
+===============================
+
+This example demonstrates the use of nanopb to communicate over network
+connections. It consists of a server that sends file listings, and of
+a client that requests the file list from the server.
+
+Example usage
+-------------
+
+user@host:~/nanopb/examples/network_server$ make        # Build the example
+protoc -ofileproto.pb fileproto.proto
+python ../../generator/nanopb_generator.py fileproto.pb
+Writing to fileproto.pb.h and fileproto.pb.c
+cc -ansi -Wall -Werror -I .. -g -O0 -I../.. -o server server.c
+    ../../pb_decode.c ../../pb_encode.c fileproto.pb.c common.c
+cc -ansi -Wall -Werror -I .. -g -O0 -I../.. -o client client.c
+    ../../pb_decode.c ../../pb_encode.c fileproto.pb.c common.c
+
+user@host:~/nanopb/examples/network_server$ ./server &  # Start the server on background
+[1] 24462
+
+petteri@oddish:~/nanopb/examples/network_server$ ./client /bin  # Request the server to list /bin
+Got connection.
+Listing directory: /bin
+1327119    bzdiff
+1327126    bzless
+1327147    ps
+1327178    ntfsmove
+1327271    mv
+1327187    mount
+1327259    false
+1327266    tempfile
+1327285    zfgrep
+1327165    gzexe
+1327204    nc.openbsd
+1327260    uname
+
+
+Details of implementation
+-------------------------
+fileproto.proto contains the portable Google Protocol Buffers protocol definition.
+It could be used as-is to implement a server or a client in any other language, for
+example Python or Java.
+
+fileproto.options contains the nanopb-specific options for the protocol file. This
+sets the amount of space allocated for file names when decoding messages.
+
+common.c/h contains functions that allow nanopb to read and write directly from
+network socket. This way there is no need to allocate a separate buffer to store
+the message.
+
+server.c contains the code to open a listening socket, to respond to clients and
+to list directory contents.
+
+client.c contains the code to connect to a server, to send a request and to print
+the response message.
+
+The code is implemented using the POSIX socket api, but it should be easy enough
+to port into any other socket api, such as lwip.

examples/simple/Makefile

@@ -0,0 +1,22 @@
+# Compiler flags to enable all warnings & debug info
+CFLAGS = -Wall -Werror -g -O0
+
+# Path to the nanopb root folder
+NANOPB_DIR = ../..
+CFLAGS += -I$(NANOPB_DIR)
+
+# C source code files that are required
+CSRC  = simple.c                   # The main program
+CSRC += simple.pb.c                # The compiled protocol definition
+CSRC += $(NANOPB_DIR)/pb_encode.c  # The nanopb encoder
+CSRC += $(NANOPB_DIR)/pb_decode.c  # The nanopb decoder
+
+# Build rule for the main program
+simple: $(CSRC)
+	$(CC) $(CFLAGS) -osimple $(CSRC)
+
+# Build rule for the protocol
+simple.pb.c: simple.proto
+	protoc -osimple.pb simple.proto
+	python $(NANOPB_DIR)/generator/nanopb_generator.py simple.pb
+

examples/simple/README

@@ -0,0 +1,30 @@
+Nanopb example "simple"
+=======================
+
+This example demonstrates the very basic use of nanopb. It encodes and
+decodes a simple message.
+
+The code uses four different API functions:
+
+  * pb_ostream_from_buffer() to declare the output buffer that is to be used
+  * pb_encode() to encode a message
+  * pb_istream_from_buffer() to declare the input buffer that is to be used
+  * pb_decode() to decode a message
+
+Example usage
+-------------
+
+On Linux, simply type "make" to build the example. After that, you can
+run it with the command: ./simple
+
+On other platforms, you first have to compile the protocol definition using
+the following two commands::
+
+  protoc -osimple.pb simple.proto
+  python nanopb_generator.py simple.pb
+
+After that, add the following four files to your project and compile:
+
+  simple.c  simple.pb.c  pb_encode.c  pb_decode.c
+
+

examples/simple/simple.c

@@ -0,0 +1,68 @@
+#include <stdio.h>
+#include <pb_encode.h>
+#include <pb_decode.h>
+#include "simple.pb.h"
+
+int main()
+{
+    /* This is the buffer where we will store our message. */
+    uint8_t buffer[128];
+    size_t message_length;
+    bool status;
+    
+    /* Encode our message */
+    {
+        /* Allocate space on the stack to store the message data.
+         *
+         * Nanopb generates simple struct definitions for all the messages.
+         * - check out the contents of simple.pb.h! */
+        SimpleMessage message;
+        
+        /* Create a stream that will write to our buffer. */
+        pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        
+        /* Fill in the lucky number */
+        message.lucky_number = 13;
+        
+        /* Now we are ready to encode the message! */
+        status = pb_encode(&stream, SimpleMessage_fields, &message);
+        message_length = stream.bytes_written;
+        
+        /* Then just check for any errors.. */
+        if (!status)
+        {
+            printf("Encoding failed: %s\n", PB_GET_ERROR(&stream));
+            return 1;
+        }
+    }
+    
+    /* Now we could transmit the message over network, store it in a file or
+     * wrap it to a pigeon's leg.
+     */
+
+    /* But because we are lazy, we will just decode it immediately. */
+    
+    {
+        /* Allocate space for the decoded message. */
+        SimpleMessage message;
+        
+        /* Create a stream that reads from the buffer. */
+        pb_istream_t stream = pb_istream_from_buffer(buffer, message_length);
+        
+        /* Now we are ready to decode the message. */
+        status = pb_decode(&stream, SimpleMessage_fields, &message);
+        
+        /* Check for errors... */
+        if (!status)
+        {
+            printf("Decoding failed: %s\n", PB_GET_ERROR(&stream));
+            return 1;
+        }
+        
+        /* Print the data contained in the message. */
+        printf("Your lucky number was %d!\n", message.lucky_number);
+    }
+    
+    return 0;
+}
+

examples/simple/simple.proto

@@ -0,0 +1,7 @@
+// A very simple protocol definition, consisting of only
+// one message.
+
+message SimpleMessage {
+    required int32 lucky_number = 1;
+}
+

examples/using_double_on_avr/Makefile

@@ -0,0 +1,29 @@
+CFLAGS = -Wall -Werror -g -O0
+
+# Path to the nanopb root directory
+NANOPB_DIR = ../..
+DEPS = double_conversion.c $(NANOPB_DIR)/pb.h \
+	$(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \
+	$(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h
+CFLAGS += -I$(NANOPB_DIR)
+
+all: run_tests
+
+clean:
+	rm -f test_conversions encode_double decode_double doubleproto.pb.c doubleproto.pb.h
+
+test_conversions: test_conversions.c double_conversion.c
+	$(CC) $(CFLAGS) -o $@ $^
+
+%: %.c $(DEPS) doubleproto.pb.h doubleproto.pb.c
+	$(CC) $(CFLAGS) -o $@ $< double_conversion.c \
+		$(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c doubleproto.pb.c
+
+doubleproto.pb.c doubleproto.pb.h: doubleproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py
+	protoc -odoubleproto.pb $<
+	python $(NANOPB_DIR)/generator/nanopb_generator.py doubleproto.pb
+
+run_tests: test_conversions encode_double decode_double
+	./test_conversions
+	./encode_double | ./decode_double
+    

examples/using_double_on_avr/README

@@ -1,3 +1,6 @@
+Nanopb example "using_double_on_avr"
+====================================
+
 Some processors/compilers, such as AVR-GCC, do not support the double
 datatype. Instead, they have sizeof(double) == 4. Because protocol
 binary format uses the double encoding directly, this causes trouble
@@ -9,7 +12,7 @@ platforms. The file double_conversion.c provides functions that
 convert these values to/from floats, without relying on compiler
 support.
 
-To use this method, you need to make two modifications to your code:
+To use this method, you need to make some modifications to your code:
 
 1) Change all 'double' fields into 'fixed64' in the .proto.
 
@@ -17,6 +20,6 @@ To use this method, you need to make two modifications to your code:
 
 3) Whenever reading a 'double' field, use double_to_float().
 
-The conversion routines should be as accurate as the float datatype can
+The conversion routines are as accurate as the float datatype can
 be. Furthermore, they should handle all special values (NaN, inf, denormalized
 numbers) correctly. There are testcases in test_conversions.c.

examples/using_union_messages/Makefile

@@ -0,0 +1,22 @@
+CFLAGS = -ansi -Wall -Werror -g -O0
+
+# Path to the nanopb root folder
+NANOPB_DIR = ../..
+DEPS = $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \
+       $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h $(NANOPB_DIR)/pb.h
+CFLAGS += -I$(NANOPB_DIR)
+
+all: encode decode
+	./encode 1 | ./decode
+	./encode 2 | ./decode
+	./encode 3 | ./decode
+
+clean:
+	rm -f encode unionproto.pb.h unionproto.pb.c
+
+%: %.c $(DEPS) unionproto.pb.h unionproto.pb.c
+	$(CC) $(CFLAGS) -o $@ $< $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c unionproto.pb.c
+
+unionproto.pb.h unionproto.pb.c: unionproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py
+	protoc -ounionproto.pb $<
+	python $(NANOPB_DIR)/generator/nanopb_generator.py unionproto.pb

examples/using_union_messages/README

@@ -0,0 +1,52 @@
+Nanopb example "using_union_messages"
+=====================================
+
+Union messages is a common technique in Google Protocol Buffers used to
+represent a group of messages, only one of which is passed at a time.
+It is described in Google's documentation:
+https://developers.google.com/protocol-buffers/docs/techniques#union
+
+This directory contains an example on how to encode and decode union messages
+with minimal memory usage. Usually, nanopb would allocate space to store
+all of the possible messages at the same time, even though at most one of
+them will be used at a time.
+
+By using some of the lower level nanopb APIs, we can manually generate the
+top level message, so that we only need to allocate the one submessage that
+we actually want. Similarly when decoding, we can manually read the tag of
+the top level message, and only then allocate the memory for the submessage
+after we already know its type.
+
+
+Example usage
+-------------
+
+Type `make` to run the example. It will build it and run commands like
+following:
+
+./encode 1 | ./decode
+Got MsgType1: 42
+./encode 2 | ./decode
+Got MsgType2: true
+./encode 3 | ./decode
+Got MsgType3: 3 1415
+
+This simply demonstrates that the "decode" program has correctly identified
+the type of the received message, and managed to decode it.
+
+
+Details of implementation
+-------------------------
+
+unionproto.proto contains the protocol used in the example. It consists of
+three messages: MsgType1, MsgType2 and MsgType3, which are collected together
+into UnionMessage.
+
+encode.c takes one command line argument, which should be a number 1-3. It
+then fills in and encodes the corresponding message, and writes it to stdout.
+
+decode.c reads a UnionMessage from stdin. Then it calls the function
+decode_unionmessage_type() to determine the type of the message. After that,
+the corresponding message is decoded and the contents of it printed to the
+screen.
+

generator/Makefile

@@ -1,2 +1,5 @@
 nanopb_pb2.py: nanopb.proto
 	protoc --python_out=. -I /usr/include -I . nanopb.proto
+
+plugin_pb2.py: plugin.proto
+	protoc --python_out=. -I /usr/include -I . plugin.proto

generator/nanopb_generator.py

@@ -1,5 +1,7 @@
+#!/usr/bin/python
+
 '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.'''
-nanopb_version = "nanopb-0.2.2-dev"
+nanopb_version = "nanopb-0.2.5-dev"
 
 try:
     import google.protobuf.descriptor_pb2 as descriptor
@@ -36,22 +38,22 @@ except:
 import time
 import os.path
 
-# Values are tuple (c type, pb type)
+# Values are tuple (c type, pb type, encoded size)
 FieldD = descriptor.FieldDescriptorProto
 datatypes = {
-    FieldD.TYPE_BOOL:       ('bool',     'BOOL'),
+    FieldD.TYPE_BOOL:       ('bool',     'BOOL',        1),
-    FieldD.TYPE_DOUBLE:     ('double',   'DOUBLE'),
+    FieldD.TYPE_DOUBLE:     ('double',   'DOUBLE',      8),
-    FieldD.TYPE_FIXED32:    ('uint32_t', 'FIXED32'),
+    FieldD.TYPE_FIXED32:    ('uint32_t', 'FIXED32',     4),
-    FieldD.TYPE_FIXED64:    ('uint64_t', 'FIXED64'),
+    FieldD.TYPE_FIXED64:    ('uint64_t', 'FIXED64',     8),
-    FieldD.TYPE_FLOAT:      ('float',    'FLOAT'),
+    FieldD.TYPE_FLOAT:      ('float',    'FLOAT',       4),
-    FieldD.TYPE_INT32:      ('int32_t',  'INT32'),
+    FieldD.TYPE_INT32:      ('int32_t',  'INT32',       5),
-    FieldD.TYPE_INT64:      ('int64_t',  'INT64'),
+    FieldD.TYPE_INT64:      ('int64_t',  'INT64',      10),
-    FieldD.TYPE_SFIXED32:   ('int32_t',  'SFIXED32'),
+    FieldD.TYPE_SFIXED32:   ('int32_t',  'SFIXED32',    4),
-    FieldD.TYPE_SFIXED64:   ('int64_t',  'SFIXED64'),
+    FieldD.TYPE_SFIXED64:   ('int64_t',  'SFIXED64',    8),
-    FieldD.TYPE_SINT32:     ('int32_t',  'SINT32'),
+    FieldD.TYPE_SINT32:     ('int32_t',  'SINT32',      5),
-    FieldD.TYPE_SINT64:     ('int64_t',  'SINT64'),
+    FieldD.TYPE_SINT64:     ('int64_t',  'SINT64',     10),
-    FieldD.TYPE_UINT32:     ('uint32_t', 'UINT32'),
+    FieldD.TYPE_UINT32:     ('uint32_t', 'UINT32',      5),
-    FieldD.TYPE_UINT64:     ('uint64_t', 'UINT64')
+    FieldD.TYPE_UINT64:     ('uint64_t', 'UINT64',     10)
 }
 
 class Names:
@@ -81,6 +83,55 @@ def names_from_type_name(type_name):
         raise NotImplementedError("Lookup of non-absolute type names is not supported")
     return Names(type_name[1:].split('.'))
 
+def varint_max_size(max_value):
+    '''Returns the maximum number of bytes a varint can take when encoded.'''
+    for i in range(1, 11):
+        if (max_value >> (i * 7)) == 0:
+            return i
+    raise ValueError("Value too large for varint: " + str(max_value))
+
+assert varint_max_size(0) == 1
+assert varint_max_size(127) == 1
+assert varint_max_size(128) == 2
+
+class EncodedSize:
+    '''Class used to represent the encoded size of a field or a message.
+    Consists of a combination of symbolic sizes and integer sizes.'''
+    def __init__(self, value = 0, symbols = []):
+        if isinstance(value, (str, Names)):
+            symbols = [str(value)]
+            value = 0
+        self.value = value
+        self.symbols = symbols
+    
+    def __add__(self, other):
+        if isinstance(other, (int, long)):
+            return EncodedSize(self.value + other, self.symbols)
+        elif isinstance(other, (str, Names)):
+            return EncodedSize(self.value, self.symbols + [str(other)])
+        elif isinstance(other, EncodedSize):
+            return EncodedSize(self.value + other.value, self.symbols + other.symbols)
+        else:
+            raise ValueError("Cannot add size: " + repr(other))
+
+    def __mul__(self, other):
+        if isinstance(other, (int, long)):
+            return EncodedSize(self.value * other, [str(other) + '*' + s for s in self.symbols])
+        else:
+            raise ValueError("Cannot multiply size: " + repr(other))
+
+    def __str__(self):
+        if not self.symbols:
+            return str(self.value)
+        else:
+            return '(' + str(self.value) + ' + ' + ' + '.join(self.symbols) + ')'
+
+    def upperlimit(self):
+        if not self.symbols:
+            return self.value
+        else:
+            return 2**32 - 1
+
 class Enum:
     def __init__(self, names, desc, enum_options):
         '''desc is EnumDescriptorProto'''
@@ -111,6 +162,7 @@ class Field:
         self.max_size = None
         self.max_count = None
         self.array_decl = ""
+        self.enc_size = None
         
         # Parse field options
         if field_options.HasField("max_size"):
@@ -139,12 +191,13 @@ class Field:
         
         # Decide the C data type to use in the struct.
         if datatypes.has_key(desc.type):
-            self.ctype, self.pbtype = datatypes[desc.type]
+            self.ctype, self.pbtype, self.enc_size = datatypes[desc.type]
         elif desc.type == FieldD.TYPE_ENUM:
             self.pbtype = 'ENUM'
             self.ctype = names_from_type_name(desc.type_name)
             if self.default is not None:
                 self.default = self.ctype + self.default
+            self.enc_size = 5 # protoc rejects enum values > 32 bits
         elif desc.type == FieldD.TYPE_STRING:
             self.pbtype = 'STRING'
             if self.max_size is None:
@@ -152,15 +205,18 @@ class Field:
             else:
                 self.ctype = 'char'
                 self.array_decl += '[%d]' % self.max_size
+                self.enc_size = varint_max_size(self.max_size) + self.max_size
         elif desc.type == FieldD.TYPE_BYTES:
             self.pbtype = 'BYTES'
             if self.max_size is None:
                 can_be_static = False
             else:
                 self.ctype = self.struct_name + self.name + 't'
+                self.enc_size = varint_max_size(self.max_size) + self.max_size
         elif desc.type == FieldD.TYPE_MESSAGE:
             self.pbtype = 'MESSAGE'
             self.ctype = self.submsgname = names_from_type_name(desc.type_name)
+            self.enc_size = None # Needs to be filled in after the message type is available
         else:
             raise NotImplementedError(desc.type)
         
@@ -244,10 +300,11 @@ class Field:
         '''Return the pb_field_t initializer to use in the constant array.
         prev_field_name is the name of the previous field or None.
         '''
-        result  = '    PB_FIELD(%3d, ' % self.tag
+        result  = '    PB_FIELD2(%3d, ' % self.tag
         result += '%-8s, ' % self.pbtype
         result += '%s, ' % self.rules
         result += '%s, ' % self.allocation
+        result += '%s, ' % ("FIRST" if not prev_field_name else "OTHER")
         result += '%s, ' % self.struct_name
         result += '%s, ' % self.name
         result += '%s, ' % (prev_field_name or self.name)
@@ -274,6 +331,51 @@ class Field:
 
         return max(self.tag, self.max_size, self.max_count)        
 
+    def encoded_size(self, allmsgs):
+        '''Return the maximum size that this field can take when encoded,
+        including the field tag. If the size cannot be determined, returns
+        None.'''
+        
+        if self.allocation != 'STATIC':
+            return None
+        
+        if self.pbtype == 'MESSAGE':
+            for msg in allmsgs:
+                if msg.name == self.submsgname:
+                    encsize = msg.encoded_size(allmsgs)
+                    if encsize is None:
+                        return None # Submessage size is indeterminate
+                        
+                    # Include submessage length prefix
+                    encsize += varint_max_size(encsize.upperlimit())
+                    break
+            else:
+                # Submessage cannot be found, this currently occurs when
+                # the submessage type is defined in a different file.
+                # Instead of direct numeric value, reference the size that
+                # has been #defined in the other file.
+                encsize = EncodedSize(self.submsgname + 'size')
+
+                # We will have to make a conservative assumption on the length
+                # prefix size, though.
+                encsize += 5
+
+        elif self.enc_size is None:
+            raise RuntimeError("Could not determine encoded size for %s.%s"
+                               % (self.struct_name, self.name))
+        else:
+            encsize = EncodedSize(self.enc_size)
+        
+        encsize += varint_max_size(self.tag << 3) # Tag + wire type
+
+        if self.rules == 'REPEATED':
+            # Decoders must be always able to handle unpacked arrays.
+            # Therefore we have to reserve space for it, even though
+            # we emit packed arrays ourselves.
+            encsize *= self.max_count
+        
+        return encsize
+
 
 class ExtensionRange(Field):
     def __init__(self, struct_name, range_start, field_options):
@@ -302,6 +404,12 @@ class ExtensionRange(Field):
     
     def tags(self):
         return ''
+        
+    def encoded_size(self, allmsgs):
+        # We exclude extensions from the count, because they cannot be known
+        # until runtime. Other option would be to return None here, but this
+        # way the value remains useful if extensions are not used.
+        return EncodedSize(0)
 
 class ExtensionField(Field):
     def __init__(self, struct_name, desc, field_options):
@@ -310,18 +418,31 @@ class ExtensionField(Field):
         Field.__init__(self, self.fullname + 'struct', desc, field_options)
         
         if self.rules != 'OPTIONAL':
-            raise NotImplementedError("Only 'optional' is supported for extension fields. "
+            self.skip = True
-               + "(%s.rules == %s)" % (self.fullname, self.rules))
+        else:
+            self.skip = False
+            self.rules = 'OPTEXT'
 
-        self.rules = 'OPTEXT'
+    def tags(self):
+        '''Return the #define for the tag number of this field.'''
+        identifier = '%s_tag' % self.fullname
+        return '#define %-40s %d\n' % (identifier, self.tag)
 
     def extension_decl(self):
         '''Declaration of the extension type in the .pb.h file'''
+        if self.skip:
+            msg = '/* Extension field %s was skipped because only "optional"\n' % self.fullname
+            msg +='   type of extension fields is currently supported. */\n'
+            return msg
+        
         return 'extern const pb_extension_type_t %s;\n' % self.fullname
 
     def extension_def(self):
         '''Definition of the extension type in the .pb.c file'''
 
+        if self.skip:
+            return ''
+
         result  = 'typedef struct {\n'
         result += str(self)
         result += '\n} %s;\n\n' % self.struct_name
@@ -418,6 +539,18 @@ class Message:
         result += '    PB_LAST_FIELD\n};'
         return result
 
+    def encoded_size(self, allmsgs):
+        '''Return the maximum size that this message can take when encoded.
+        If the size cannot be determined, returns None.
+        '''
+        size = EncodedSize(0)
+        for field in self.fields:
+            fsize = field.encoded_size(allmsgs)
+            if fsize is None:
+                return None
+            size += fsize
+        
+        return size
 
 
 # ---------------------------------------------------------------------------
@@ -475,7 +608,8 @@ def parse_file(fdesc, file_options):
     
     for names, extension in iterate_extensions(fdesc, base_name):
         field_options = get_nanopb_suboptions(extension, file_options, names)
-        extensions.append(ExtensionField(names, extension, field_options))
+        if field_options.type != nanopb_pb2.FT_IGNORE:
+            extensions.append(ExtensionField(names, extension, field_options))
     
     # Fix field default values where enum short names are used.
     for enum in enums:
@@ -580,20 +714,31 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio
     for msg in sort_dependencies(messages):
         for field in msg.fields:
             yield field.tags()
+    for extension in extensions:
+        yield extension.tags()
     yield '\n'
     
     yield '/* Struct field encoding specification for nanopb */\n'
     for msg in messages:
         yield msg.fields_declaration() + '\n'
+    yield '\n'
     
-    yield '\n#ifdef __cplusplus\n'
+    yield '/* Maximum encoded size of messages (where known) */\n'
+    for msg in messages:
+        msize = msg.encoded_size(messages)
+        if msize is not None:
+            identifier = '%s_size' % msg.name
+            yield '#define %-40s %s\n' % (identifier, msize)
+    yield '\n'
+    
+    yield '#ifdef __cplusplus\n'
     yield '} /* extern "C" */\n'
     yield '#endif\n'
     
     # End of header
     yield '\n#endif\n'
 
-def generate_source(headername, enums, messages, extensions):
+def generate_source(headername, enums, messages, extensions, options):
     '''Generate content for a source file.'''
     
     yield '/* Automatically generated nanopb constant definitions */\n'
@@ -771,73 +916,126 @@ optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", def
 optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[],
     help="Set generator option (max_size, max_count etc.).")
 
-def process(filenames, options):
+def process_file(filename, fdesc, options):
-    '''Process the files given on the command line.'''
+    '''Process a single file.
+    filename: The full path to the .proto or .pb source file, as string.
+    fdesc: The loaded FileDescriptorSet, or None to read from the input file.
+    options: Command line options as they come from OptionsParser.
+    
+    Returns a dict:
+        {'headername': Name of header file,
+         'headerdata': Data for the .h header file,
+         'sourcename': Name of the source code file,
+         'sourcedata': Data for the .c source code file
+        }
+    '''
+    toplevel_options = nanopb_pb2.NanoPBOptions()
+    for s in options.settings:
+        text_format.Merge(s, toplevel_options)
+    
+    if not fdesc:
+        data = open(filename, 'rb').read()
+        fdesc = descriptor.FileDescriptorSet.FromString(data).file[0]
+    
+    # Check if there is a separate .options file
+    try:
+        optfilename = options.options_file % os.path.splitext(filename)[0]
+    except TypeError:
+        # No %s specified, use the filename as-is
+        optfilename = options.options_file
+    
+    if options.verbose:
+        print 'Reading options from ' + optfilename
+    
+    if os.path.isfile(optfilename):
+        Globals.separate_options = read_options_file(open(optfilename, "rU"))
+    else:
+        Globals.separate_options = []
+    
+    # Parse the file
+    file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename]))
+    enums, messages, extensions = parse_file(fdesc, file_options)
+    
+    # Decide the file names
+    noext = os.path.splitext(filename)[0]
+    headername = noext + '.' + options.extension + '.h'
+    sourcename = noext + '.' + options.extension + '.c'
+    headerbasename = os.path.basename(headername)
+    
+    # List of .proto files that should not be included in the C header file
+    # even if they are mentioned in the source .proto.
+    excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude
+    dependencies = [d for d in fdesc.dependency if d not in excludes]
+    
+    headerdata = ''.join(generate_header(dependencies, headerbasename, enums,
+                                         messages, extensions, options))
+
+    sourcedata = ''.join(generate_source(headerbasename, enums,
+                                         messages, extensions, options))
+
+    return {'headername': headername, 'headerdata': headerdata,
+            'sourcename': sourcename, 'sourcedata': sourcedata}
+    
+def main_cli():
+    '''Main function when invoked directly from the command line.'''
+    
+    options, filenames = optparser.parse_args()
     
     if not filenames:
         optparser.print_help()
-        return False
+        sys.exit(1)
     
     if options.quiet:
         options.verbose = False
 
     Globals.verbose_options = options.verbose
     
-    toplevel_options = nanopb_pb2.NanoPBOptions()
-    for s in options.settings:
-        text_format.Merge(s, toplevel_options)
-    
     for filename in filenames:
-        data = open(filename, 'rb').read()
+        results = process_file(filename, None, options)
-        fdesc = descriptor.FileDescriptorSet.FromString(data)
-        
-        # Check if any separate options are specified
-        try:
-            optfilename = options.options_file % os.path.splitext(filename)[0]
-        except TypeError:
-            # No %s specified, use the filename as-is
-            optfilename = options.options_file
-        
-        if options.verbose:
-            print 'Reading options from ' + optfilename
-        
-        if os.path.isfile(optfilename):
-            Globals.separate_options = read_options_file(open(optfilename, "rU"))
-        else:
-            Globals.separate_options = []
-        
-        # Parse the file
-        file_options = get_nanopb_suboptions(fdesc.file[0], toplevel_options, Names([filename]))
-        enums, messages, extensions = parse_file(fdesc.file[0], file_options)
-        
-        noext = os.path.splitext(filename)[0]
-        headername = noext + '.' + options.extension + '.h'
-        sourcename = noext + '.' + options.extension + '.c'
-        headerbasename = os.path.basename(headername)
         
         if not options.quiet:
-            print "Writing to " + headername + " and " + sourcename
+            print "Writing to " + results['headername'] + " and " + results['sourcename']
-        
+    
-        # List of .proto files that should not be included in the C header file
+        open(results['headername'], 'w').write(results['headerdata'])
-        # even if they are mentioned in the source .proto.
+        open(results['sourcename'], 'w').write(results['sourcedata'])        
-        excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude
-        dependencies = [d for d in fdesc.file[0].dependency if d not in excludes]
-        
-        header = open(headername, 'w')
-        for part in generate_header(dependencies, headerbasename, enums,
-                                    messages, extensions, options):
-            header.write(part)
 
-        source = open(sourcename, 'w')
+def main_plugin():
-        for part in generate_source(headerbasename, enums, messages, extensions):
+    '''Main function when invoked as a protoc plugin.'''
-            source.write(part)
 
-    return True
+    import plugin_pb2
+    data = sys.stdin.read()
+    request = plugin_pb2.CodeGeneratorRequest.FromString(data)
+    
+    import shlex
+    args = shlex.split(request.parameter)
+    options, dummy = optparser.parse_args(args)
+    
+    # We can't go printing stuff to stdout
+    Globals.verbose_options = False
+    options.verbose = False
+    options.quiet = True
+    
+    response = plugin_pb2.CodeGeneratorResponse()
+    
+    for filename in request.file_to_generate:
+        for fdesc in request.proto_file:
+            if fdesc.name == filename:
+                results = process_file(filename, fdesc, options)
+                
+                f = response.file.add()
+                f.name = results['headername']
+                f.content = results['headerdata']
+
+                f = response.file.add()
+                f.name = results['sourcename']
+                f.content = results['sourcedata']    
+    
+    sys.stdout.write(response.SerializeToString())
 
 if __name__ == '__main__':
-    options, filenames = optparser.parse_args()
+    # Check if we are running as a plugin under protoc
-    status = process(filenames, options)
+    if 'protoc-gen-' in sys.argv[0]:
-    
+        main_plugin()
-    if not status:
+    else:
-        sys.exit(1)
+        main_cli()
-    
+

generator/plugin.proto

@@ -0,0 +1,145 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// http://code.google.com/p/protobuf/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//
+// WARNING:  The plugin interface is currently EXPERIMENTAL and is subject to
+//   change.
+//
+// protoc (aka the Protocol Compiler) can be extended via plugins.  A plugin is
+// just a program that reads a CodeGeneratorRequest from stdin and writes a
+// CodeGeneratorResponse to stdout.
+//
+// Plugins written using C++ can use google/protobuf/compiler/plugin.h instead
+// of dealing with the raw protocol defined here.
+//
+// A plugin executable needs only to be placed somewhere in the path.  The
+// plugin should be named "protoc-gen-$NAME", and will then be used when the
+// flag "--${NAME}_out" is passed to protoc.
+
+package google.protobuf.compiler;
+
+import "google/protobuf/descriptor.proto";
+
+// An encoded CodeGeneratorRequest is written to the plugin's stdin.
+message CodeGeneratorRequest {
+  // The .proto files that were explicitly listed on the command-line.  The
+  // code generator should generate code only for these files.  Each file's
+  // descriptor will be included in proto_file, below.
+  repeated string file_to_generate = 1;
+
+  // The generator parameter passed on the command-line.
+  optional string parameter = 2;
+
+  // FileDescriptorProtos for all files in files_to_generate and everything
+  // they import.  The files will appear in topological order, so each file
+  // appears before any file that imports it.
+  //
+  // protoc guarantees that all proto_files will be written after
+  // the fields above, even though this is not technically guaranteed by the
+  // protobuf wire format.  This theoretically could allow a plugin to stream
+  // in the FileDescriptorProtos and handle them one by one rather than read
+  // the entire set into memory at once.  However, as of this writing, this
+  // is not similarly optimized on protoc's end -- it will store all fields in
+  // memory at once before sending them to the plugin.
+  repeated FileDescriptorProto proto_file = 15;
+}
+
+// The plugin writes an encoded CodeGeneratorResponse to stdout.
+message CodeGeneratorResponse {
+  // Error message.  If non-empty, code generation failed.  The plugin process
+  // should exit with status code zero even if it reports an error in this way.
+  //
+  // This should be used to indicate errors in .proto files which prevent the
+  // code generator from generating correct code.  Errors which indicate a
+  // problem in protoc itself -- such as the input CodeGeneratorRequest being
+  // unparseable -- should be reported by writing a message to stderr and
+  // exiting with a non-zero status code.
+  optional string error = 1;
+
+  // Represents a single generated file.
+  message File {
+    // The file name, relative to the output directory.  The name must not
+    // contain "." or ".." components and must be relative, not be absolute (so,
+    // the file cannot lie outside the output directory).  "/" must be used as
+    // the path separator, not "\".
+    //
+    // If the name is omitted, the content will be appended to the previous
+    // file.  This allows the generator to break large files into small chunks,
+    // and allows the generated text to be streamed back to protoc so that large
+    // files need not reside completely in memory at one time.  Note that as of
+    // this writing protoc does not optimize for this -- it will read the entire
+    // CodeGeneratorResponse before writing files to disk.
+    optional string name = 1;
+
+    // If non-empty, indicates that the named file should already exist, and the
+    // content here is to be inserted into that file at a defined insertion
+    // point.  This feature allows a code generator to extend the output
+    // produced by another code generator.  The original generator may provide
+    // insertion points by placing special annotations in the file that look
+    // like:
+    //   @@protoc_insertion_point(NAME)
+    // The annotation can have arbitrary text before and after it on the line,
+    // which allows it to be placed in a comment.  NAME should be replaced with
+    // an identifier naming the point -- this is what other generators will use
+    // as the insertion_point.  Code inserted at this point will be placed
+    // immediately above the line containing the insertion point (thus multiple
+    // insertions to the same point will come out in the order they were added).
+    // The double-@ is intended to make it unlikely that the generated code
+    // could contain things that look like insertion points by accident.
+    //
+    // For example, the C++ code generator places the following line in the
+    // .pb.h files that it generates:
+    //   // @@protoc_insertion_point(namespace_scope)
+    // This line appears within the scope of the file's package namespace, but
+    // outside of any particular class.  Another plugin can then specify the
+    // insertion_point "namespace_scope" to generate additional classes or
+    // other declarations that should be placed in this scope.
+    //
+    // Note that if the line containing the insertion point begins with
+    // whitespace, the same whitespace will be added to every line of the
+    // inserted text.  This is useful for languages like Python, where
+    // indentation matters.  In these languages, the insertion point comment
+    // should be indented the same amount as any inserted code will need to be
+    // in order to work correctly in that context.
+    //
+    // The code generator that generates the initial file and the one which
+    // inserts into it must both run as part of a single invocation of protoc.
+    // Code generators are executed in the order in which they appear on the
+    // command line.
+    //
+    // If |insertion_point| is present, |name| must also be present.
+    optional string insertion_point = 2;
+
+    // The file contents.
+    optional string content = 15;
+  }
+  repeated File file = 15;
+}

generator/plugin_pb2.py

@@ -0,0 +1,161 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+
+from google.protobuf import descriptor
+from google.protobuf import message
+from google.protobuf import reflection
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+
+import google.protobuf.descriptor_pb2
+
+DESCRIPTOR = descriptor.FileDescriptor(
+  name='plugin.proto',
+  package='google.protobuf.compiler',
+  serialized_pb='\n\x0cplugin.proto\x12\x18google.protobuf.compiler\x1a google/protobuf/descriptor.proto\"}\n\x14\x43odeGeneratorRequest\x12\x18\n\x10\x66ile_to_generate\x18\x01 \x03(\t\x12\x11\n\tparameter\x18\x02 \x01(\t\x12\x38\n\nproto_file\x18\x0f \x03(\x0b\x32$.google.protobuf.FileDescriptorProto\"\xaa\x01\n\x15\x43odeGeneratorResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\x42\n\x04\x66ile\x18\x0f \x03(\x0b\x32\x34.google.protobuf.compiler.CodeGeneratorResponse.File\x1a>\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x17\n\x0finsertion_point\x18\x02 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x0f \x01(\t')
+
+
+
+
+_CODEGENERATORREQUEST = descriptor.Descriptor(
+  name='CodeGeneratorRequest',
+  full_name='google.protobuf.compiler.CodeGeneratorRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    descriptor.FieldDescriptor(
+      name='file_to_generate', full_name='google.protobuf.compiler.CodeGeneratorRequest.file_to_generate', index=0,
+      number=1, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    descriptor.FieldDescriptor(
+      name='parameter', full_name='google.protobuf.compiler.CodeGeneratorRequest.parameter', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=unicode("", "utf-8"),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    descriptor.FieldDescriptor(
+      name='proto_file', full_name='google.protobuf.compiler.CodeGeneratorRequest.proto_file', index=2,
+      number=15, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  serialized_start=76,
+  serialized_end=201,
+)
+
+
+_CODEGENERATORRESPONSE_FILE = descriptor.Descriptor(
+  name='File',
+  full_name='google.protobuf.compiler.CodeGeneratorResponse.File',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    descriptor.FieldDescriptor(
+      name='name', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=unicode("", "utf-8"),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    descriptor.FieldDescriptor(
+      name='insertion_point', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.insertion_point', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=unicode("", "utf-8"),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    descriptor.FieldDescriptor(
+      name='content', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.content', index=2,
+      number=15, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=unicode("", "utf-8"),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  serialized_start=312,
+  serialized_end=374,
+)
+
+_CODEGENERATORRESPONSE = descriptor.Descriptor(
+  name='CodeGeneratorResponse',
+  full_name='google.protobuf.compiler.CodeGeneratorResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    descriptor.FieldDescriptor(
+      name='error', full_name='google.protobuf.compiler.CodeGeneratorResponse.error', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=unicode("", "utf-8"),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    descriptor.FieldDescriptor(
+      name='file', full_name='google.protobuf.compiler.CodeGeneratorResponse.file', index=1,
+      number=15, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[_CODEGENERATORRESPONSE_FILE, ],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  serialized_start=204,
+  serialized_end=374,
+)
+
+_CODEGENERATORREQUEST.fields_by_name['proto_file'].message_type = google.protobuf.descriptor_pb2._FILEDESCRIPTORPROTO
+_CODEGENERATORRESPONSE_FILE.containing_type = _CODEGENERATORRESPONSE;
+_CODEGENERATORRESPONSE.fields_by_name['file'].message_type = _CODEGENERATORRESPONSE_FILE
+DESCRIPTOR.message_types_by_name['CodeGeneratorRequest'] = _CODEGENERATORREQUEST
+DESCRIPTOR.message_types_by_name['CodeGeneratorResponse'] = _CODEGENERATORRESPONSE
+
+class CodeGeneratorRequest(message.Message):
+  __metaclass__ = reflection.GeneratedProtocolMessageType
+  DESCRIPTOR = _CODEGENERATORREQUEST
+  
+  # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorRequest)
+
+class CodeGeneratorResponse(message.Message):
+  __metaclass__ = reflection.GeneratedProtocolMessageType
+  
+  class File(message.Message):
+    __metaclass__ = reflection.GeneratedProtocolMessageType
+    DESCRIPTOR = _CODEGENERATORRESPONSE_FILE
+    
+    # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse.File)
+  DESCRIPTOR = _CODEGENERATORRESPONSE
+  
+  # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse)
+
+# @@protoc_insertion_point(module_scope)

pb.h

@@ -43,7 +43,7 @@
 
 /* Version of the nanopb library. Just in case you want to check it in
  * your own program. */
-#define NANOPB_VERSION nanopb-0.2.2-dev
+#define NANOPB_VERSION nanopb-0.2.5-dev
 
 /* Include all the system headers needed by nanopb. You will need the
  * definitions of the following:
@@ -173,6 +173,20 @@ typedef uint8_t pb_type_t;
 #define PB_HTYPE(x) ((x) & PB_HTYPE_MASK)
 #define PB_LTYPE(x) ((x) & PB_LTYPE_MASK)
 
+/* Data type used for storing sizes of struct fields
+ * and array counts.
+ */
+#if defined(PB_FIELD_32BIT)
+    typedef uint32_t pb_size_t;
+    typedef int32_t pb_ssize_t;
+#elif defined(PB_FIELD_16BIT)
+    typedef uint16_t pb_size_t;
+    typedef int16_t pb_ssize_t;
+#else
+    typedef uint8_t pb_size_t;
+    typedef int8_t pb_ssize_t;
+#endif
+
 /* This structure is used in auto-generated constants
  * to specify struct fields.
  * You can change field sizes if you need structures
@@ -184,29 +198,12 @@ typedef uint8_t pb_type_t;
 PB_PACKED_STRUCT_START
 typedef struct _pb_field_t pb_field_t;
 struct _pb_field_t {
-
+    pb_size_t tag;
-#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
-    uint8_t tag;
     pb_type_t type;
-    uint8_t data_offset; /* Offset of field data, relative to previous field. */
+    pb_size_t data_offset; /* Offset of field data, relative to previous field. */
-    int8_t size_offset; /* Offset of array size or has-boolean, relative to data */
+    pb_ssize_t size_offset; /* Offset of array size or has-boolean, relative to data */
-    uint8_t data_size; /* Data size in bytes for a single item */
+    pb_size_t data_size; /* Data size in bytes for a single item */
-    uint8_t array_size; /* Maximum number of entries in array */
+    pb_size_t array_size; /* Maximum number of entries in array */
-#elif defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
-    uint16_t tag;
-    pb_type_t type;
-    uint8_t data_offset;
-    int8_t size_offset;
-    uint16_t data_size;
-    uint16_t array_size;
-#else
-    uint32_t tag;
-    pb_type_t type;
-    uint8_t data_offset;
-    int8_t size_offset;
-    uint32_t data_size;
-    uint32_t array_size;
-#endif
     
     /* Field definitions for submessage
      * OR default value for all other non-array, non-callback types
@@ -215,6 +212,17 @@ struct _pb_field_t {
 } pb_packed;
 PB_PACKED_STRUCT_END
 
+/* Make sure that the standard integer types are of the expected sizes.
+ * All kinds of things may break otherwise.. atleast all fixed* types. */
+STATIC_ASSERT(sizeof(int8_t) == 1, INT8_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(uint8_t) == 1, UINT8_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(int16_t) == 2, INT16_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(uint16_t) == 2, UINT16_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(int32_t) == 4, INT32_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(uint32_t) == 4, UINT32_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(int64_t) == 8, INT64_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(uint64_t) == 8, UINT64_T_WRONG_SIZE)
+
 /* This structure is used for 'bytes' arrays.
  * It has the number of bytes in the beginning, and after that an array.
  * Note that actual structs used will have a different length of bytes array.
@@ -320,58 +328,69 @@ struct _pb_extension_t {
 };
 
 /* These macros are used to declare pb_field_t's in the constant array. */
+/* Size of a structure member, in bytes. */
 #define pb_membersize(st, m) (sizeof ((st*)0)->m)
+/* Number of entries in an array. */
 #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0]))
+/* Delta from start of one member to the start of another member. */
 #define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2))
-#define pb_delta_end(st, m1, m2) (int)(offsetof(st, m1) == offsetof(st, m2) \
+/* Marks the end of the field list */
-                                  ? offsetof(st, m1) \
-                                  : offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2))
 #define PB_LAST_FIELD {0,(pb_type_t) 0,0,0,0,0,0}
 
+/* Macros for filling in the data_offset field */
+/* data_offset for first field in a message */
+#define PB_DATAOFFSET_FIRST(st, m1, m2) (offsetof(st, m1))
+/* data_offset for subsequent fields */
+#define PB_DATAOFFSET_OTHER(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2))
+/* Choose first/other based on m1 == m2 (deprecated, remains for backwards compatibility) */
+#define PB_DATAOFFSET_CHOOSE(st, m1, m2) (int)(offsetof(st, m1) == offsetof(st, m2) \
+                                  ? PB_DATAOFFSET_FIRST(st, m1, m2) \
+                                  : PB_DATAOFFSET_OTHER(st, m1, m2))
+
 /* Required fields are the simplest. They just have delta (padding) from
  * previous field end, and the size of the field. Pointer is used for
  * submessages and default values.
  */
-#define PB_REQUIRED_STATIC(tag, st, m, pm, ltype, ptr) \
+#define PB_REQUIRED_STATIC(tag, st, m, fd, ltype, ptr) \
     {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | ltype, \
-    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+    fd, 0, pb_membersize(st, m), 0, ptr}
 
 /* Optional fields add the delta to the has_ variable. */
-#define PB_OPTIONAL_STATIC(tag, st, m, pm, ltype, ptr) \
+#define PB_OPTIONAL_STATIC(tag, st, m, fd, ltype, ptr) \
     {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \
-    pb_delta_end(st, m, pm), \
+    fd, \
     pb_delta(st, has_ ## m, m), \
     pb_membersize(st, m), 0, ptr}
 
 /* Repeated fields have a _count field and also the maximum number of entries. */
-#define PB_REPEATED_STATIC(tag, st, m, pm, ltype, ptr) \
+#define PB_REPEATED_STATIC(tag, st, m, fd, ltype, ptr) \
     {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | ltype, \
-    pb_delta_end(st, m, pm), \
+    fd, \
     pb_delta(st, m ## _count, m), \
     pb_membersize(st, m[0]), \
     pb_arraysize(st, m), ptr}
 
 /* Callbacks are much like required fields except with special datatype. */
-#define PB_REQUIRED_CALLBACK(tag, st, m, pm, ltype, ptr) \
+#define PB_REQUIRED_CALLBACK(tag, st, m, fd, ltype, ptr) \
     {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REQUIRED | ltype, \
-    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+    fd, 0, pb_membersize(st, m), 0, ptr}
 
-#define PB_OPTIONAL_CALLBACK(tag, st, m, pm, ltype, ptr) \
+#define PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) \
     {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \
-    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+    fd, 0, pb_membersize(st, m), 0, ptr}
     
-#define PB_REPEATED_CALLBACK(tag, st, m, pm, ltype, ptr) \
+#define PB_REPEATED_CALLBACK(tag, st, m, fd, ltype, ptr) \
     {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REPEATED | ltype, \
-    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+    fd, 0, pb_membersize(st, m), 0, ptr}
 
 /* Optional extensions don't have the has_ field, as that would be redundant. */
-#define PB_OPTEXT_STATIC(tag, st, m, pm, ltype, ptr) \
+#define PB_OPTEXT_STATIC(tag, st, m, fd, ltype, ptr) \
     {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \
     0, \
     0, \
     pb_membersize(st, m), 0, ptr}
 
-#define PB_OPTEXT_CALLBACK(tag, st, m, pm, ltype, ptr) \
+#define PB_OPTEXT_CALLBACK(tag, st, m, fd, ltype, ptr) \
     {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \
     0, 0, pb_membersize(st, m), 0, ptr}
 
@@ -410,8 +429,21 @@ struct _pb_extension_t {
  */
 
 #define PB_FIELD(tag, type, rules, allocation, message, field, prevfield, ptr) \
-    PB_ ## rules ## _ ## allocation(tag, message, field, prevfield, \
+    PB_ ## rules ## _ ## allocation(tag, message, field, \
-                                    PB_LTYPE_MAP_ ## type, ptr)
+        PB_DATAOFFSET_CHOOSE(message, field, prevfield), \
+        PB_LTYPE_MAP_ ## type, ptr)
+
+/* This is a new version of the macro used by nanopb generator from
+ * version 0.2.3 onwards. It avoids the use of a ternary expression in
+ * the initialization, which confused some compilers.
+ *
+ * - Placement: FIRST or OTHER, depending on if this is the first field in structure.
+ *
+ */
+#define PB_FIELD2(tag, type, rules, allocation, placement, message, field, prevfield, ptr) \
+    PB_ ## rules ## _ ## allocation(tag, message, field, \
+        PB_DATAOFFSET_ ## placement(message, field, prevfield), \
+        PB_LTYPE_MAP_ ## type, ptr)
 
 
 /* These macros are used for giving out error messages.

pb_decode.c

@@ -3,11 +3,13 @@
  * 2011 Petteri Aimonen <jpa@kapsi.fi>
  */
 
-/* The warn_unused_result attribute appeared first in gcc-3.4.0 */
+/* Use the GCC warn_unused_result attribute to check that all return values
+ * are propagated correctly. On other compilers and gcc before 3.4.0 just
+ * ignore the annotation.
+ */
 #if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4)
     #define checkreturn
 #else
-    /* Verify that we remember to check all return values for proper error propagation */
     #define checkreturn __attribute__((warn_unused_result))
 #endif
 
@@ -15,26 +17,49 @@
 #include "pb.h"
 #include "pb_decode.h"
 
-typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn;
+/**************************************
+ * Declarations internal to this file *
+ **************************************/
 
-/* --- Function pointers to field decoders ---
+/* Iterator for pb_field_t list */
- * Order in the array must match pb_action_t LTYPE numbering.
+typedef struct {
- */
+    const pb_field_t *start; /* Start of the pb_field_t array */
-static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = {
+    const pb_field_t *pos; /* Current position of the iterator */
-    &pb_dec_varint,
+    unsigned field_index; /* Zero-based index of the field. */
-    &pb_dec_svarint,
+    unsigned required_field_index; /* Zero-based index that counts only the required fields */
-    &pb_dec_fixed32,
+    void *dest_struct; /* Pointer to the destination structure to decode to */
-    &pb_dec_fixed64,
+    void *pData; /* Pointer where to store current field value */
-    
+    void *pSize; /* Pointer where to store the size of current array field */
-    &pb_dec_bytes,
+} pb_field_iterator_t;
-    &pb_dec_string,
-    &pb_dec_submessage,
-    NULL /* extensions */
-};
 
-/**************
+
- * pb_istream *
+static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count);
- **************/
+static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest);
+static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8_t *buf, size_t *size);
+static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct);
+static bool pb_field_next(pb_field_iterator_t *iter);
+static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag);
+static bool checkreturn decode_ltype(pb_istream_t *stream, const pb_field_t *field, void *dest, pb_type_t type);
+static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
+static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
+static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
+static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);
+static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
+static bool checkreturn find_extension_field(pb_field_iterator_t *iter);
+static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct);
+static bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_skip_varint(pb_istream_t *stream);
+static bool checkreturn pb_skip_string(pb_istream_t *stream);
+
+/*******************************
+ * pb_istream_t implementation *
+ *******************************/
 
 static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count)
 {
@@ -84,6 +109,26 @@ bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count)
     return true;
 }
 
+/* Read a single byte from input stream. buf may not be NULL.
+ * This is an optimization for the varint decoding. */
+static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf)
+{
+    if (!stream->bytes_left)
+        PB_RETURN_ERROR(stream, "end-of-stream");
+
+#ifndef PB_BUFFER_ONLY
+    if (!stream->callback(stream, buf, 1))
+        PB_RETURN_ERROR(stream, "io error");
+#else
+    *buf = *(uint8_t*)stream->state;
+    stream->state = (uint8_t*)stream->state + 1;
+#endif
+
+    stream->bytes_left--;
+    
+    return true;    
+}
+
 pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize)
 {
     pb_istream_t stream;
@@ -109,7 +154,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest)
     uint8_t byte;
     uint32_t result;
     
-    if (!pb_read(stream, &byte, 1))
+    if (!pb_readbyte(stream, &byte))
         return false;
     
     if (!(byte & 0x80))
@@ -128,7 +173,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest)
             if (bitpos >= 32)
                 PB_RETURN_ERROR(stream, "varint overflow");
             
-            if (!pb_read(stream, &byte, 1))
+            if (!pb_readbyte(stream, &byte))
                 return false;
             
             result |= (uint32_t)(byte & 0x7F) << bitpos;
@@ -151,7 +196,7 @@ bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest)
         if (bitpos >= 64)
             PB_RETURN_ERROR(stream, "varint overflow");
         
-        if (!pb_read(stream, &byte, 1))
+        if (!pb_readbyte(stream, &byte))
             return false;
 
         result |= (uint64_t)(byte & 0x7F) << bitpos;
@@ -253,41 +298,25 @@ static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire
 /* Decode string length from stream and return a substream with limited length.
  * Remember to close the substream using pb_close_string_substream().
  */
-bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream)
+bool checkreturn pb_make_string_substream(pb_istream_t *stream, size_t *remaining_length)
 {
     uint32_t size;
     if (!pb_decode_varint32(stream, &size))
         return false;
-    
+
-    *substream = *stream;
+    if (stream->bytes_left < size)
-    if (substream->bytes_left < size)
         PB_RETURN_ERROR(stream, "parent stream too short");
     
-    substream->bytes_left = size;
+    *remaining_length = stream->bytes_left - size;
-    stream->bytes_left -= size;
+    stream->bytes_left = size;
     return true;
 }
 
-void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream)
+void pb_close_string_substream(pb_istream_t *stream, size_t remaining_length)
 {
-    stream->state = substream->state;
+    stream->bytes_left += remaining_length;
-
-#ifndef PB_NO_ERRMSG
-    stream->errmsg = substream->errmsg;
-#endif
 }
 
-/* Iterator for pb_field_t list */
-typedef struct {
-    const pb_field_t *start; /* Start of the pb_field_t array */
-    const pb_field_t *pos; /* Current position of the iterator */
-    unsigned field_index; /* Zero-based index of the field. */
-    unsigned required_field_index; /* Zero-based index that counts only the required fields */
-    void *dest_struct; /* Pointer to the destination structure to decode to */
-    void *pData; /* Pointer where to store current field value */
-    void *pSize; /* Pointer where to store the size of current array field */
-} pb_field_iterator_t;
-
 static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct)
 {
     iter->start = iter->pos = fields;
@@ -352,22 +381,37 @@ static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag)
  * Decode a single field *
  *************************/
 
+/* Invoke a decoder function based on the ltype of a field. */
+static bool checkreturn decode_ltype(pb_istream_t *stream,
+    const pb_field_t *field, void *dest, pb_type_t type)
+{
+    switch (PB_LTYPE(type))
+    {
+        case PB_LTYPE_VARINT:       return pb_dec_varint(stream, field, dest);
+        case PB_LTYPE_SVARINT:      return pb_dec_svarint(stream, field, dest);
+        case PB_LTYPE_FIXED32:      return pb_dec_fixed32(stream, field, dest);
+        case PB_LTYPE_FIXED64:      return pb_dec_fixed64(stream, field, dest);
+        case PB_LTYPE_BYTES:        return pb_dec_bytes(stream, field, dest);
+        case PB_LTYPE_STRING:       return pb_dec_string(stream, field, dest);
+        case PB_LTYPE_SUBMESSAGE:   return pb_dec_submessage(stream, field, dest);
+        
+        default: PB_RETURN_ERROR(stream, "invalid field type");
+    }
+}
+
 static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
 {
     pb_type_t type;
-    pb_decoder_t func;
-    
     type = iter->pos->type;
-    func = PB_DECODERS[PB_LTYPE(type)];
 
     switch (PB_HTYPE(type))
     {
         case PB_HTYPE_REQUIRED:
-            return func(stream, iter->pos, iter->pData);
+            return decode_ltype(stream, iter->pos, iter->pData, type);
             
         case PB_HTYPE_OPTIONAL:
             *(bool*)iter->pSize = true;
-            return func(stream, iter->pos, iter->pData);
+            return decode_ltype(stream, iter->pos, iter->pData, type);
     
         case PB_HTYPE_REPEATED:
             if (wire_type == PB_WT_STRING
@@ -376,24 +420,25 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t
                 /* Packed array */
                 bool status = true;
                 size_t *size = (size_t*)iter->pSize;
-                pb_istream_t substream;
+                size_t remaining_length;
-                if (!pb_make_string_substream(stream, &substream))
+                if (!pb_make_string_substream(stream, &remaining_length))
                     return false;
                 
-                while (substream.bytes_left && *size < iter->pos->array_size)
+                while (stream->bytes_left)
                 {
                     void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size);
-                    if (!func(&substream, iter->pos, pItem))
+                    
+                    if (*size >= iter->pos->array_size)
+                        PB_RETURN_ERROR(stream, "array overflow");
+                
+                    if (!decode_ltype(stream, iter->pos, pItem, type))
                     {
                         status = false;
                         break;
                     }
                     (*size)++;
                 }
-                pb_close_string_substream(stream, &substream);
+                pb_close_string_substream(stream, remaining_length);
-                
-                if (substream.bytes_left != 0)
-                    PB_RETURN_ERROR(stream, "array overflow");
                 
                 return status;
             }
@@ -406,7 +451,7 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t
                     PB_RETURN_ERROR(stream, "array overflow");
                 
                 (*size)++;
-                return func(stream, iter->pos, pItem);
+                return decode_ltype(stream, iter->pos, pItem, type);
             }
 
         default:
@@ -429,18 +474,18 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type
     
     if (wire_type == PB_WT_STRING)
     {
-        pb_istream_t substream;
+        size_t remaining_length;
         
-        if (!pb_make_string_substream(stream, &substream))
+        if (!pb_make_string_substream(stream, &remaining_length))
             return false;
         
         do
         {
-            if (!pCallback->funcs.decode(&substream, iter->pos, arg))
+            if (!pCallback->funcs.decode(stream, iter->pos, arg))
                 PB_RETURN_ERROR(stream, "callback failed");
-        } while (substream.bytes_left);
+        } while (stream->bytes_left);
         
-        pb_close_string_substream(stream, &substream);
+        pb_close_string_substream(stream, remaining_length);
         return true;
     }
     else
@@ -478,7 +523,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
 
 /* Default handler for extension fields. Expects a pb_field_t structure
  * in extension->type->arg. */
-static bool checkreturn default_extension_handler(pb_istream_t *stream,
+static bool checkreturn default_extension_decoder(pb_istream_t *stream,
     pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type)
 {
     const pb_field_t *field = (const pb_field_t*)extension->type->arg;
@@ -513,7 +558,7 @@ static bool checkreturn decode_extension(pb_istream_t *stream,
         if (extension->type->decode)
             status = extension->type->decode(stream, extension, tag, wire_type);
         else
-            status = default_extension_handler(stream, extension, tag, wire_type);
+            status = default_extension_decoder(stream, extension, tag, wire_type);
 
         if (!status)
             return false;
@@ -697,14 +742,14 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void
 
 bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct)
 {
-    pb_istream_t substream;
+    size_t remaining_length;
     bool status;
     
-    if (!pb_make_string_substream(stream, &substream))
+    if (!pb_make_string_substream(stream, &remaining_length))
         return false;
     
-    status = pb_decode(&substream, fields, dest_struct);
+    status = pb_decode(stream, fields, dest_struct);
-    pb_close_string_substream(stream, &substream);
+    pb_close_string_substream(stream, remaining_length);
     return status;
 }
 
@@ -847,10 +892,10 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo
 bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest)
 {
     bool status;
-    pb_istream_t substream;
+    size_t remaining_length;
     const pb_field_t* submsg_fields = (const pb_field_t*)field->ptr;
     
-    if (!pb_make_string_substream(stream, &substream))
+    if (!pb_make_string_substream(stream, &remaining_length))
         return false;
     
     if (field->ptr == NULL)
@@ -859,10 +904,10 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field
     /* New array entries need to be initialized, while required and optional
      * submessages have already been initialized in the top-level pb_decode. */
     if (PB_HTYPE(field->type) == PB_HTYPE_REPEATED)
-        status = pb_decode(&substream, submsg_fields, dest);
+        status = pb_decode(stream, submsg_fields, dest);
     else
-        status = pb_decode_noinit(&substream, submsg_fields, dest);
+        status = pb_decode_noinit(stream, submsg_fields, dest);
     
-    pb_close_string_substream(stream, &substream);
+    pb_close_string_substream(stream, remaining_length);
     return status;
 }

pb_decode.h

@@ -12,6 +12,39 @@
 extern "C" {
 #endif
 
+/* Structure for defining custom input streams. You will need to provide
+ * a callback function to read the bytes from your storage, which can be
+ * for example a file or a network socket.
+ * 
+ * The callback must conform to these rules:
+ *
+ * 1) Return false on IO errors. This will cause decoding to abort.
+ * 2) You can use state to store your own data (e.g. buffer pointer),
+ *    and rely on pb_read to verify that no-body reads past bytes_left.
+ * 3) Your callback may be used with substreams, in which case bytes_left
+ *    is different than from the main stream. Don't use bytes_left to compute
+ *    any pointers.
+ */
+struct _pb_istream_t
+{
+#ifdef PB_BUFFER_ONLY
+    /* Callback pointer is not used in buffer-only configuration.
+     * Having an int pointer here allows binary compatibility but
+     * gives an error if someone tries to assign callback function.
+     */
+    int *callback;
+#else
+    bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count);
+#endif
+
+    void *state; /* Free field for use by callback implementation */
+    size_t bytes_left;
+    
+#ifndef PB_NO_ERRMSG
+    const char *errmsg;
+#endif
+};
+
 /***************************
  * Main decoding functions *
  ***************************/
@@ -66,39 +99,6 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize);
  */
 bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count);
 
-/* Structure for defining custom input streams. You will need to provide
- * a callback function to read the bytes from your storage, which can be
- * for example a file or a network socket.
- * 
- * The callback must conform to these rules:
- *
- * 1) Return false on IO errors. This will cause decoding to abort.
- * 2) You can use state to store your own data (e.g. buffer pointer),
- *    and rely on pb_read to verify that no-body reads past bytes_left.
- * 3) Your callback may be used with substreams, in which case bytes_left
- *    is different than from the main stream. Don't use bytes_left to compute
- *    any pointers.
- */
-struct _pb_istream_t
-{
-#ifdef PB_BUFFER_ONLY
-    /* Callback pointer is not used in buffer-only configuration.
-     * Having an int pointer here allows binary compatibility but
-     * gives an error if someone tries to assign callback function.
-     */
-    int *callback;
-#else
-    bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count);
-#endif
-
-    void *state; /* Free field for use by callback implementation */
-    size_t bytes_left;
-    
-#ifndef PB_NO_ERRMSG
-    const char *errmsg;
-#endif
-};
-
 
 /************************************************
  * Helper functions for writing field callbacks *
@@ -128,27 +128,8 @@ bool pb_decode_fixed32(pb_istream_t *stream, void *dest);
 bool pb_decode_fixed64(pb_istream_t *stream, void *dest);
 
 /* Make a limited-length substream for reading a PB_WT_STRING field. */
-bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream);
+bool pb_make_string_substream(pb_istream_t *stream, size_t *remaining_length);
-void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream);
+void pb_close_string_substream(pb_istream_t *stream, size_t remaining_length);
-
-
-/*******************************
- * Internal / legacy functions *
- *******************************/
-
-#ifdef NANOPB_INTERNALS
-bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest);
-bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest);
-bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest);
-bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest);
-
-bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest);
-bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest);
-bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest);
-
-bool pb_skip_varint(pb_istream_t *stream);
-bool pb_skip_string(pb_istream_t *stream);
-#endif
 
 #ifdef __cplusplus
 } /* extern "C" */

pb_encode.c

@@ -7,32 +7,36 @@
 #include "pb.h"
 #include "pb_encode.h"
 
-/* The warn_unused_result attribute appeared first in gcc-3.4.0 */
+/* Use the GCC warn_unused_result attribute to check that all return values
+ * are propagated correctly. On other compilers and gcc before 3.4.0 just
+ * ignore the annotation.
+ */
 #if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4)
     #define checkreturn
 #else
-    /* Verify that we remember to check all return values for proper error propagation */
     #define checkreturn __attribute__((warn_unused_result))
 #endif
 
-typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src) checkreturn;
+/**************************************
+ * Declarations internal to this file *
+ **************************************/
+static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count);
+static bool checkreturn encode_ltype(pb_ostream_t *stream, const pb_field_t *field, const void *src, pb_type_t type);
+static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, const void *pData, size_t count);
+static bool checkreturn encode_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData);
+static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension);
+static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData);
+static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src);
 
-/* --- Function pointers to field encoders ---
+/*******************************
- * Order in the array must match pb_action_t LTYPE numbering.
+ * pb_ostream_t implementation *
- */
+ *******************************/
-static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = {
-    &pb_enc_varint,
-    &pb_enc_svarint,
-    &pb_enc_fixed32,
-    &pb_enc_fixed64,
-    
-    &pb_enc_bytes,
-    &pb_enc_string,
-    &pb_enc_submessage,
-    NULL /* extensions */
-};
-
-/* pb_ostream_t implementation */
 
 static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count)
 {
@@ -49,7 +53,7 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize)
 {
     pb_ostream_t stream;
 #ifdef PB_BUFFER_ONLY
-    stream.callback = (void*)1; /* Just some marker value */
+    stream.callback = (void*)1; /* Just a marker value */
 #else
     stream.callback = &buf_write;
 #endif
@@ -82,11 +86,32 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count
     return true;
 }
 
-/* Main encoding stuff */
+/*************************
+ * Encode a single field *
+ *************************/
+
+/* Invoke an encoder function based on the ltype of a field. */
+static bool checkreturn encode_ltype(pb_ostream_t *stream,
+    const pb_field_t *field, const void *src, pb_type_t type)
+{
+    switch (PB_LTYPE(type))
+    {
+        case PB_LTYPE_VARINT:       return pb_enc_varint(stream, field, src);
+        case PB_LTYPE_SVARINT:      return pb_enc_svarint(stream, field, src);
+        case PB_LTYPE_FIXED32:      return pb_enc_fixed32(stream, field, src);
+        case PB_LTYPE_FIXED64:      return pb_enc_fixed64(stream, field, src);
+        case PB_LTYPE_BYTES:        return pb_enc_bytes(stream, field, src);
+        case PB_LTYPE_STRING:       return pb_enc_string(stream, field, src);
+        case PB_LTYPE_SUBMESSAGE:   return pb_enc_submessage(stream, field, src);
+        
+        default: PB_RETURN_ERROR(stream, "invalid field type");
+    }
+}
+    
 
 /* Encode a static array. Handles the size calculations and possible packing. */
 static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field,
-                         const void *pData, size_t count, pb_encoder_t func)
+                         const void *pData, size_t count)
 {
     size_t i;
     const void *p;
@@ -94,6 +119,9 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
     
     if (count == 0)
         return true;
+        
+    if (count > field->array_size)
+        PB_RETURN_ERROR(stream, "array max size exceeded");
     
     /* We always pack arrays if the datatype allows it. */
     if (PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE)
@@ -116,7 +144,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
             p = pData;
             for (i = 0; i < count; i++)
             {
-                if (!func(&sizestream, field, p))
+                if (!encode_ltype(&sizestream, field, p, field->type))
                     return false;
                 p = (const char*)p + field->data_size;
             }
@@ -133,7 +161,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
         p = pData;
         for (i = 0; i < count; i++)
         {
-            if (!func(stream, field, p))
+            if (!encode_ltype(stream, field, p, field->type))
                 return false;
             p = (const char*)p + field->data_size;
         }
@@ -145,7 +173,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
         {
             if (!pb_encode_tag_for_field(stream, field))
                 return false;
-            if (!func(stream, field, p))
+            if (!encode_ltype(stream, field, p, field->type))
                 return false;
             p = (const char*)p + field->data_size;
         }
@@ -159,12 +187,9 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
 static bool checkreturn encode_static_field(pb_ostream_t *stream,
     const pb_field_t *field, const void *pData)
 {
-    pb_encoder_t func;
     const void *pSize;
     bool dummy = true;
     
-    func = PB_ENCODERS[PB_LTYPE(field->type)];
-    
     if (field->size_offset)
         pSize = (const char*)pData + field->size_offset;
     else
@@ -175,7 +200,7 @@ static bool checkreturn encode_static_field(pb_ostream_t *stream,
         case PB_HTYPE_REQUIRED:
             if (!pb_encode_tag_for_field(stream, field))
                 return false;
-            if (!func(stream, field, pData))
+            if (!encode_ltype(stream, field, pData, field->type))
                 return false;
             break;
         
@@ -185,13 +210,13 @@ static bool checkreturn encode_static_field(pb_ostream_t *stream,
                 if (!pb_encode_tag_for_field(stream, field))
                     return false;
             
-                if (!func(stream, field, pData))
+                if (!encode_ltype(stream, field, pData, field->type))
                     return false;
             }
             break;
         
         case PB_HTYPE_REPEATED:
-            if (!encode_array(stream, field, pData, *(const size_t*)pSize, func))
+            if (!encode_array(stream, field, pData, *(const size_t*)pSize))
                 return false;
             break;
         
@@ -242,7 +267,7 @@ static bool checkreturn encode_field(pb_ostream_t *stream,
 
 /* Default handler for extension fields. Expects to have a pb_field_t
  * pointer in the extension->type->arg field. */
-static bool checkreturn default_extension_handler(pb_ostream_t *stream,
+static bool checkreturn default_extension_encoder(pb_ostream_t *stream,
     const pb_extension_t *extension)
 {
     const pb_field_t *field = (const pb_field_t*)extension->type->arg;
@@ -263,7 +288,7 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream,
         if (extension->type->encode)
             status = extension->type->encode(stream, extension);
         else
-            status = default_extension_handler(stream, extension);
+            status = default_extension_encoder(stream, extension);
 
         if (!status)
             return false;
@@ -274,6 +299,10 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream,
     return true;
 }
 
+/*********************
+ * Encode all fields *
+ *********************/
+
 bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct)
 {
     const pb_field_t *field = fields;
@@ -316,7 +345,9 @@ bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const
     return pb_encode_submessage(stream, fields, src_struct);
 }
 
-/* Helper functions */
+/********************
+ * Helper functions *
+ ********************/
 bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value)
 {
     uint8_t buffer[10];
@@ -518,7 +549,10 @@ bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, c
 bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
     const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src;
-    UNUSED(field);
+
+    if (bytes->size + offsetof(pb_bytes_array_t, bytes) > field->data_size)
+        PB_RETURN_ERROR(stream, "bytes size exceeded");
+    
     return pb_encode_string(stream, bytes->bytes, bytes->size);
 }
 

pb_encode.h

@@ -12,6 +12,40 @@
 extern "C" {
 #endif
 
+/* Structure for defining custom output streams. You will need to provide
+ * a callback function to write the bytes to your storage, which can be
+ * for example a file or a network socket.
+ *
+ * The callback must conform to these rules:
+ *
+ * 1) Return false on IO errors. This will cause encoding to abort.
+ * 2) You can use state to store your own data (e.g. buffer pointer).
+ * 3) pb_write will update bytes_written after your callback runs.
+ * 4) Substreams will modify max_size and bytes_written. Don't use them
+ *    to calculate any pointers.
+ */
+struct _pb_ostream_t
+{
+#ifdef PB_BUFFER_ONLY
+    /* Callback pointer is not used in buffer-only configuration.
+     * Having an int pointer here allows binary compatibility but
+     * gives an error if someone tries to assign callback function.
+     * Also, NULL pointer marks a 'sizing stream' that does not
+     * write anything.
+     */
+    int *callback;
+#else
+    bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count);
+#endif
+    void *state;          /* Free field for use by callback implementation. */
+    size_t max_size;      /* Limit number of output bytes written (or use SIZE_MAX). */
+    size_t bytes_written; /* Number of bytes written so far. */
+    
+#ifndef PB_NO_ERRMSG
+    const char *errmsg;
+#endif
+};
+
 /***************************
  * Main encoding functions *
  ***************************/
@@ -70,40 +104,6 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize);
  */
 bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count);
 
-/* Structure for defining custom output streams. You will need to provide
- * a callback function to write the bytes to your storage, which can be
- * for example a file or a network socket.
- *
- * The callback must conform to these rules:
- *
- * 1) Return false on IO errors. This will cause encoding to abort.
- * 2) You can use state to store your own data (e.g. buffer pointer).
- * 3) pb_write will update bytes_written after your callback runs.
- * 4) Substreams will modify max_size and bytes_written. Don't use them
- *    to calculate any pointers.
- */
-struct _pb_ostream_t
-{
-#ifdef PB_BUFFER_ONLY
-    /* Callback pointer is not used in buffer-only configuration.
-     * Having an int pointer here allows binary compatibility but
-     * gives an error if someone tries to assign callback function.
-     * Also, NULL pointer marks a 'sizing stream' that does not
-     * write anything.
-     */
-    int *callback;
-#else
-    bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count);
-#endif
-    void *state;          /* Free field for use by callback implementation. */
-    size_t max_size;      /* Limit number of output bytes written (or use SIZE_MAX). */
-    size_t bytes_written; /* Number of bytes written so far. */
-    
-#ifndef PB_NO_ERRMSG
-    const char *errmsg;
-#endif
-};
-
 
 /************************************************
  * Helper functions for writing field callbacks *
@@ -143,24 +143,6 @@ bool pb_encode_fixed64(pb_ostream_t *stream, const void *value);
  */
 bool pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct);
 
-
-/*******************************
- * Internal / legacy functions *
- *******************************/
-
-#ifdef NANOPB_INTERNALS
-bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-bool pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-bool pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-#endif
-
-/* This function is not recommended for new programs. Use pb_encode_submessage()
- * instead, it has the same functionality with a less confusing interface. */
-bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-
 #ifdef __cplusplus
 } /* extern "C" */
 #endif

tests/Makefile

@@ -1,143 +1,6 @@
-CFLAGS=-ansi -Wall -Werror -I .. -g -O0
+all:
-DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h \
+	scons
-	callbacks2.pb.h callbacks.pb.h unittests.h unittestproto.pb.h \
-	alltypes.pb.h missing_fields.pb.h
-TESTS=  decode_unittests encode_unittests \
-	test_decode1 test_decode2 test_decode3 test_decode3_buf \
-	test_encode1 test_encode2 test_encode3 test_encode3_buf \
-	test_decode_callbacks test_encode_callbacks \
-	test_missing_fields test_no_messages test_funny_name \
-	test_multiple_files test_cxxcompile test_options \
-	bc_encode bc_decode test_encode_extensions test_decode_extensions
-
-# More strict checks for the core part of nanopb
-CC_VERSION=$(shell $(CC) -v 2>&1)
-CFLAGS_CORE=
-ifneq (,$(findstring gcc,$(CC_VERSION)))
-	CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op -Wconversion
-	CFLAGS+=--coverage -fstack-protector-all
-	LDFLAGS+=--coverage
-endif
-ifneq (,$(findstring clang,$(CC_VERSION)))
-	CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wconversion
-endif
-
-# Also use mudflap if it is available
-# To enable, run with    make -B USE_MUDFLAP=y
-USE_MUDFLAP ?= n
-ifeq ($(USE_MUDFLAP),y)
-	CFLAGS += -fmudflap
-	LDFLAGS += -lmudflap -fmudflap
-endif
-
-all: breakpoints $(TESTS) run_unittests
 
 clean:
-	rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno *.pb.h *.pb.c
+	scons -c
 
-%.pb.o: %.pb.c %.pb.h
-	$(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-
-%.o: %.c
-%.o: %.c $(DEPS)
-	$(CC) $(CFLAGS) -c -o $@ $<
-
-pb_encode.o: ../pb_encode.c $(DEPS)
-	$(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-pb_decode.o: ../pb_decode.c $(DEPS)
-	$(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-
-# Test for compilability with c++ compiler
-
-pb_encode.cxx.o: ../pb_encode.c $(DEPS)
-	$(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-pb_decode.cxx.o: ../pb_decode.c $(DEPS)
-	$(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-
-# Test for PB_BUF_ONLY compilation option
-
-pb_encode.buf.o: ../pb_encode.c $(DEPS)
-	$(CC) -DPB_BUFFER_ONLY $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-pb_decode.buf.o: ../pb_decode.c $(DEPS)
-	$(CC) -DPB_BUFFER_ONLY $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-%.buf.o: %.c $(DEPS)
-	$(CC) -DPB_BUFFER_ONLY $(CFLAGS) -c -o $@ $<	
-test_encode3_buf: test_encode3.buf.o pb_encode.buf.o alltypes.pb.o
-	$(CC) $(LDFLAGS) $^ -o $@
-test_decode3_buf: test_decode3.buf.o pb_decode.buf.o alltypes.pb.o
-	$(CC) $(LDFLAGS) $^ -o $@
-
-test_cxxcompile: pb_encode.cxx.o pb_decode.cxx.o
-test_decode1: test_decode1.o pb_decode.o person.pb.o
-test_decode2: test_decode2.o pb_decode.o person.pb.o
-test_decode3: test_decode3.o pb_decode.o alltypes.pb.o
-test_encode1: test_encode1.o pb_encode.o person.pb.o
-test_encode2: test_encode2.o pb_encode.o person.pb.o
-test_encode3: test_encode3.o pb_encode.o alltypes.pb.o
-test_multiple_files: test_multiple_files.o pb_encode.o callbacks2.pb.o callbacks.pb.o
-test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o
-test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o
-test_missing_fields: test_missing_fields.o pb_encode.o pb_decode.o missing_fields.pb.o
-decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o
-encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o
-test_no_messages: no_messages.pb.h no_messages.pb.c no_messages.pb.o
-test_funny_name: funny-proto+name.pb.h funny-proto+name.pb.o
-bc_encode: bc_alltypes.pb.o pb_encode.o bc_encode.o
-bc_decode: bc_alltypes.pb.o pb_decode.o bc_decode.o
-test_encode_extensions: test_encode_extensions.c pb_encode.o alltypes.pb.o extensions.pb.o
-test_decode_extensions: test_decode_extensions.c pb_decode.o alltypes.pb.o extensions.pb.o
-
-%.pb: %.proto
-	protoc -I. -I../generator -I/usr/include -o$@ $<
-
-%.pb.c %.pb.h: %.pb ../generator/nanopb_generator.py
-	python ../generator/nanopb_generator.py $<
-
-breakpoints: ../*.c *.c
-	grep -n 'return false;' $^ | cut -d: -f-2 | xargs -n 1 echo b > $@
-
-coverage: run_unittests
-	gcov pb_encode.gcda
-	gcov pb_decode.gcda
-
-run_unittests: $(TESTS)
-	rm -f *.gcda
-	
-	./decode_unittests > /dev/null
-	./encode_unittests > /dev/null
-	
-	[ "`./test_encode1 | ./test_decode1`" = \
-	"`./test_encode1 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ]
-
-	[ "`./test_encode2 | ./test_decode1`" = \
-	"`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ]
-
-	[ "`./test_encode2 | ./test_decode2`" = \
-	"`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ]
-	
-	[ "`./test_decode2 < person_with_extra_field.pb`" = \
-	"`./test_encode2 | ./test_decode2`" ]
-	
-	[ "`./test_encode_callbacks | ./test_decode_callbacks`" = \
-	"`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ]
-
-	./test_encode3 | ./test_decode3
-	./test_encode3 1 | ./test_decode3 1
-	./test_encode3 1 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null
-	./test_encode3_buf 1 | ./test_decode3_buf 1
-	./test_decode3 < alltypes_with_extra_fields.pb
-	./bc_encode | ./bc_decode
-	./test_encode_extensions | ./test_decode_extensions
-
-	./test_missing_fields
-
-test_options: options.pb.h options.expected options.pb.o
-	cat options.expected | while read -r p; do \
-	    if ! grep -q "$$p" $<; then \
-	        echo Expected: "$$p"; \
-	        exit 1; \
-	    fi \
-	done
-
-run_fuzztest: test_decode3
-	bash -c 'ulimit -c unlimited; I=1; while true; do cat /dev/urandom | ./test_decode3 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done'

tests/SConstruct

@@ -0,0 +1,119 @@
+Help('''
+Type 'scons' to build and run all the available test cases.
+It will automatically detect your platform and C compiler and
+build appropriately.
+
+You can modify the behavious using following options:
+CC          Name of C compiler
+CXX         Name of C++ compiler
+CCFLAGS     Flags to pass to the C compiler
+CXXFLAGS    Flags to pass to the C++ compiler
+
+For example, for a clang build, use:
+scons CC=clang CXX=clang++
+''')
+
+import os
+env = Environment(ENV = os.environ)
+
+# Allow overriding the compiler with scons CC=???
+if 'CC' in ARGUMENTS: env.Replace(CC = ARGUMENTS['CC'])
+if 'CXX' in ARGUMENTS: env.Replace(CXX = ARGUMENTS['CXX'])
+if 'CFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CFLAGS'])
+if 'CXXFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CXXFLAGS'])
+
+# Add the builders defined in site_init.py
+add_nanopb_builders(env)
+
+# Path to the files shared by tests, and to the nanopb core.
+env.Append(CPPPATH = ["#../", "$COMMON"])
+
+# Path for finding nanopb.proto
+env.Append(PROTOCPATH = '#../generator')
+
+# Check the compilation environment, unless we are just cleaning up.
+if not env.GetOption('clean'):
+    conf = Configure(env)
+
+    # If the platform doesn't support C99, use our own header file instead.
+    stdbool = conf.CheckCHeader('stdbool.h')
+    stdint = conf.CheckCHeader('stdint.h')
+    stddef = conf.CheckCHeader('stddef.h')
+    string = conf.CheckCHeader('string.h')
+    if not stdbool or not stdint or not stddef or not string:
+        conf.env.Append(CPPDEFINES = {'PB_SYSTEM_HEADER': '\\"pb_syshdr.h\\"'})
+        conf.env.Append(CPPPATH = "#../compat")
+        
+        if stdbool: conf.env.Append(CPPDEFINES = {'HAVE_STDBOOL_H': 1})
+        if stdint: conf.env.Append(CPPDEFINES = {'HAVE_STDINT_H': 1})
+        if stddef: conf.env.Append(CPPDEFINES = {'HAVE_STDDEF_H': 1})
+        if string: conf.env.Append(CPPDEFINES = {'HAVE_STRING_H': 1})
+        
+    # Check if we can use pkg-config to find protobuf include path
+    status, output = conf.TryAction('pkg-config protobuf --variable=includedir > $TARGET')
+    if status:
+        conf.env.Append(PROTOCPATH = output.strip())
+    else:
+        conf.env.Append(PROTOCPATH = '/usr/include')
+    
+    # Check if libmudflap is available (only with GCC)
+    if 'gcc' in env['CC']:
+        if conf.CheckLib('mudflap'):
+            conf.env.Append(CCFLAGS = '-fmudflap')
+            conf.env.Append(LINKFLAGS = '-lmudflap -fmudflap')
+    
+    # End the config stuff
+    env = conf.Finish()
+
+# Initialize the CCFLAGS according to the compiler
+if 'gcc' in env['CC']:
+    # GNU Compiler Collection
+    
+    # Debug info, warnings as errors
+    env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror --coverage -fstack-protector-all')
+    env.Append(LINKFLAGS = '--coverage')
+    
+    # We currently need uint64_t anyway, even though ANSI C90 otherwise..
+    env.Append(CFLAGS = '-Wno-long-long')
+
+    # More strict checks on the nanopb core
+    env.Append(CORECFLAGS = '-Wextra -Wcast-qual -Wlogical-op -Wconversion')
+    env.Append(CORECFLAGS = ' -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls')
+    env.Append(CORECFLAGS = ' -Wstack-protector')
+elif 'clang' in env['CC']:
+    # CLang
+    env.Append(CFLAGS = '-ansi -g -O0 -Wall -Werror')
+    env.Append(CORECFLAGS = ' -Wextra -Wcast-qual -Wconversion')
+elif 'cl' in env['CC']:
+    # Microsoft Visual C++
+    
+    # Debug info on, warning level 2 for tests, warnings as errors
+    env.Append(CFLAGS = '/Zi /W2 /WX')
+    env.Append(LINKFLAGS = '/DEBUG')
+    
+    # More strict checks on the nanopb core
+    env.Append(CORECFLAGS = '/W4')
+    
+    # PB_RETURN_ERROR triggers C4127 because of while(0)
+    env.Append(CFLAGS = '/wd4127')
+elif 'tcc' in env['CC']:
+    # Tiny C Compiler
+    env.Append(CFLAGS = '-Wall -Werror -g')
+
+env.SetDefault(CORECFLAGS = '')
+
+if 'clang++' in env['CXX']:
+    env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers')
+elif 'g++' in env['CXX']:
+    env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers')
+elif 'cl' in env['CXX']:
+    env.Append(CXXFLAGS = '/Zi /W2 /WX')
+    
+# Now include the SConscript files from all subdirectories
+import os.path
+env['VARIANT_DIR'] = 'build'
+env['BUILD'] = '#' + env['VARIANT_DIR']
+env['COMMON'] = '#' + env['VARIANT_DIR'] + '/common'
+for subdir in Glob('*/SConscript'):
+    SConscript(subdir, exports = 'env', variant_dir = env['VARIANT_DIR'] + '/' + os.path.dirname(str(subdir)))
+

tests/alltypes/SConscript

@@ -0,0 +1,12 @@
+# Build and run a test that encodes and decodes a message that contains
+# all of the Protocol Buffers data types.
+
+Import("env")
+
+env.NanopbProto(["alltypes", "alltypes.options"])
+enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o"])
+dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_alltypes.output"])
+

tests/alltypes/decode_alltypes.c

@@ -8,6 +8,7 @@
 #include <stdlib.h>
 #include <pb_decode.h>
 #include "alltypes.pb.h"
+#include "test_helpers.h"
 
 #define TEST(x) if (!(x)) { \
     printf("Test " #x " failed.\n"); \
@@ -176,15 +177,19 @@ bool check_alltypes(pb_istream_t *stream, int mode)
 
 int main(int argc, char **argv)
 {
+    uint8_t buffer[1024];
+    size_t count;
+    pb_istream_t stream;
+
     /* Whether to expect the optional values or the default values. */
     int mode = (argc > 1) ? atoi(argv[1]) : 0;
     
     /* Read the data into buffer */
-    uint8_t buffer[1024];
+    SET_BINARY_MODE(stdin);
-    size_t count = fread(buffer, 1, sizeof(buffer), stdin);
+    count = fread(buffer, 1, sizeof(buffer), stdin);
     
     /* Construct a pb_istream_t for reading from the buffer */
-    pb_istream_t stream = pb_istream_from_buffer(buffer, count);
+    stream = pb_istream_from_buffer(buffer, count);
     
     /* Decode and print out the stuff */
     if (!check_alltypes(&stream, mode))

tests/alltypes/encode_alltypes.c

@@ -6,6 +6,7 @@
 #include <string.h>
 #include <pb_encode.h>
 #include "alltypes.pb.h"
+#include "test_helpers.h"
 
 int main(int argc, char **argv)
 {
@@ -113,18 +114,21 @@ int main(int argc, char **argv)
     
     alltypes.end = 1099;
     
-    uint8_t buffer[1024];
-    pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
-    
-    /* Now encode it and check if we succeeded. */
-    if (pb_encode(&stream, AllTypes_fields, &alltypes))
     {
-        fwrite(buffer, 1, stream.bytes_written, stdout);
+        uint8_t buffer[AllTypes_size];
-        return 0; /* Success */
+        pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
-    }
+        
-    else
+        /* Now encode it and check if we succeeded. */
-    {
+        if (pb_encode(&stream, AllTypes_fields, &alltypes))
-        fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
+        {
-        return 1; /* Failure */
+            SET_BINARY_MODE(stdout);
+            fwrite(buffer, 1, stream.bytes_written, stdout);
+            return 0; /* Success */
+        }
+        else
+        {
+            fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
+            return 1; /* Failure */
+        }
     }
 }

tests/backwards_compatibility/SConscript

@@ -0,0 +1,11 @@
+# Check that the old generated .pb.c/.pb.h files are still compatible with the
+# current version of nanopb.
+
+Import("env")
+
+enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_encode.o"])
+dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_legacy.output"])
+

tests/backwards_compatibility/alltypes_legacy.c

@@ -5,7 +5,7 @@
  * incompatible changes made to the generator in future versions.
  */
 
-#include "bc_alltypes.pb.h"
+#include "alltypes_legacy.h"
 
 const char SubMessage_substuff1_default[16] = "1";
 const int32_t SubMessage_substuff2_default = 2;

tests/backwards_compatibility/decode_legacy.c

@@ -1,16 +1,16 @@
 /* Tests the decoding of all types.
- * This is a backwards-compatibility test, using bc_alltypes.pb.h.
+ * This is a backwards-compatibility test, using alltypes_legacy.h.
- * It is similar to test_decode3, but duplicated in order to allow
+ * It is similar to decode_alltypes, but duplicated in order to allow
- * test_decode3 to test any new features introduced later.
+ * decode_alltypes to test any new features introduced later.
  *
- * Run e.g. ./bc_encode | ./bc_decode
+ * Run e.g. ./encode_legacy | ./decode_legacy
  */
 
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <pb_decode.h>
-#include "bc_alltypes.pb.h"
+#include "alltypes_legacy.h"
 
 #define TEST(x) if (!(x)) { \
     printf("Test " #x " failed.\n"); \

tests/backwards_compatibility/encode_legacy.c

@@ -1,14 +1,15 @@
 /* Attempts to test all the datatypes supported by ProtoBuf.
- * This is a backwards-compatibility test, using bc_alltypes.pb.h.
+ * This is a backwards-compatibility test, using alltypes_legacy.h.
- * It is similar to test_encode3, but duplicated in order to allow
+ * It is similar to encode_alltypes, but duplicated in order to allow
- * test_encode3 to test any new features introduced later.
+ * encode_alltypes to test any new features introduced later.
  */
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <pb_encode.h>
-#include "bc_alltypes.pb.h"
+#include "alltypes_legacy.h"
+#include "test_helpers.h"
 
 int main(int argc, char **argv)
 {
@@ -113,19 +114,22 @@ int main(int argc, char **argv)
     }
     
     alltypes.end = 1099;
-    
+
-    uint8_t buffer[1024];
+    {    
-    pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        uint8_t buffer[1024];
-    
+        pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
-    /* Now encode it and check if we succeeded. */
+        
-    if (pb_encode(&stream, AllTypes_fields, &alltypes))
+        /* Now encode it and check if we succeeded. */
-    {
+        if (pb_encode(&stream, AllTypes_fields, &alltypes))
-        fwrite(buffer, 1, stream.bytes_written, stdout);
+        {
-        return 0; /* Success */
+            SET_BINARY_MODE(stdout);
-    }
+            fwrite(buffer, 1, stream.bytes_written, stdout);
-    else
+            return 0; /* Success */
-    {
+        }
-        fprintf(stderr, "Encoding failed!\n");
+        else
-        return 1; /* Failure */
+        {
+            fprintf(stderr, "Encoding failed!\n");
+            return 1; /* Failure */
+        }
     }
 }

tests/basic_buffer/SConscript

@@ -0,0 +1,12 @@
+# Build and run a basic round-trip test using memory buffer encoding.
+
+Import("env")
+
+enc = env.Program(["encode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"])
+dec = env.Program(["decode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_buffer.output"])
+env.Decode(["encode_buffer.output", "$COMMON/person.proto"], MESSAGE = "Person")
+env.Compare(["decode_buffer.output", "encode_buffer.decoded"])
+

tests/basic_buffer/decode_buffer.c

@@ -9,6 +9,7 @@
 #include <stdio.h>
 #include <pb_decode.h>
 #include "person.pb.h"
+#include "test_helpers.h"
 
 /* This function is called once from main(), it handles
    the decoding and printing. */
@@ -59,9 +60,13 @@ bool print_person(pb_istream_t *stream)
 
 int main()
 {
+    uint8_t buffer[Person_size];
+    pb_istream_t stream;
+    size_t count;
+    
     /* Read the data into buffer */
-    uint8_t buffer[512];
+    SET_BINARY_MODE(stdin);
-    size_t count = fread(buffer, 1, sizeof(buffer), stdin);
+    count = fread(buffer, 1, sizeof(buffer), stdin);
     
     if (!feof(stdin))
     {
@@ -70,7 +75,7 @@ int main()
     }
     
     /* Construct a pb_istream_t for reading from the buffer */
-    pb_istream_t stream = pb_istream_from_buffer(buffer, count);
+    stream = pb_istream_from_buffer(buffer, count);
     
     /* Decode and print out the stuff */
     if (!print_person(&stream))

tests/basic_buffer/encode_buffer.c

@@ -6,9 +6,13 @@
 #include <stdio.h>
 #include <pb_encode.h>
 #include "person.pb.h"
+#include "test_helpers.h"
 
 int main()
 {
+    uint8_t buffer[Person_size];
+    pb_ostream_t stream;
+    
     /* Initialize the structure with constants */
     Person person = {"Test Person 99", 99, true, "test@person.com",
         3, {{"555-12345678", true, Person_PhoneType_MOBILE},
@@ -16,12 +20,13 @@ int main()
             {"1234-5678", true, Person_PhoneType_WORK},
         }};
 
-    uint8_t buffer[512];
+    stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
-    pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
     
     /* Now encode it and check if we succeeded. */
     if (pb_encode(&stream, Person_fields, &person))
-    {
+    {    
+        /* Write the result data to stdout */
+        SET_BINARY_MODE(stdout);
         fwrite(buffer, 1, stream.bytes_written, stdout);
         return 0; /* Success */
     }

tests/basic_stream/SConscript

@@ -0,0 +1,12 @@
+# Build and run a basic round-trip test using direct stream encoding.
+
+Import("env")
+
+enc = env.Program(["encode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"])
+dec = env.Program(["decode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_stream.output"])
+env.Decode(["encode_stream.output", "$COMMON/person.proto"], MESSAGE = "Person")
+env.Compare(["decode_stream.output", "encode_stream.decoded"])
+

tests/basic_stream/decode_stream.c

@@ -4,6 +4,7 @@
 #include <stdio.h>
 #include <pb_decode.h>
 #include "person.pb.h"
+#include "test_helpers.h"
 
 /* This function is called once from main(), it handles
    the decoding and printing.
@@ -69,10 +70,10 @@ bool callback(pb_istream_t *stream, uint8_t *buf, size_t count)
 
 int main()
 {
-    /* Maximum size is specified to prevent infinite length messages from
+    pb_istream_t stream = {&callback, NULL, SIZE_MAX};
-     * hanging this in the fuzz test.
+    stream.state = stdin;
-     */
+    SET_BINARY_MODE(stdin);
-    pb_istream_t stream = {&callback, stdin, 10000};
+
     if (!print_person(&stream))
     {
         printf("Parsing failed: %s\n", PB_GET_ERROR(&stream));

tests/basic_stream/encode_stream.c

@@ -4,6 +4,7 @@
 #include <stdio.h>
 #include <pb_encode.h>
 #include "person.pb.h"
+#include "test_helpers.h"
 
 /* This binds the pb_ostream_t into the stdout stream */
 bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count)
@@ -22,7 +23,9 @@ int main()
         }};
     
     /* Prepare the stream, output goes directly to stdout */
-    pb_ostream_t stream = {&streamcallback, stdout, SIZE_MAX, 0};
+    pb_ostream_t stream = {&streamcallback, NULL, SIZE_MAX, 0};
+    stream.state = stdout;
+    SET_BINARY_MODE(stdout);
     
     /* Now encode it and check if we succeeded. */
     if (pb_encode(&stream, Person_fields, &person))

tests/buffer_only/SConscript

@@ -0,0 +1,23 @@
+# Run the alltypes test case, but compile with PB_BUFFER_ONLY=1
+
+Import("env")
+
+# Take copy of the files for custom build.
+c = Copy("$TARGET", "$SOURCE")
+env.Command("pb_encode.c", "#../pb_encode.c", c)
+env.Command("pb_decode.c", "#../pb_decode.c", c)
+env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c)
+env.Command("alltypes.pb.c", "$BUILD/alltypes/alltypes.pb.c", c)
+env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c)
+
+# Define the compilation options
+opts = env.Clone()
+opts.Append(CPPDEFINES = {'PB_BUFFER_ONLY': 1})
+
+# Now build and run the test normally.
+enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"])
+dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_alltypes.output"])

tests/callbacks/SConscript

@@ -0,0 +1,14 @@
+# Test the functionality of the callback fields.
+
+Import("env")
+
+env.NanopbProto("callbacks")
+enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_encode.o"])
+dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_callbacks.output"])
+
+env.Decode(["encode_callbacks.output", "callbacks.proto"], MESSAGE = "TestMessage")
+env.Compare(["decode_callbacks.output", "encode_callbacks.decoded"])
+

tests/callbacks/decode_callbacks.c

@@ -5,6 +5,7 @@
 #include <stdio.h>
 #include <pb_decode.h>
 #include "callbacks.pb.h"
+#include "test_helpers.h"
 
 bool print_string(pb_istream_t *stream, const pb_field_t *field, void **arg)
 {
@@ -50,21 +51,24 @@ bool print_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg)
     if (!pb_decode_fixed64(stream, &value))
         return false;
     
-    printf((char*)*arg, (long long)value);
+    printf((char*)*arg, (long)value);
     return true;
 }
 
 int main()
 {
     uint8_t buffer[1024];
-    size_t length = fread(buffer, 1, 1024, stdin);
+    size_t length;
-    pb_istream_t stream = pb_istream_from_buffer(buffer, length);
+    pb_istream_t stream;
-    
     /* Note: empty initializer list initializes the struct with all-0.
      * This is recommended so that unused callbacks are set to NULL instead
      * of crashing at runtime.
      */
-    TestMessage testmessage = {};
+    TestMessage testmessage = {{{NULL}}};
+    
+    SET_BINARY_MODE(stdin);
+    length = fread(buffer, 1, 1024, stdin);
+    stream = pb_istream_from_buffer(buffer, length);    
     
     testmessage.submsg.stringvalue.funcs.decode = &print_string;
     testmessage.submsg.stringvalue.arg = "submsg {\n  stringvalue: \"%s\"\n";
@@ -73,7 +77,7 @@ int main()
     testmessage.submsg.fixed32value.funcs.decode = &print_fixed32;
     testmessage.submsg.fixed32value.arg = "  fixed32value: %ld\n";
     testmessage.submsg.fixed64value.funcs.decode = &print_fixed64;
-    testmessage.submsg.fixed64value.arg = "  fixed64value: %lld\n}\n";
+    testmessage.submsg.fixed64value.arg = "  fixed64value: %ld\n}\n";
     
     testmessage.stringvalue.funcs.decode = &print_string;
     testmessage.stringvalue.arg = "stringvalue: \"%s\"\n";
@@ -82,7 +86,7 @@ int main()
     testmessage.fixed32value.funcs.decode = &print_fixed32;
     testmessage.fixed32value.arg = "fixed32value: %ld\n";
     testmessage.fixed64value.funcs.decode = &print_fixed64;
-    testmessage.fixed64value.arg = "fixed64value: %lld\n";
+    testmessage.fixed64value.arg = "fixed64value: %ld\n";
     testmessage.repeatedstring.funcs.decode = &print_string;
     testmessage.repeatedstring.arg = "repeatedstring: \"%s\"\n";
     

tests/callbacks/encode_callbacks.c

@@ -4,6 +4,7 @@
 #include <string.h>
 #include <pb_encode.h>
 #include "callbacks.pb.h"
+#include "test_helpers.h"
 
 bool encode_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
 {
@@ -25,19 +26,21 @@ bool encode_int32(pb_ostream_t *stream, const pb_field_t *field, void * const *a
 
 bool encode_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
 {
+    uint32_t value = 42;
+
     if (!pb_encode_tag_for_field(stream, field))
         return false;
     
-    uint32_t value = 42;
     return pb_encode_fixed32(stream, &value);
 }
 
 bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
 {
+    uint64_t value = 42;
+
     if (!pb_encode_tag_for_field(stream, field))
         return false;
     
-    uint64_t value = 42;
     return pb_encode_fixed64(stream, &value);
 }
 
@@ -60,8 +63,10 @@ bool encode_repeatedstring(pb_ostream_t *stream, const pb_field_t *field, void *
 int main()
 {
     uint8_t buffer[1024];
-    pb_ostream_t stream = pb_ostream_from_buffer(buffer, 1024);
+    pb_ostream_t stream;
-    TestMessage testmessage = {};
+    TestMessage testmessage = {{{NULL}}};
+    
+    stream = pb_ostream_from_buffer(buffer, 1024);
     
     testmessage.stringvalue.funcs.encode = &encode_string;
     testmessage.int32value.funcs.encode = &encode_int32;
@@ -79,6 +84,7 @@ int main()
     if (!pb_encode(&stream, TestMessage_fields, &testmessage))
         return 1;
     
+    SET_BINARY_MODE(stdout);
     if (fwrite(buffer, stream.bytes_written, 1, stdout) != 1)
         return 2;
     

tests/common/SConscript

@@ -0,0 +1,17 @@
+# Build the common files needed by multiple test cases
+
+Import('env')
+
+# Protocol definitions for the encode/decode_unittests
+env.NanopbProto("unittestproto")
+
+# Protocol definitions for basic_buffer/stream tests
+env.NanopbProto("person")
+
+# Binaries of the pb_decode.c and pb_encode.c
+# These are built using more strict warning flags.
+strict = env.Clone()
+strict.Append(CFLAGS = strict['CORECFLAGS'])
+strict.Object("pb_decode.o", "#../pb_decode.c")
+strict.Object("pb_encode.o", "#../pb_encode.c")
+

tests/common/test_helpers.h

@@ -0,0 +1,17 @@
+/* Compatibility helpers for the test programs. */
+
+#ifndef _TEST_HELPERS_H_
+#define _TEST_HELPERS_H_
+
+#ifdef _WIN32
+#include <io.h>
+#include <fcntl.h>
+#define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
+
+#else
+#define SET_BINARY_MODE(file)
+
+#endif
+
+
+#endif

tests/common/unittestproto.proto

@@ -12,6 +12,10 @@ message StringMessage {
     required string data = 1 [(nanopb).max_size = 10];
 }
 
+message BytesMessage {
+    required bytes data = 1 [(nanopb).max_size = 16];
+}
+
 message CallbackArray {
     // We cheat a bit and use this message for testing other types, too.
     // Nanopb does not care about the actual defined data type for callback

tests/cxx_main_program/SConscript

@@ -0,0 +1,20 @@
+# Run the alltypes test case, but compile it as C++ instead.
+# In fact, compile the entire nanopb using C++ compiler.
+
+Import("env")
+
+# Copy the files to .cxx extension in order to force C++ build.
+c = Copy("$TARGET", "$SOURCE")
+env.Command("pb_encode.cxx", "#../pb_encode.c", c)
+env.Command("pb_decode.cxx", "#../pb_decode.c", c)
+env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c)
+env.Command("alltypes.pb.cxx", "$BUILD/alltypes/alltypes.pb.c", c)
+env.Command("encode_alltypes.cxx", "$BUILD/alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.cxx", "$BUILD/alltypes/decode_alltypes.c", c)
+
+# Now build and run the test normally.
+enc = env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx"])
+dec = env.Program(["decode_alltypes.cxx", "alltypes.pb.cxx", "pb_decode.cxx"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_alltypes.output"])

tests/decode_unittests/SConscript

@@ -0,0 +1,4 @@
+Import('env')
+p = env.Program(["decode_unittests.c", "$COMMON/unittestproto.pb.c"])
+env.RunTest(p)
+

tests/decode_unittests/decode_unittests.c

@@ -1,8 +1,8 @@
-#define NANOPB_INTERNALS
+/* This includes the whole .c file to get access to static functions. */
+#include "pb_decode.c"
 
 #include <stdio.h>
 #include <string.h>
-#include "pb_decode.h"
 #include "unittests.h"
 #include "unittestproto.pb.h"
 
@@ -291,7 +291,7 @@ int main()
     
     {
         pb_istream_t s;
-        IntegerContainer dest = {};
+        IntegerContainer dest = {{0}};
         
         COMMENT("Testing pb_decode_delimited")
         TEST((s = S("\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05"),

tests/encode_unittests/SConscript

@@ -0,0 +1,5 @@
+# Build and run the stand-alone unit tests for the nanopb encoder part.
+
+Import('env')
+p = env.Program(["encode_unittests.c", "$COMMON/unittestproto.pb.c"])
+env.RunTest(p)

tests/encode_unittests/encode_unittests.c

@@ -1,8 +1,8 @@
-#define NANOPB_INTERNALS
+/* This includes the whole .c file to get access to static functions. */
+#include "pb_encode.c"
 
 #include <stdio.h>
 #include <string.h>
-#include "pb_encode.h"
 #include "unittests.h"
 #include "unittestproto.pb.h"
 
@@ -172,9 +172,9 @@ int main()
         struct { size_t size; uint8_t bytes[5]; } value = {5, {'x', 'y', 'z', 'z', 'y'}};
     
         COMMENT("Test pb_enc_bytes")
-        TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x05xyzzy"))
+        TEST(WRITES(pb_enc_bytes(&s, &BytesMessage_fields[0], &value), "\x05xyzzy"))
         value.size = 0;
-        TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x00"))
+        TEST(WRITES(pb_enc_bytes(&s, &BytesMessage_fields[0], &value), "\x00"))
     }
     
     {
@@ -223,6 +223,20 @@ int main()
         TEST(!pb_encode(&s, FloatArray_fields, &msg))
     }
     
+    {
+        uint8_t buffer[50];
+        pb_ostream_t s;
+        FloatArray msg = {1, {99.0f}};
+        
+        COMMENT("Test array size limit in pb_encode")
+        
+        s = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        TEST((msg.data_count = 10) && pb_encode(&s, FloatArray_fields, &msg))
+        
+        s = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        TEST((msg.data_count = 11) && !pb_encode(&s, FloatArray_fields, &msg))
+    }
+    
     {
         uint8_t buffer[10];
         pb_ostream_t s;
@@ -244,6 +258,20 @@ int main()
                     "\x0A\x07\x0A\x05\x01\x02\x03\x04\x05"))
     }
     
+    {
+        uint8_t buffer[32];
+        pb_ostream_t s;
+        BytesMessage msg = {{3, "xyz"}};
+        
+        COMMENT("Test pb_encode with bytes message.")
+        TEST(WRITES(pb_encode(&s, BytesMessage_fields, &msg),
+                    "\x0A\x03xyz"))
+        
+        msg.data.size = 17; /* More than maximum */
+        TEST(!pb_encode(&s, BytesMessage_fields, &msg))
+    }
+        
+    
     {
         uint8_t buffer[20];
         pb_ostream_t s;
@@ -280,6 +308,19 @@ int main()
         TEST(!pb_encode(&s, CallbackContainerContainer_fields, &msg2))
     }
     
+    {
+        uint8_t buffer[StringMessage_size];
+        pb_ostream_t s;
+        StringMessage msg = {"0123456789"};
+        
+        s = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        
+        COMMENT("Test that StringMessage_size is correct")
+
+        TEST(pb_encode(&s, StringMessage_fields, &msg));
+        TEST(s.bytes_written == StringMessage_size);
+    }
+    
     if (status != 0)
         fprintf(stdout, "\n\nSome tests FAILED!\n");
     

tests/extensions/SConscript

@@ -0,0 +1,16 @@
+# Test the support for extension fields.
+
+Import("env")
+
+# We use the files from the alltypes test case
+incpath = env.Clone()
+incpath.Append(PROTOCPATH = '$BUILD/alltypes')
+incpath.Append(CPPPATH = '$BUILD/alltypes')
+
+incpath.NanopbProto(["extensions", "extensions.options"])
+enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_encode.o"])
+dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_extensions.output"])
+

tests/extensions/decode_extensions.c

@@ -0,0 +1,58 @@
+/* Test decoding of extension fields. */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <pb_decode.h>
+#include "alltypes.pb.h"
+#include "extensions.pb.h"
+#include "test_helpers.h"
+
+#define TEST(x) if (!(x)) { \
+    printf("Test " #x " failed.\n"); \
+    return 2; \
+    }
+
+int main(int argc, char **argv)
+{
+    uint8_t buffer[1024];
+    size_t count;
+    pb_istream_t stream;
+    
+    AllTypes alltypes = {0};
+    int32_t extensionfield1;
+    pb_extension_t ext1;
+    ExtensionMessage extensionfield2;
+    pb_extension_t ext2;
+    
+    /* Read the message data */
+    SET_BINARY_MODE(stdin);
+    count = fread(buffer, 1, sizeof(buffer), stdin);
+    stream = pb_istream_from_buffer(buffer, count);
+    
+    /* Add the extensions */
+    alltypes.extensions = &ext1;        
+
+    ext1.type = &AllTypes_extensionfield1;
+    ext1.dest = &extensionfield1;
+    ext1.next = &ext2;
+    
+    ext2.type = &ExtensionMessage_AllTypes_extensionfield2;
+    ext2.dest = &extensionfield2;
+    ext2.next = NULL;
+    
+    /* Decode the message */
+    if (!pb_decode(&stream, AllTypes_fields, &alltypes))
+    {
+        printf("Parsing failed: %s\n", PB_GET_ERROR(&stream));
+        return 1;
+    }
+
+    /* Check that the extensions decoded properly */
+    TEST(extensionfield1 == 12345)
+    TEST(strcmp(extensionfield2.test1, "test") == 0)
+    TEST(extensionfield2.test2 == 54321)
+    
+    return 0;
+}
+

tests/extensions/encode_extensions.c

@@ -0,0 +1,54 @@
+/* Tests extension fields.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pb_encode.h>
+#include "alltypes.pb.h"
+#include "extensions.pb.h"
+#include "test_helpers.h"
+
+int main(int argc, char **argv)
+{
+    uint8_t buffer[1024];
+    pb_ostream_t stream;
+
+    AllTypes alltypes = {0};
+    int32_t extensionfield1 = 12345;
+    pb_extension_t ext1;
+    ExtensionMessage extensionfield2 = {"test", 54321};
+    pb_extension_t ext2;
+
+    /* Set up the extensions */
+    alltypes.extensions = &ext1;
+
+    ext1.type = &AllTypes_extensionfield1;
+    ext1.dest = &extensionfield1;
+    ext1.next = &ext2;
+    
+    ext2.type = &ExtensionMessage_AllTypes_extensionfield2;
+    ext2.dest = &extensionfield2;
+    ext2.next = NULL;
+
+    /* Set up the output stream */
+    stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+    
+    /* Now encode the message and check if we succeeded. */
+    if (pb_encode(&stream, AllTypes_fields, &alltypes))
+    {
+        SET_BINARY_MODE(stdout);
+        fwrite(buffer, 1, stream.bytes_written, stdout);
+        return 0; /* Success */
+    }
+    else
+    {
+        fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
+        return 1; /* Failure */
+    }
+    
+    /* Check that the field tags are properly generated */
+    (void)AllTypes_extensionfield1_tag;
+    (void)ExtensionMessage_AllTypes_extensionfield2_tag;
+}
+

tests/extensions/extensions.proto

@@ -7,6 +7,8 @@ extend AllTypes {
 message ExtensionMessage {
     extend AllTypes {
         optional ExtensionMessage AllTypes_extensionfield2 = 254;
+	required ExtensionMessage AllTypes_extensionfield3 = 253;
+	repeated ExtensionMessage AllTypes_extensionfield4 = 252;
     }
     
     required string test1 = 1;

tests/extra_fields/SConscript

@@ -0,0 +1,14 @@
+# Test that the decoder properly handles unknown fields in the input.
+
+Import("env")
+
+dec = env.GetBuildPath('$BUILD/basic_buffer/${PROGPREFIX}decode_buffer${PROGSUFFIX}')
+env.RunTest('person_with_extra_field.output', [dec, "person_with_extra_field.pb"])
+env.Compare(["person_with_extra_field.output", "person_with_extra_field.expected"])
+
+dec = env.GetBuildPath('$BUILD/basic_stream/${PROGPREFIX}decode_stream${PROGSUFFIX}')
+env.RunTest('person_with_extra_field_stream.output', [dec, "person_with_extra_field.pb"])
+env.Compare(["person_with_extra_field_stream.output", "person_with_extra_field.expected"])
+
+dec2 = env.GetBuildPath('$BUILD/alltypes/${PROGPREFIX}decode_alltypes${PROGSUFFIX}')
+env.RunTest('alltypes_with_extra_fields.output', [dec2, 'alltypes_with_extra_fields.pb'])

tests/extra_fields/person_with_extra_field.expected

@@ -0,0 +1,14 @@
+name: "Test Person 99"
+id: 99
+email: "test@person.com"
+phone {
+  number: "555-12345678"
+  type: MOBILE
+}
+phone {
+  number: "99-2342"
+}
+phone {
+  number: "1234-5678"
+  type: WORK
+}

tests/field_size_16/SConscript

@@ -0,0 +1,24 @@
+# Run the alltypes test case, but compile with PB_FIELD_16BIT=1.
+# Also the .proto file has been modified to have high indexes.
+
+Import("env")
+
+# Take copy of the files for custom build.
+c = Copy("$TARGET", "$SOURCE")
+env.Command("pb_encode.c", "#../pb_encode.c", c)
+env.Command("pb_decode.c", "#../pb_decode.c", c)
+env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c)
+
+env.NanopbProto(["alltypes", "alltypes.options"])
+
+# Define the compilation options
+opts = env.Clone()
+opts.Append(CPPDEFINES = {'PB_FIELD_16BIT': 1})
+
+# Now build and run the test normally.
+enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"])
+dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_alltypes.output"])

tests/field_size_16/alltypes.options

@@ -0,0 +1,3 @@
+* max_size:16
+* max_count:5
+

tests/field_size_16/alltypes.proto

@@ -0,0 +1,90 @@
+message SubMessage {
+    required string substuff1 = 1 [default = "1"];
+    required int32 substuff2 = 2 [default = 2];
+    optional fixed32 substuff3 = 65535 [default = 3];
+}
+
+message EmptyMessage {
+
+}
+
+enum MyEnum {
+    Zero = 0;
+    First = 1;
+    Second = 2;
+    Truth = 42;
+}
+
+message AllTypes {
+    required int32      req_int32   = 1;
+    required int64      req_int64   = 2;
+    required uint32     req_uint32  = 3;
+    required uint64     req_uint64  = 4;
+    required sint32     req_sint32  = 5;
+    required sint64     req_sint64  = 6;
+    required bool       req_bool    = 7;
+    
+    required fixed32    req_fixed32 = 8;
+    required sfixed32   req_sfixed32= 9;
+    required float      req_float   = 10;
+    
+    required fixed64    req_fixed64 = 11;
+    required sfixed64   req_sfixed64= 12;
+    required double     req_double  = 13;
+    
+    required string     req_string  = 14;
+    required bytes      req_bytes   = 15;
+    required SubMessage req_submsg  = 16;
+    required MyEnum     req_enum    = 17;
+    required EmptyMessage req_emptymsg = 18;
+    
+    
+    repeated int32      rep_int32   = 21;
+    repeated int64      rep_int64   = 22;
+    repeated uint32     rep_uint32  = 23;
+    repeated uint64     rep_uint64  = 24;
+    repeated sint32     rep_sint32  = 25;
+    repeated sint64     rep_sint64  = 26;
+    repeated bool       rep_bool    = 27;
+    
+    repeated fixed32    rep_fixed32 = 28;
+    repeated sfixed32   rep_sfixed32= 29;
+    repeated float      rep_float   = 30;
+    
+    repeated fixed64    rep_fixed64 = 10031;
+    repeated sfixed64   rep_sfixed64= 10032;
+    repeated double     rep_double  = 10033;
+    
+    repeated string     rep_string  = 10034;
+    repeated bytes      rep_bytes   = 10035;
+    repeated SubMessage rep_submsg  = 10036;
+    repeated MyEnum     rep_enum    = 10037;
+    repeated EmptyMessage rep_emptymsg = 10038;
+    
+    optional int32      opt_int32   = 10041 [default = 4041];
+    optional int64      opt_int64   = 10042 [default = 4042];
+    optional uint32     opt_uint32  = 10043 [default = 4043];
+    optional uint64     opt_uint64  = 10044 [default = 4044];
+    optional sint32     opt_sint32  = 10045 [default = 4045];
+    optional sint64     opt_sint64  = 10046 [default = 4046];
+    optional bool       opt_bool    = 10047 [default = false];
+    
+    optional fixed32    opt_fixed32 = 10048 [default = 4048];
+    optional sfixed32   opt_sfixed32= 10049 [default = 4049];
+    optional float      opt_float   = 10050 [default = 4050];
+    
+    optional fixed64    opt_fixed64 = 10051 [default = 4051];
+    optional sfixed64   opt_sfixed64= 10052 [default = 4052];
+    optional double     opt_double  = 10053 [default = 4053];
+    
+    optional string     opt_string  = 10054 [default = "4054"];
+    optional bytes      opt_bytes   = 10055 [default = "4055"];
+    optional SubMessage opt_submsg  = 10056;
+    optional MyEnum     opt_enum    = 10057 [default = Second];
+    optional EmptyMessage opt_emptymsg = 10058;
+
+    // Just to make sure that the size of the fields has been calculated
+    // properly, i.e. otherwise a bug in last field might not be detected.
+    required int32      end = 10099;
+}
+

tests/field_size_32/SConscript

@@ -0,0 +1,24 @@
+# Run the alltypes test case, but compile with PB_FIELD_32BIT=1.
+# Also the .proto file has been modified to have high indexes.
+
+Import("env")
+
+# Take copy of the files for custom build.
+c = Copy("$TARGET", "$SOURCE")
+env.Command("pb_encode.c", "#../pb_encode.c", c)
+env.Command("pb_decode.c", "#../pb_decode.c", c)
+env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c)
+
+env.NanopbProto(["alltypes", "alltypes.options"])
+
+# Define the compilation options
+opts = env.Clone()
+opts.Append(CPPDEFINES = {'PB_FIELD_32BIT': 1})
+
+# Now build and run the test normally.
+enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"])
+dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_alltypes.output"])

tests/field_size_32/alltypes.options

@@ -0,0 +1,3 @@
+* max_size:16
+* max_count:5
+

tests/field_size_32/alltypes.proto

@@ -0,0 +1,90 @@
+message SubMessage {
+    required string substuff1 = 1 [default = "1"];
+    required int32 substuff2 = 2 [default = 2];
+    optional fixed32 substuff3 = 12365535 [default = 3];
+}
+
+message EmptyMessage {
+
+}
+
+enum MyEnum {
+    Zero = 0;
+    First = 1;
+    Second = 2;
+    Truth = 42;
+}
+
+message AllTypes {
+    required int32      req_int32   = 1;
+    required int64      req_int64   = 2;
+    required uint32     req_uint32  = 3;
+    required uint64     req_uint64  = 4;
+    required sint32     req_sint32  = 5;
+    required sint64     req_sint64  = 6;
+    required bool       req_bool    = 7;
+    
+    required fixed32    req_fixed32 = 8;
+    required sfixed32   req_sfixed32= 9;
+    required float      req_float   = 10;
+    
+    required fixed64    req_fixed64 = 11;
+    required sfixed64   req_sfixed64= 12;
+    required double     req_double  = 13;
+    
+    required string     req_string  = 14;
+    required bytes      req_bytes   = 15;
+    required SubMessage req_submsg  = 16;
+    required MyEnum     req_enum    = 17;
+    required EmptyMessage req_emptymsg = 18;
+    
+    
+    repeated int32      rep_int32   = 21;
+    repeated int64      rep_int64   = 22;
+    repeated uint32     rep_uint32  = 23;
+    repeated uint64     rep_uint64  = 24;
+    repeated sint32     rep_sint32  = 25;
+    repeated sint64     rep_sint64  = 26;
+    repeated bool       rep_bool    = 27;
+    
+    repeated fixed32    rep_fixed32 = 28;
+    repeated sfixed32   rep_sfixed32= 29;
+    repeated float      rep_float   = 30;
+    
+    repeated fixed64    rep_fixed64 = 10031;
+    repeated sfixed64   rep_sfixed64= 10032;
+    repeated double     rep_double  = 10033;
+    
+    repeated string     rep_string  = 10034;
+    repeated bytes      rep_bytes   = 10035;
+    repeated SubMessage rep_submsg  = 10036;
+    repeated MyEnum     rep_enum    = 10037;
+    repeated EmptyMessage rep_emptymsg = 10038;
+    
+    optional int32      opt_int32   = 10041 [default = 4041];
+    optional int64      opt_int64   = 10042 [default = 4042];
+    optional uint32     opt_uint32  = 10043 [default = 4043];
+    optional uint64     opt_uint64  = 10044 [default = 4044];
+    optional sint32     opt_sint32  = 10045 [default = 4045];
+    optional sint64     opt_sint64  = 10046 [default = 4046];
+    optional bool       opt_bool    = 10047 [default = false];
+    
+    optional fixed32    opt_fixed32 = 10048 [default = 4048];
+    optional sfixed32   opt_sfixed32= 10049 [default = 4049];
+    optional float      opt_float   = 10050 [default = 4050];
+    
+    optional fixed64    opt_fixed64 = 10051 [default = 4051];
+    optional sfixed64   opt_sfixed64= 10052 [default = 4052];
+    optional double     opt_double  = 10053 [default = 4053];
+    
+    optional string     opt_string  = 10054 [default = "4054"];
+    optional bytes      opt_bytes   = 10055 [default = "4055"];
+    optional SubMessage opt_submsg  = 10056;
+    optional MyEnum     opt_enum    = 10057 [default = Second];
+    optional EmptyMessage opt_emptymsg = 10058;
+
+    // Just to make sure that the size of the fields has been calculated
+    // properly, i.e. otherwise a bug in last field might not be detected.
+    required int32      end = 13432099;
+}
+

tests/message_sizes/SConscript

@@ -0,0 +1,11 @@
+# Test the generation of message size #defines
+
+Import('env')
+
+incpath = env.Clone()
+incpath.Append(PROTOCPATH = '#message_sizes')
+
+incpath.NanopbProto("messages1")
+incpath.NanopbProto("messages2")
+
+incpath.Program(['dummy.c', 'messages1.pb.c', 'messages2.pb.c'])

tests/message_sizes/dummy.c

@@ -0,0 +1,9 @@
+/* Just test that the file can be compiled successfully. */
+
+#include "messages2.pb.h"
+
+int main()
+{
+    return xmit_size;
+}
+

tests/message_sizes/messages1.proto

@@ -0,0 +1,27 @@
+enum MessageStatus {
+    FAIL = 0;
+    OK = 1;
+};
+
+message MessageInfo {
+   required fixed32 msg_id = 1;
+   optional fixed32 interface_id = 2;
+}
+
+message MessageResponseInfo {
+   required fixed64 interface_id = 1;
+   required fixed32 seq = 2;
+   required fixed32 msg_id = 3;
+}
+
+message MessageHeader {
+   required MessageInfo info = 1;
+   optional MessageResponseInfo response_info = 2;
+   optional MessageResponse response = 3;
+}
+
+message MessageResponse {
+   required MessageStatus status = 1;
+   required fixed32 seq = 2;
+}
+

tests/message_sizes/messages2.proto

@@ -0,0 +1,8 @@
+import 'nanopb.proto';
+import 'messages1.proto';
+
+message xmit {
+   required MessageHeader header = 1;
+   required bytes data = 2 [(nanopb).max_size = 128];
+}
+

tests/missing_fields/SConscript

@@ -0,0 +1,8 @@
+# Check that the decoder properly detects when required fields are missing.
+
+Import("env")
+
+env.NanopbProto("missing_fields")
+test = env.Program(["missing_fields.c", "missing_fields.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_decode.o"])
+env.RunTest(test)
+