Add tests for different compilation options

Otherwise Microsoft Visual C++ threats them as C++ classes instead of plain
structs, forbidding use in C linkage functions.

Thanks to Markus Schwarzenberg for the patch.

Update issue 84
Status: Started

.gitignore

@@ -0,0 +1,21 @@
+*.gcda
+*.gcno
+*.gcov
+*.o
+*.pb.c
+*.pb.h
+*.pb
+*~
+*.tar.gz
+.sconsign.dblite
+julkaisu.txt
+docs/*.html
+docs/generator_flow.png
+example/client
+example/server
+example_avr_double/decode_double
+example_avr_double/encode_double
+example_avr_double/test_conversions
+example_unions/decode
+example_unions/encode
+generator/nanopb_pb2.pyc

CHANGELOG

@@ -1,4 +1,60 @@
-nanopb-0.1.7 (not yet released)
+nanopb-0.2.2
+ Add support for extension fields (issue 17)
+ Fix unknown fields in empty message (issue 78)
+ Include the field tags in the generated .pb.h file.
+ Add pb_decode_delimited and pb_encode_delimited wrapper functions (issue 74)
+ Add a section in top of pb.h for changing compilation settings (issue 76)
+ Documentation improvements (issues 12, 77 and others)
+ Improved tests
+
+nanopb-0.2.1
+ NOTE: The default callback function signature has changed.
+       If you don't want to update your code, define PB_OLD_CALLBACK_STYLE.
+ 
+ Change the callback function to use void** (issue 69)
+ Add support for defining the nanopb options in a separate file (issue 12)
+ Add support for packed structs in IAR and MSVC (in addition to GCC) (issue 66)
+ Implement error message support for the encoder side (issue 7)
+ Handle unterminated strings when encoding (issue 68)
+ Fix bug with empty strings in repeated string callbacks (issue 73)
+ Fix regression in 0.2.0 with optional callback fields (issue 70)
+ Fix bugs with empty message types (issues 64, 65)
+ Fix some compiler warnings on clang (issue 67)
+ Some portability improvements (issues 60, 62)
+ Various new generator options
+ Improved tests
+
+nanopb-0.2.0
+ NOTE: This release requires you to regenerate all .pb.c
+       files. Files generated by older versions will not
+       compile anymore.
+
+ Reformat generated .pb.c files using macros (issue 58)
+ Rename PB_HTYPE_ARRAY -> PB_HTYPE_REPEATED
+ Separate PB_HTYPE to PB_ATYPE and PB_HTYPE
+ Move STATIC_ASSERTs to .pb.c file
+ Added CMake file (by Pavel Ilin)
+ Add option to give file extension to generator (by Michael Haberler)
+ Documentation updates
+
+nanopb-0.1.9
+ Fixed error message bugs (issues 52, 56)
+ Sanitize #ifndef filename (issue 50)
+ Performance improvements
+ Add compile-time option PB_BUFFER_ONLY
+ Add Java package name to nanopb.proto
+ Check for sizeof(double) == 8 (issue 54)
+ Added generator option to ignore some fields. (issue 51)
+ Added generator option to make message structs packed. (issue 49)
+ Add more test cases.
+
+nanopb-0.1.8
+ Fix bugs in the enum short names introduced in 0.1.7 (issues 42, 43)
+ Fix STATIC_ASSERT macro when using multiple .proto files. (issue 41)
+ Fix missing initialization of istream.errmsg
+ Make tests/Makefile work for non-gcc compilers (issue 40)
+
+nanopb-0.1.7
  Remove "skip" mode from pb_istream_t callbacks. Example implementation had a bug. (issue 37)
  Add option to use shorter names for enum values (issue 38)
  Improve options support in generator (issues 12, 30)

cmake/FindNanopb.cmake

@@ -0,0 +1,224 @@
+# Locate and configure the nanopb library.
+#
+# The following varialbes have to be set:
+#
+#   NANOPB_SRC_ROOT_FOLDER  - Path to nanopb source folder
+#
+# The following variables can be set and are optional:
+#
+#
+#   PROTOBUF_SRC_ROOT_FOLDER - When compiling with MSVC, if this cache variable is set
+#                              the protobuf-default VS project build locations
+#                              (vsprojects/Debug & vsprojects/Release) will be searched
+#                              for libraries and binaries.
+#
+#   NANOPB_IMPORT_DIRS       - List of additional directories to be searched for
+#                              imported .proto files.
+#
+#   NANOPB_GENERATE_CPP_APPEND_PATH - By default -I will be passed to protoc
+#                                     for each directory where a proto file is referenced.
+#                                     Set to FALSE if you want to disable this behaviour.
+#
+# Defines the following variables:
+#
+#   NANOPB_FOUND - Found the nanopb library (source&header files, generator tool, protoc compiler tool)
+#   NANOPB_INCLUDE_DIRS - Include directories for Google Protocol Buffers
+#
+# The following cache variables are also available to set or use:
+#   NANOPB_GENERATOR_EXECUTABLE - The nanopb generator
+#   PROTOBUF_PROTOC_EXECUTABLE - The protoc compiler
+#
+#  ====================================================================
+#
+# NANOPB_GENERATE_CPP (public function)
+#   SRCS = Variable to define with autogenerated
+#          source files
+#   HDRS = Variable to define with autogenerated
+#          header files
+#   ARGN = proto files
+#
+#  ====================================================================
+#  Example:
+#
+#   set(NANOPB_SRC_ROOT_FOLDER "/path/to/nanopb")
+#   set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${NANOPB_SRC_ROOT_FOLDER}/cmake)
+#   find_package( Nanopb REQUIRED )
+#   include_directories(${NANOPB_INCLUDE_DIRS})
+#
+#   NANOPB_GENERATE_CPP(PROTO_SRCS PROTO_HDRS foo.proto)
+#
+#   include_directories(${CMAKE_CURRENT_BINARY_DIR})
+#   add_executable(bar bar.cc ${PROTO_SRCS} ${PROTO_HDRS})
+#
+#  ====================================================================
+
+#=============================================================================
+# Copyright 2009 Kitware, Inc.
+# Copyright 2009-2011 Philip Lowman <philip@yhbt.com>
+# Copyright 2008 Esben Mose Hansen, Ange Optimization ApS
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+#   notice, this list of conditions and the following disclaimer in the
+#   documentation and/or other materials provided with the distribution.
+#
+# * Neither the names of Kitware, Inc., the Insight Software Consortium,
+#   nor the names of their contributors may be used to endorse or promote
+#   products derived from this software without specific prior written
+#   permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#=============================================================================
+#
+# Changes
+# 2013.01.31 - Pavlo Ilin - used Modules/FindProtobuf.cmake from cmake 2.8.10 to
+#                           write FindNanopb.cmake
+#
+#=============================================================================
+
+
+function(NANOPB_GENERATE_CPP SRCS HDRS)
+  if(NOT ARGN)
+    return()
+  endif()
+
+  if(NANOPB_GENERATE_CPP_APPEND_PATH)
+    # Create an include path for each file specified
+    foreach(FIL ${ARGN})
+      get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+      get_filename_component(ABS_PATH ${ABS_FIL} PATH)
+
+      list(FIND _nanobp_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _nanobp_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  else()
+    set(_nanobp_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+
+  if(DEFINED NANOPB_IMPORT_DIRS)
+    foreach(DIR ${NANOPB_IMPORT_DIRS})
+      get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+      list(FIND _nanobp_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _nanobp_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  endif()
+
+  set(${SRCS})
+  set(${HDRS})
+  get_filename_component(GENERATOR_PATH ${NANOPB_GENERATOR_EXECUTABLE} PATH)
+
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb"
+      COMMAND  ${PROTOBUF_PROTOC_EXECUTABLE}
+      ARGS -I${GENERATOR_PATH} -I${CMAKE_CURRENT_BINARY_DIR} ${_nanobp_include_path} -o${FIL_WE}.pb ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM )
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      COMMAND python
+      ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb
+      DEPENDS ${FIL_WE}.pb
+      COMMENT "Running nanopb generator on ${FIL_WE}.pb"
+      VERBATIM )
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set(${SRCS} ${${SRCS}} ${NANOPB_SRCS} PARENT_SCOPE)
+  set(${HDRS} ${${HDRS}} ${NANOPB_HDRS} PARENT_SCOPE)
+
+endfunction()
+
+
+
+#
+# Main.
+#
+
+# By default have NANOPB_GENERATE_CPP macro pass -I to protoc
+# for each directory where a proto file is referenced.
+if(NOT DEFINED NANOPB_GENERATE_CPP_APPEND_PATH)
+  set(NANOPB_GENERATE_CPP_APPEND_PATH TRUE)
+endif()
+
+# Find the include directory
+find_path(NANOPB_INCLUDE_DIRS
+    pb.h
+    PATHS ${NANOPB_SRC_ROOT_FOLDER}
+)
+mark_as_advanced(NANOPB_INCLUDE_DIRS)
+
+# Find nanopb source files
+set(NANOPB_SRCS)
+set(NANOPB_HDRS)
+list(APPEND _nanopb_srcs pb_decode.c pb_encode.c)
+list(APPEND _nanopb_hdrs pb_decode.h pb_encode.h pb.h)
+
+foreach(FIL ${_nanopb_srcs})
+  find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_SRC_ROOT_FOLDER} ${NANOPB_INCLUDE_DIRS})
+  list(APPEND NANOPB_SRCS "${${FIL}__nano_pb_file}")
+  mark_as_advanced(${FIL}__nano_pb_file)
+endforeach()
+
+foreach(FIL ${_nanopb_hdrs})
+  find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_INCLUDE_DIRS})
+  mark_as_advanced(${FIL}__nano_pb_file)
+  list(APPEND NANOPB_HDRS "${${FIL}__nano_pb_file}")
+endforeach()
+
+# Find the protoc Executable
+find_program(PROTOBUF_PROTOC_EXECUTABLE
+    NAMES protoc
+    DOC "The Google Protocol Buffers Compiler"
+    PATHS
+    ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Release
+    ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Debug
+)
+mark_as_advanced(PROTOBUF_PROTOC_EXECUTABLE)
+
+# Find nanopb generator
+find_file(NANOPB_GENERATOR_EXECUTABLE
+    NAMES nanopb_generator.py
+    DOC "nanopb generator"
+    PATHS
+    ${NANOPB_SRC_ROOT_FOLDER}/generator
+)
+mark_as_advanced(NANOPB_GENERATOR_EXECUTABLE)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(NANOPB DEFAULT_MSG
+  NANOPB_INCLUDE_DIRS
+  NANOPB_SRCS NANOPB_HDRS
+  NANOPB_GENERATOR_EXECUTABLE
+  PROTOBUF_PROTOC_EXECUTABLE
+  )

compat/pb_syshdr.h

@@ -0,0 +1,94 @@
+/* This is an example of a header file for platforms/compilers that do
+ * not come with stdint.h/stddef.h/stdbool.h/string.h. To use it, define
+ * PB_SYSTEM_HEADER as "pb_syshdr.h", including the quotes, and add the
+ * compat folder to your include path.
+ *
+ * It is very likely that you will need to customize this file to suit
+ * your platform. For any compiler that supports C99, this file should
+ * not be necessary.
+ */
+
+#ifndef _PB_SYSHDR_H_
+#define _PB_SYSHDR_H_
+
+/* stdint.h subset */
+#ifdef HAVE_STDINT_H
+#include <stdint.h>
+#else
+/* You will need to modify these to match the word size of your platform. */
+typedef signed char int8_t;
+typedef unsigned char uint8_t;
+typedef signed short int16_t;
+typedef unsigned short uint16_t;
+typedef signed int int32_t;
+typedef unsigned int uint32_t;
+typedef signed long long int64_t;
+typedef unsigned long long uint64_t;
+#endif
+
+/* stddef.h subset */
+#ifdef HAVE_STDDEF_H
+#include <stddef.h>
+#else
+
+typedef uint32_t size_t;
+#define offsetof(st, m) ((size_t)(&((st *)0)->m))
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+#endif
+
+/* stdbool.h subset */
+#ifdef HAVE_STDBOOL_H
+#include <stdbool.h>
+#else
+
+#ifndef __cplusplus
+typedef int bool;
+#define false 0
+#define true 1
+#endif
+
+#endif
+
+/* string.h subset */
+#ifdef HAVE_STRING_H
+#include <string.h>
+#else
+
+/* Implementations are from the Public Domain C Library (PDCLib). */
+static size_t strlen( const char * s )
+{
+    size_t rc = 0;
+    while ( s[rc] )
+    {
+        ++rc;
+    }
+    return rc;
+}
+
+static void * memcpy( void *s1, const void *s2, size_t n )
+{
+    char * dest = (char *) s1;
+    const char * src = (const char *) s2;
+    while ( n-- )
+    {
+        *dest++ = *src++;
+    }
+    return s1;
+}
+
+static void * memset( void * s, int c, size_t n )
+{
+    unsigned char * p = (unsigned char *) s;
+    while ( n-- )
+    {
+        *p++ = (unsigned char) c;
+    }
+    return s;
+}
+#endif
+
+#endif

docs/concepts.rst

@@ -10,47 +10,40 @@ The things outlined here are the underlying concepts of the nanopb design.
 
 Proto files
 ===========
-All Protocol Buffers implementations use .proto files to describe the message format.
-The point of these files is to be a portable interface description language.
+All Protocol Buffers implementations use .proto files to describe the message
+format. The point of these files is to be a portable interface description
+language.
 
 Compiling .proto files for nanopb
 ---------------------------------
-Nanopb uses the Google's protoc compiler to parse the .proto file, and then a python script to generate the C header and source code from it::
+Nanopb uses the Google's protoc compiler to parse the .proto file, and then a
+python script to generate the C header and source code from it::
 
     user@host:~$ protoc -omessage.pb message.proto
     user@host:~$ python ../generator/nanopb_generator.py message.pb
     Writing to message.h and message.c
     user@host:~$
 
-Compiling .proto files with nanopb options
-------------------------------------------
-Nanopb defines two extensions for message fields described in .proto files: *max_size* and *max_count*.
-These are the maximum size of a string and maximum count of items in an array::
+Modifying generator behaviour
+-----------------------------
+Using generator options, you can set maximum sizes for fields in order to
+allocate them statically. The preferred way to do this is to create an .options
+file with the same name as your .proto file::
 
-    required string name = 1 [(nanopb).max_size = 40];
-    repeated PhoneNumber phone = 4 [(nanopb).max_count = 5];
-
-To use these extensions, you need to place an import statement in the beginning of the file::
-
-    import "nanopb.proto";
-
-This file, in turn, requires the file *google/protobuf/descriptor.proto*. This is usually installed under */usr/include*. Therefore, to compile a .proto file which uses options, use a protoc command similar to::
-
-    protoc -I/usr/include -Inanopb/generator -I. -omessage.pb message.proto
-
-The options can be defined in file, message and field scopes::
-
-    option (nanopb_fileopt).max_size = 20; // File scope
-    message Message
-    {
-        option (nanopb_msgopt).max_size = 30; // Message scope
-        required string fieldsize = 1 [(nanopb).max_size = 40]; // Field scope
+   # Foo.proto
+   message Foo {
+      required string name = 1;
    }
 
-It is also possible to give the options on command line, but then they will affect the whole file. For example::
+::
 
-    user@host:~$ python ../generator/nanopb_generator.py -s 'max_size: 20' message.pb
+   # Foo.options
+   Foo.name max_size:16
 
+For more information on this, see the `Proto file options`_ section in the
+reference manual.
+
+.. _`Proto file options`: reference.html#proto-file-options
 
 Streams
 =======
@@ -181,7 +174,9 @@ Field callbacks
 ===============
 When a field has dynamic length, nanopb cannot statically allocate storage for it. Instead, it allows you to handle the field in whatever way you want, using a callback function.
 
-The `pb_callback_t`_ structure contains a function pointer and a *void* pointer you can use for passing data to the callback. If the function pointer is NULL, the field will be skipped. The actual behavior of the callback function is different in encoding and decoding modes.
+The `pb_callback_t`_ structure contains a function pointer and a *void* pointer called *arg* you can use for passing data to the callback. If the function pointer is NULL, the field will be skipped. A pointer to the *arg* is passed to the function, so that it can modify it and retrieve the value.
+
+The actual behavior of the callback function is different in encoding and decoding modes. In encoding mode, the callback is called once and should write out everything, including field tags. In decoding mode, the callback is called repeatedly for every data item.
 
 .. _`pb_callback_t`: reference.html#pb-callback-t
 
@@ -189,7 +184,7 @@ Encoding callbacks
 ------------------
 ::
 
-    bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, const void *arg);
+    bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, void * const *arg);
 
 When encoding, the callback should write out complete fields, including the wire type and field number tag. It can write as many or as few fields as it likes. For example, if you want to write out an array as *repeated* field, you should do it all in a single call.
 
@@ -203,7 +198,7 @@ If the callback is used in a submessage, it will be called multiple times during
 
 This callback writes out a dynamically sized string::
 
-    bool write_string(pb_ostream_t *stream, const pb_field_t *field, const void *arg)
+    bool write_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
     {
         char *str = get_string_from_somewhere();
         if (!pb_encode_tag_for_field(stream, field))
@@ -216,7 +211,7 @@ Decoding callbacks
 ------------------
 ::
 
-    bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg);
+    bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void **arg);
 
 When decoding, the callback receives a length-limited substring that reads the contents of a single field. The field tag has already been read. For *string* and *bytes*, the length value has already been parsed, and is available at *stream->bytes_left*.
 
@@ -226,7 +221,7 @@ The callback will be called multiple times for repeated fields. For packed field
 
 This callback reads multiple integers and prints them::
 
-    bool read_ints(pb_istream_t *stream, const pb_field_t *field, void *arg)
+    bool read_ints(pb_istream_t *stream, const pb_field_t *field, void **arg)
     {
         while (stream->bytes_left)
         {
@@ -255,20 +250,56 @@ For example this submessage in the Person.proto file::
 generates this field description array for the structure *Person_PhoneNumber*::
 
  const pb_field_t Person_PhoneNumber_fields[3] = {
-    {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING,
-    offsetof(Person_PhoneNumber, number), 0,
-    pb_membersize(Person_PhoneNumber, number), 0, 0},
-
-    {2, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT,
-    pb_delta(Person_PhoneNumber, type, number),
-    pb_delta(Person_PhoneNumber, has_type, type),
-    pb_membersize(Person_PhoneNumber, type), 0,
-    &Person_PhoneNumber_type_default},
-
+    PB_FIELD(  1, STRING  , REQUIRED, STATIC, Person_PhoneNumber, number, number, 0),
+    PB_FIELD(  2, ENUM    , OPTIONAL, STATIC, Person_PhoneNumber, type, number, &Person_PhoneNumber_type_default),
     PB_LAST_FIELD
  };
 
 
+Extension fields
+================
+Protocol Buffers supports a concept of `extension fields`_, which are
+additional fields to a message, but defined outside the actual message.
+The definition can even be in a completely separate .proto file.
+
+The base message is declared as extensible by keyword *extensions* in
+the .proto file::
+
+ message MyMessage {
+     .. fields ..
+     extensions 100 to 199;
+ }
+
+For each extensible message, *nanopb_generator.py* declares an additional
+callback field called *extensions*. The field and associated datatype
+*pb_extension_t* forms a linked list of handlers. When an unknown field is
+encountered, the decoder calls each handler in turn until either one of them
+handles the field, or the list is exhausted.
+
+The actual extensions are declared using the *extend* keyword in the .proto,
+and are in the global namespace::
+
+ extend MyMessage {
+     optional int32 myextension = 100;
+ }
+
+For each extension, *nanopb_generator.py* creates a constant of type
+*pb_extension_type_t*. To link together the base message and the extension,
+you have to:
+
+1. Allocate storage for your field, matching the datatype in the .proto.
+   For example, for a *int32* field, you need a *int32_t* variable to store
+   the value.
+2. Create a *pb_extension_t* constant, with pointers to your variable and
+   to the generated *pb_extension_type_t*.
+3. Set the *message.extensions* pointer to point to the *pb_extension_t*.
+
+An example of this is available in *tests/test_encode_extensions.c* and
+*tests/test_decode_extensions.c*.
+
+.. _`extension fields`: https://developers.google.com/protocol-buffers/docs/proto#extensions
+
+
 Return values and error handling
 ================================
 
@@ -276,8 +307,8 @@ Most functions in nanopb return bool: *true* means success, *false* means failur
 
 The error messages help in guessing what is the underlying cause of the error. The most common error conditions are:
 
-1) Running out of memory. Because everything is allocated from the stack, nanopb can't detect this itself. Encoding or decoding the same type of a message always takes the same amount of stack space. Therefore, if it works once, it works always.
-2) Invalid field description. These are usually stored as constants, so if it works under the debugger, it always does.
+1) Running out of memory, i.e. stack overflow.
+2) Invalid field descriptors (would usually mean a bug in the generator).
 3) IO errors in your own stream callbacks.
 4) Errors that happen in your callback functions.
 5) Exceeding the max_size or bytes_left of a stream.

docs/index.rst

@@ -36,23 +36,26 @@ Features and limitations
 **Features**
 
 #) Pure C runtime
-#) Small code size (2–10 kB depending on processor)
-#) Small ram usage (typically 200 bytes)
+#) Small code size (2–10 kB depending on processor, plus any message definitions)
+#) Small ram usage (typically ~300 bytes, plus any message structs)
 #) Allows specifying maximum size for strings and arrays, so that they can be allocated statically.
 #) No malloc needed: everything can be allocated statically or on the stack.
 #) You can use either encoder or decoder alone to cut the code size in half.
+#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, packed arrays, extension fields.
+#) Callback mechanism for handling messages larger than can fit in available RAM.
+#) Extensive set of tests.
 
 **Limitations**
 
 #) User must provide callbacks when decoding arrays or strings without maximum size. Malloc support could be added as a separate module.
-#) Some speed has been sacrificed for code size. For example varint calculations are always done in 64 bits.
+#) Some speed has been sacrificed for code size.
 #) Encoding is focused on writing to streams. For memory buffers only it could be made more efficient.
 #) The deprecated Protocol Buffers feature called "groups" is not supported.
 #) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file.
 #) Unknown fields are not preserved when decoding and re-encoding a message.
 #) Reflection (runtime introspection) is not supported. E.g. you can't request a field by giving its name in a string.
 #) Numeric arrays are always encoded as packed, even if not marked as packed in .proto. This causes incompatibility with decoders that do not support packed format.
-#) Cyclic references between messages are not supported. They could be supported in callback-mode if there was an option in the generator to set the mode.
+#) Cyclic references between messages are supported only in callback mode.
 
 Getting started
 ===============
@@ -100,14 +103,9 @@ Nanopb should compile with most ansi-C compatible compilers. It however requires
 
 If these header files do not come with your compiler, you should be able to find suitable replacements online. Mostly the requirements are very simple, just a few basic functions and typedefs.
 
+Alternatively, you can define *PB_SYSTEM_HEADER*, which should be the name of a single header file including all the necessary definitions.
+
 Debugging and testing
 =====================
 Extensive unittests are included under the *tests* folder. Just type *make* there to run the tests.
 
-This also generates a file called *breakpoints* which includes all lines returning *false* in nanopb. You can use this in gdb by typing *source breakpoints*, after which gdb will break on first nanopb error.
-
-Wishlist
-========
-#) A specialized encoder for encoding to a memory buffer. Should serialize in reverse order to avoid having to determine submessage size beforehand.
-#) A cleaner rewrite of the Python-based source generator.
-#) Better performance for 16- and 8-bit platforms: use smaller datatypes where possible.

docs/reference.rst

@@ -6,27 +6,187 @@ Nanopb: API reference
 
 .. contents ::
 
+
+
+
 Compilation options
 ===================
-The following options can be specified using -D switch given to the C compiler:
+The following options can be specified in one of two ways:
+
+1. Using the -D switch on the C compiler command line.
+2. By #defining them at the top of pb.h.
+
+You must have the same settings for the nanopb library and all code that
+includes pb.h.
+
+============================  ================================================
+__BIG_ENDIAN__                 Set this if your platform stores integers and
+                               floats in big-endian format. Mixed-endian
+                               systems (different layout for ints and floats)
+                               are currently not supported.
+NANOPB_INTERNALS               Set this to expose the field encoder functions
+                               that are hidden since nanopb-0.1.3.
+PB_MAX_REQUIRED_FIELDS         Maximum number of required fields to check for
+                               presence. Default value is 64. Increases stack
+                               usage 1 byte per every 8 fields. Compiler
+                               warning will tell if you need this.
+PB_FIELD_16BIT                 Add support for tag numbers > 255 and fields
+                               larger than 255 bytes or 255 array entries.
+                               Increases code size 3 bytes per each field.
+                               Compiler error will tell if you need this.
+PB_FIELD_32BIT                 Add support for tag numbers > 65535 and fields
+                               larger than 65535 bytes or 65535 array entries.
+                               Increases code size 9 bytes per each field.
+                               Compiler error will tell if you need this.
+PB_NO_ERRMSG                   Disables the support for error messages; only
+                               error information is the true/false return
+                               value. Decreases the code size by a few hundred
+                               bytes.
+PB_BUFFER_ONLY                 Disables the support for custom streams. Only
+                               supports encoding and decoding with memory
+                               buffers. Speeds up execution and decreases code
+                               size slightly.
+PB_OLD_CALLBACK_STYLE          Use the old function signature (void\* instead
+                               of void\*\*) for callback fields. This was the
+                               default until nanopb-0.2.1.
+PB_SYSTEM_HEADER               Replace the standard header files with a single
+                               header file. It should define all the required
+                               functions and typedefs listed on the
+                               `overview page`_. Value must include quotes,
+                               for example *#define PB_SYSTEM_HEADER "foo.h"*.
+============================  ================================================
+
+The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow
+raising some datatype limits to suit larger messages. Their need is recognized
+automatically by C-preprocessor #if-directives in the generated .pb.h files.
+The default setting is to use the smallest datatypes (least resources used).
+
+.. _`overview page`: index.html#compiler-requirements
+
+
+Proto file options
+==================
+The generator behaviour can be adjusted using these options, defined in the
+'nanopb.proto' file in the generator folder:
+
+============================  ================================================
+max_size                       Allocated size for *bytes* and *string* fields.
+max_count                      Allocated number of entries in arrays
+                               (*repeated* fields).
+type                           Type of the generated field. Default value
+                               is *FT_DEFAULT*, which selects automatically.
+                               You can use *FT_CALLBACK*, *FT_STATIC* or
+                               *FT_IGNORE* to force a callback field, a static
+                               field or to completely ignore the field.
+long_names                     Prefix the enum name to the enum value in
+                               definitions, i.e. *EnumName_EnumValue*. Enabled
+                               by default.
+packed_struct                  Make the generated structures packed.
+                               NOTE: This cannot be used on CPUs that break
+                               on unaligned accesses to variables.
+============================  ================================================
+
+These options can be defined for the .proto files before they are converted
+using the nanopb-generatory.py. There are three ways to define the options:
+
+1. Using a separate .options file.
+   This is the preferred way as of nanopb-0.2.1, because it has the best
+   compatibility with other protobuf libraries.
+2. Defining the options on the command line of nanopb_generator.py.
+   This only makes sense for settings that apply to a whole file.
+3. Defining the options in the .proto file using the nanopb extensions.
+   This is the way used in nanopb-0.1, and will remain supported in the
+   future. It however sometimes causes trouble when using the .proto file
+   with other protobuf libraries.
+
+The effect of the options is the same no matter how they are given. The most
+common purpose is to define maximum size for string fields in order to
+statically allocate them.
+
+Defining the options in a .options file
+---------------------------------------
+The preferred way to define options is to have a separate file
+'myproto.options' in the same directory as the 'myproto.proto'. ::
+
+    # myproto.proto
+    message MyMessage {
+        required string name = 1;
+        repeated int32 ids = 4;
+    }
+
+::
+
+    # myproto.options
+    MyMessage.name         max_size:40
+    MyMessage.ids          max_count:5
+
+The generator will automatically search for this file and read the
+options from it. The file format is as follows:
+
+* Lines starting with '#' or '//' are regarded as comments.
+* Blank lines are ignored.
+* All other lines should start with a field name pattern, followed by one or
+  more options. For example: *"MyMessage.myfield max_size:5 max_count:10"*.
+* The field name pattern is matched against a string of form *'Message.field'*.
+  For nested messages, the string is *'Message.SubMessage.field'*.
+* The field name pattern may use the notation recognized by Python fnmatch():
+
+  - *\** matches any part of string, like 'Message.\*' for all fields
+  - *\?* matches any single character
+  - *[seq]* matches any of characters 's', 'e' and 'q'
+  - *[!seq]* matches any other character
+
+* The options are written as *'option_name:option_value'* and several options
+  can be defined on same line, separated by whitespace.
+* Options defined later in the file override the ones specified earlier, so
+  it makes sense to define wildcard options first in the file and more specific
+  ones later.
+  
+If preferred, the name of the options file can be set using the command line
+switch *-f* to nanopb_generator.py.
+
+Defining the options on command line
+------------------------------------
+The nanopb_generator.py has a simple command line option *-s OPTION:VALUE*.
+The setting applies to the whole file that is being processed.
+
+Defining the options in the .proto file
+---------------------------------------
+The .proto file format allows defining custom options for the fields.
+The nanopb library comes with *nanopb.proto* which does exactly that, allowing
+you do define the options directly in the .proto file::
+
+    import "nanopb.proto";
+    
+    message MyMessage {
+        required string name = 1 [(nanopb).max_size = 40];
+        repeated int32 ids = 4   [(nanopb).max_count = 5];
+    }
+
+A small complication is that you have to set the include path of protoc so that
+nanopb.proto can be found. This file, in turn, requires the file
+*google/protobuf/descriptor.proto*. This is usually installed under
+*/usr/include*. Therefore, to compile a .proto file which uses options, use a
+protoc command similar to::
+
+    protoc -I/usr/include -Inanopb/generator -I. -omessage.pb message.proto
+
+The options can be defined in file, message and field scopes::
+
+    option (nanopb_fileopt).max_size = 20; // File scope
+    message Message
+    {
+        option (nanopb_msgopt).max_size = 30; // Message scope
+        required string fieldsize = 1 [(nanopb).max_size = 40]; // Field scope
+    }
+
+
+
+
+
+
 
-============================  ================================================================================================
-__BIG_ENDIAN__                 Set this if your platform stores integers and floats in big-endian format.
-                               Mixed-endian systems (different layout for ints and floats) are currently not supported.
-NANOPB_INTERNALS               Set this to expose the field encoder functions that are hidden since nanopb-0.1.3.
-PB_MAX_REQUIRED_FIELDS         Maximum number of required fields to check for presence. Default value is 64. Increases stack
-                               usage 1 byte per every 8 fields. Compiler warning will tell if you need this.
-PB_FIELD_16BIT                 Add support for tag numbers > 255 and fields larger than 255 bytes or 255 array entries.
-                               Increases code size 3 bytes per each field. Compiler error will tell if you need this.
-PB_FIELD_32BIT                 Add support for tag numbers > 65535 and fields larger than 65535 bytes or 65535 array entries.
-                               Increases code size 9 bytes per each field. Compiler error will tell if you need this.
-PB_NO_ERRMSG                   Disables the support for error messages; only error information is the true/false return value.
-                               Decreases the code size by a few hundred bytes.
-============================  ================================================================================================
 
-The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow raising some datatype limits to suit larger messages.
-Their need is recognized automatically by C-preprocessor #if-directives in the generated .pb.h files. The default setting is to use
-the smallest datatypes (least resources used).
 
 pb.h
 ====
@@ -35,22 +195,23 @@ pb_type_t
 ---------
 Defines the encoder/decoder behaviour that should be used for a field. ::
 
-    typedef enum { ... } pb_type_t;
+    typedef uint8_t pb_type_t;
 
-The low-order byte of the enumeration values defines the function that can be used for encoding and decoding the field data:
+The low-order nibble of the enumeration values defines the function that can be used for encoding and decoding the field data:
 
 ==================== ===== ================================================
 LTYPE identifier     Value Storage format
 ==================== ===== ================================================
 PB_LTYPE_VARINT      0x00  Integer.
 PB_LTYPE_SVARINT     0x01  Integer, zigzag encoded.
-PB_LTYPE_FIXED       0x02  Integer or floating point.
-PB_LTYPE_BYTES       0x03  Structure with *size_t* field and byte array.
-PB_LTYPE_STRING      0x04  Null-terminated string.
-PB_LTYPE_SUBMESSAGE  0x05  Submessage structure.
+PB_LTYPE_FIXED32     0x02  32-bit integer or floating point.
+PB_LTYPE_FIXED64     0x03  64-bit integer or floating point.
+PB_LTYPE_BYTES       0x04  Structure with *size_t* field and byte array.
+PB_LTYPE_STRING      0x05  Null-terminated string.
+PB_LTYPE_SUBMESSAGE  0x06  Submessage structure.
 ==================== ===== ================================================
 
-The high-order byte defines whether the field is required, optional, repeated or callback:
+The bits 4-5 define whether the field is required, optional or repeated:
 
 ==================== ===== ================================================
 HTYPE identifier     Value Field handling
@@ -58,13 +219,24 @@ HTYPE identifier     Value Field handling
 PB_HTYPE_REQUIRED    0x00  Verify that field exists in decoded message.
 PB_HTYPE_OPTIONAL    0x10  Use separate *has_<field>* boolean to specify
                            whether the field is present.
-PB_HTYPE_ARRAY       0x20  A repeated field with preallocated array.
+                           (Unless it is a callback)
+PB_HTYPE_REPEATED    0x20  A repeated field with preallocated array.
                            Separate *<field>_count* for number of items.
-PB_HTYPE_CALLBACK    0x30  A field with dynamic storage size, data is
-                           actually a pointer to a structure containing a
-                           callback function.
+                           (Unless it is a callback)
 ==================== ===== ================================================
 
+The bits 6-7 define the how the storage for the field is allocated:
+
+==================== ===== ================================================
+ATYPE identifier     Value Allocation method
+==================== ===== ================================================
+PB_ATYPE_STATIC      0x00  Statically allocated storage in the structure.
+PB_ATYPE_CALLBACK    0x40  A field with dynamic storage size. Struct field
+                           actually contains a pointer to a callback
+                           function.
+==================== ===== ================================================
+
+
 pb_field_t
 ----------
 Describes a single structure field with memory position in relation to others. The descriptions are usually autogenerated. ::
@@ -81,7 +253,7 @@ Describes a single structure field with memory position in relation to others. T
     } pb_packed;
 
 :tag:           Tag number of the field or 0 to terminate a list of fields.
-:type:          LTYPE and HTYPE of the field.
+:type:          LTYPE, HTYPE and ATYPE of the field.
 :data_offset:   Offset of field data, relative to the end of the previous field.
 :size_offset:   Offset of *bool* flag for optional fields or *size_t* count for arrays, relative to field data.
 :data_size:     Size of a single data entry, in bytes. For PB_LTYPE_BYTES, the size of the byte array inside the containing structure. For PB_HTYPE_CALLBACK, size of the C data type if known.
@@ -108,14 +280,16 @@ Part of a message structure, for fields with type PB_HTYPE_CALLBACK::
     typedef struct _pb_callback_t pb_callback_t;
     struct _pb_callback_t {
         union {
-            bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg);
-            bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, const void *arg);
+            bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void **arg);
+            bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, void * const *arg);
         } funcs;
         
         void *arg;
     };
 
-The *arg* is passed to the callback when calling. It can be used to store any information that the callback might need.
+A pointer to the *arg* is passed to the callback when calling. It can be used to store any information that the callback might need.
+
+Previously the function received just the value of *arg* instead of a pointer to it. This old behaviour can be enabled by defining *PB_OLD_CALLBACK_STYLE*.
 
 When calling `pb_encode`_, *funcs.encode* is used, and similarly when calling `pb_decode`_, *funcs.decode* is used. The function pointers are stored in the same memory location but are of incompatible types. You can set the function pointer to NULL to skip the field.
 
@@ -130,6 +304,76 @@ Protocol Buffers wire types. These are used with `pb_encode_tag`_. ::
         PB_WT_32BIT  = 5
     } pb_wire_type_t;
 
+pb_extension_type_t
+-------------------
+Defines the handler functions and auxiliary data for a field that extends
+another message. Usually autogenerated by *nanopb_generator.py*::
+
+    typedef struct {
+        bool (*decode)(pb_istream_t *stream, pb_extension_t *extension,
+                   uint32_t tag, pb_wire_type_t wire_type);
+        bool (*encode)(pb_ostream_t *stream, const pb_extension_t *extension);
+        const void *arg;
+    } pb_extension_type_t;
+
+In the normal case, the function pointers are *NULL* and the decoder and
+encoder use their internal implementations. The internal implementations
+assume that *arg* points to a *pb_field_t* that describes the field in question.
+
+To implement custom processing of unknown fields, you can provide pointers
+to your own functions. Their functionality is mostly the same as for normal
+callback fields, except that they get called for any unknown field when decoding.
+
+pb_extension_t
+--------------
+Ties together the extension field type and the storage for the field value::
+
+    typedef struct {
+        const pb_extension_type_t *type;
+        void *dest;
+        pb_extension_t *next;
+    } pb_extension_t;
+
+:type:      Pointer to the structure that defines the callback functions.
+:dest:      Pointer to the variable that stores the field value
+            (as used by the default extension callback functions.)
+:next:      Pointer to the next extension handler, or *NULL*.
+
+PB_GET_ERROR
+------------
+Get the current error message from a stream, or a placeholder string if
+there is no error message::
+
+    #define PB_GET_ERROR(stream) (string expression)
+
+This should be used for printing errors, for example::
+
+    if (!pb_decode(...))
+    {
+        printf("Decode failed: %s\n", PB_GET_ERROR(stream));
+    }
+
+The macro only returns pointers to constant strings (in code memory),
+so that there is no need to release the returned pointer.
+
+PB_RETURN_ERROR
+---------------
+Set the error message and return false::
+
+    #define PB_RETURN_ERROR(stream,msg) (sets error and returns false)
+
+This should be used to handle error conditions inside nanopb functions
+and user callback functions::
+
+    if (error_condition)
+    {
+        PB_RETURN_ERROR(stream, "something went wrong");
+    }
+
+The *msg* parameter must be a constant string.
+
+
+
 pb_encode.h
 ===========
 
@@ -279,6 +523,17 @@ In Protocol Buffers format, the submessage size must be written before the subme
 
 If the submessage contains callback fields, the callback function might misbehave and write out a different amount of data on the second call. This situation is recognized and *false* is returned, but garbage will be written to the output before the problem is detected.
 
+
+
+
+
+
+
+
+
+
+
+
 pb_decode.h
 ===========
 
@@ -381,7 +636,7 @@ Remove the data for a field from the stream, without actually decoding it::
 
     For decoding numeric (including enumerated and boolean) values, use `pb_decode_varint`_, `pb_decode_svarint`_, `pb_decode_fixed32`_ and `pb_decode_fixed64`_. They take a pointer to a 32- or 64-bit C variable, which you may then cast to smaller datatype for storage.
 
-    For decoding strings and bytes fields, the length has already been decoded. You can therefore check the total length in *stream->state* and read the data using `pb_read`_.
+    For decoding strings and bytes fields, the length has already been decoded. You can therefore check the total length in *stream->bytes_left* and read the data using `pb_read`_.
 
     Finally, for decoding submessages in a callback, simply use `pb_decode`_ and pass it the *SubMessage_fields* descriptor array.
 

example/client.c

@@ -23,7 +23,7 @@
 #include "fileproto.pb.h"
 #include "common.h"
 
-bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void *arg)
+bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void **arg)
 {
     FileInfo fileinfo;
     

example/fileproto.options

@@ -0,0 +1,13 @@
+# This file defines the nanopb-specific options for the messages defined
+# in fileproto.proto.
+#
+# If you come from high-level programming background, the hardcoded
+# maximum lengths may disgust you. However, if your microcontroller only
+# has a few kB of ram to begin with, setting reasonable limits for
+# filenames is ok.
+#
+# On the other hand, using the callback interface, it is not necessary
+# to set a limit on the number of files in the response.
+
+ListFilesRequest.path 	max_size:128
+FileInfo.name 		max_size:128

example/fileproto.proto

@@ -1,22 +1,14 @@
-import "nanopb.proto";
-
 // This defines protocol for a simple server that lists files.
 //
-// If you come from high-level programming background, the hardcoded
-// maximum lengths may disgust you. However, if your microcontroller only
-// has a few kB of ram to begin with, setting reasonable limits for
-// filenames is ok.
-//
-// On the other hand, using the callback interface, it is not necessary
-// to set a limit on the number of files in the response.
+// See also the nanopb-specific options in fileproto.options.
 
 message ListFilesRequest {
-    optional string path = 1 [default = "/", (nanopb).max_size = 128];
+    optional string path = 1 [default = "/"];
 }
 
 message FileInfo {
     required uint64 inode = 1;
-    required string name = 2 [(nanopb).max_size = 128];
+    required string name = 2;
 }
 
 message ListFilesResponse {

example/server.c

@@ -23,9 +23,9 @@
 #include "fileproto.pb.h"
 #include "common.h"
 
-bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, const void *arg)
+bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
 {
-    DIR *dir = (DIR*) arg;
+    DIR *dir = (DIR*) *arg;
     struct dirent *file;
     FileInfo fileinfo;
     

example_avr_double/Makefile

@@ -0,0 +1,22 @@
+CFLAGS=-Wall -Werror -I .. -g -O0
+DEPS=double_conversion.c ../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h
+
+all: run_tests
+
+clean:
+	rm -f test_conversions encode_double decode_double doubleproto.pb.c doubleproto.pb.h
+
+test_conversions: test_conversions.c double_conversion.c
+	$(CC) $(CFLAGS) -o $@ $^
+
+%: %.c $(DEPS) doubleproto.pb.h doubleproto.pb.c
+	$(CC) $(CFLAGS) -o $@ $< double_conversion.c ../pb_decode.c ../pb_encode.c doubleproto.pb.c
+
+doubleproto.pb.c doubleproto.pb.h: doubleproto.proto ../generator/nanopb_generator.py
+	protoc -I. -I../generator -I/usr/include -odoubleproto.pb $<
+	python ../generator/nanopb_generator.py doubleproto.pb
+
+run_tests: test_conversions encode_double decode_double
+	./test_conversions
+	./encode_double | ./decode_double
+    

example_avr_double/README.txt

@@ -0,0 +1,22 @@
+Some processors/compilers, such as AVR-GCC, do not support the double
+datatype. Instead, they have sizeof(double) == 4. Because protocol
+binary format uses the double encoding directly, this causes trouble
+if the protocol in .proto requires double fields.
+
+This directory contains a solution to this problem. It uses uint64_t
+to store the raw wire values, because its size is correct on all
+platforms. The file double_conversion.c provides functions that
+convert these values to/from floats, without relying on compiler
+support.
+
+To use this method, you need to make two modifications to your code:
+
+1) Change all 'double' fields into 'fixed64' in the .proto.
+
+2) Whenever writing to a 'double' field, use float_to_double().
+
+3) Whenever reading a 'double' field, use double_to_float().
+
+The conversion routines should be as accurate as the float datatype can
+be. Furthermore, they should handle all special values (NaN, inf, denormalized
+numbers) correctly. There are testcases in test_conversions.c.

example_avr_double/decode_double.c

@@ -0,0 +1,33 @@
+/* Decodes a double value into a float variable.
+ * Used to read double values with AVR code, which doesn't support double directly.
+ */
+
+#include <stdio.h>
+#include <pb_decode.h>
+#include "double_conversion.h"
+#include "doubleproto.pb.h"
+
+int main()
+{
+    uint8_t buffer[32];
+    size_t count = fread(buffer, 1, sizeof(buffer), stdin);
+    pb_istream_t stream = pb_istream_from_buffer(buffer, count);
+    
+    AVRDoubleMessage message;
+    pb_decode(&stream, AVRDoubleMessage_fields, &message);
+    
+    float v1 = double_to_float(message.field1);
+    float v2 = double_to_float(message.field2);
+
+    printf("Values: %f %f\n", v1, v2);
+    
+    if (v1 == 1234.5678f &&
+        v2 == 0.00001f)
+    {
+        return 0;
+    }
+    else
+    {
+        return 1;
+    }
+}

example_avr_double/double_conversion.c

@@ -0,0 +1,123 @@
+/* Conversion routines for platforms that do not support 'double' directly. */
+
+#include "double_conversion.h"
+#include <math.h>
+
+typedef union {
+    float f;
+    uint32_t i;
+} conversion_t;
+
+/* Note: IEE 754 standard specifies float formats as follows:
+ * Single precision: sign,  8-bit exp, 23-bit frac.
+ * Double precision: sign, 11-bit exp, 52-bit frac.
+ */
+
+uint64_t float_to_double(float value)
+{
+    conversion_t in;
+    in.f = value;
+    uint8_t sign;
+    int16_t exponent;
+    uint64_t mantissa;
+    
+    /* Decompose input value */
+    sign = (in.i >> 31) & 1;
+    exponent = ((in.i >> 23) & 0xFF) - 127;
+    mantissa = in.i & 0x7FFFFF;
+    
+    if (exponent == 128)
+    {
+        /* Special value (NaN etc.) */
+        exponent = 1024;
+    }
+    else if (exponent == -127)
+    {
+        if (!mantissa)
+        {
+            /* Zero */
+            exponent = -1023;
+        }
+        else
+        {
+            /* Denormalized */
+            mantissa <<= 1;
+            while (!(mantissa & 0x800000))
+            {
+                mantissa <<= 1;
+                exponent--;
+            }
+            mantissa &= 0x7FFFFF;
+        }
+    }
+    
+    /* Combine fields */
+    mantissa <<= 29;
+    mantissa |= (uint64_t)(exponent + 1023) << 52;
+    mantissa |= (uint64_t)sign << 63;
+    
+    return mantissa;
+}
+
+float double_to_float(uint64_t value)
+{
+    uint8_t sign;
+    int16_t exponent;
+    uint32_t mantissa;
+    conversion_t out;
+
+    /* Decompose input value */
+    sign = (value >> 63) & 1;
+    exponent = ((value >> 52) & 0x7FF) - 1023;
+    mantissa = (value >> 28) & 0xFFFFFF; /* Highest 24 bits */
+ 
+    /* Figure if value is in range representable by floats. */
+    if (exponent == 1024)
+    {
+        /* Special value */
+        exponent = 128;
+    }
+    else if (exponent > 127)
+    {
+        /* Too large */        
+        if (sign)
+            return -INFINITY;
+        else
+            return INFINITY;
+    }
+    else if (exponent < -150)
+    {
+        /* Too small */
+        if (sign)
+            return -0.0f;
+        else
+            return 0.0f;
+    }
+    else if (exponent < -126)
+    {
+        /* Denormalized */
+        mantissa |= 0x1000000;
+        mantissa >>= (-126 - exponent);
+        exponent = -127;
+    }
+ 
+    /* Round off mantissa */
+    mantissa = (mantissa + 1) >> 1;
+    
+    /* Check if mantissa went over 2.0 */
+    if (mantissa & 0x800000)
+    {
+        exponent += 1;
+        mantissa &= 0x7FFFFF;
+        mantissa >>= 1;
+    }
+    
+    /* Combine fields */
+    out.i = mantissa;
+    out.i |= (uint32_t)(exponent + 127) << 23;
+    out.i |= (uint32_t)sign << 31;
+    
+    return out.f;
+}
+
+

example_avr_double/double_conversion.h

@@ -0,0 +1,26 @@
+/* AVR-GCC does not have real double datatype. Instead its double
+ * is equal to float, i.e. 32 bit value. If you need to communicate
+ * with other systems that use double in their .proto files, you
+ * need to do some conversion.
+ *
+ * These functions use bitwise operations to mangle floats into doubles
+ * and then store them in uint64_t datatype.
+ */
+
+#ifndef DOUBLE_CONVERSION
+#define DOUBLE_CONVERSION
+
+#include <stdint.h>
+
+/* Convert native 4-byte float into a 8-byte double. */
+extern uint64_t float_to_double(float value);
+
+/* Convert 8-byte double into native 4-byte float.
+ * Values are rounded to nearest, 0.5 away from zero.
+ * Overflowing values are converted to Inf or -Inf.
+ */
+extern float double_to_float(uint64_t value);
+
+
+#endif
+

example_avr_double/doubleproto.proto

@@ -0,0 +1,13 @@
+// A message containing doubles, as used by other applications.
+message DoubleMessage {
+    required double field1 = 1;
+    required double field2 = 2;
+}
+
+// A message containing doubles, but redefined using uint64_t.
+// For use in AVR code.
+message AVRDoubleMessage {
+    required fixed64 field1 = 1;
+    required fixed64 field2 = 2;
+}
+

example_avr_double/encode_double.c

@@ -0,0 +1,25 @@
+/* Encodes a float value into a double on the wire.
+ * Used to emit doubles from AVR code, which doesn't support double directly.
+ */
+
+#include <stdio.h>
+#include <pb_encode.h>
+#include "double_conversion.h"
+#include "doubleproto.pb.h"
+
+int main()
+{
+    AVRDoubleMessage message = {
+        float_to_double(1234.5678f),
+        float_to_double(0.00001f)
+    };
+    
+    uint8_t buffer[32];
+    pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+    
+    pb_encode(&stream, AVRDoubleMessage_fields, &message);
+    fwrite(buffer, 1, stream.bytes_written, stdout);
+
+    return 0;
+}
+

example_avr_double/test_conversions.c

@@ -0,0 +1,56 @@
+#include "double_conversion.h"
+#include <math.h>
+#include <stdio.h>
+
+static const double testvalues[] = {
+           0.0,        -0.0,         0.1,         -0.1,
+          M_PI,       -M_PI,  123456.789,  -123456.789,
+      INFINITY,   -INFINITY,         NAN, INFINITY - INFINITY,
+          1e38,       -1e38,        1e39,        -1e39,
+         1e-38,      -1e-38,       1e-39,       -1e-39,
+   3.14159e-37,-3.14159e-37, 3.14159e-43, -3.14159e-43,
+         1e-60,      -1e-60,       1e-45,       -1e-45,
+    0.99999999999999, -0.99999999999999, 127.999999999999, -127.999999999999
+};
+
+#define TESTVALUES_COUNT (sizeof(testvalues)/sizeof(testvalues[0]))
+
+int main()
+{
+    int status = 0;
+    int i;
+    for (i = 0; i < TESTVALUES_COUNT; i++)
+    {
+        double orig = testvalues[i];
+        float expected_float = (float)orig;
+        double expected_double = (double)expected_float;
+        
+        float got_float = double_to_float(*(uint64_t*)&orig);
+        uint64_t got_double = float_to_double(got_float);
+        
+        uint32_t e1 = *(uint32_t*)&expected_float;
+        uint32_t g1 = *(uint32_t*)&got_float;
+        uint64_t e2 = *(uint64_t*)&expected_double;
+        uint64_t g2 = got_double;
+        
+        if (g1 != e1)
+        {
+            printf("%3d double_to_float fail: %08x != %08x\n", i, g1, e1);
+            status = 1;
+        }
+        
+        if (g2 != e2)
+        {
+            printf("%3d float_to_double fail: %016llx != %016llx\n", i,
+                (unsigned long long)g2,
+                (unsigned long long)e2);
+            status = 1;
+        }
+    }
+
+    return status;
+}
+
+    
+    
+

generator/nanopb.proto

@@ -7,12 +7,18 @@
 
 import "google/protobuf/descriptor.proto";
 
+option java_package = "fi.kapsi.koti.jpa.nanopb";
+
 enum FieldType {
     FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible.
     FT_CALLBACK = 1; // Always generate a callback field.
     FT_STATIC = 2; // Generate a static field or raise an exception if not possible.
+    FT_IGNORE = 3; // Ignore the field completely.
 }
 
+// This is the inner options message, which basically defines options for
+// a field. When it is used in message or file scope, it applies to all
+// fields.
 message NanoPBOptions {
   // Allocated size for 'bytes' and 'string' fields.
   optional int32 max_size = 1;
@@ -25,8 +31,16 @@ message NanoPBOptions {
   
   // Use long names for enums, i.e. EnumName_EnumValue.
   optional bool long_names = 4 [default = true];
+  
+  // Add 'packed' attribute to generated structs.
+  // Note: this cannot be used on CPUs that break on unaligned
+  // accesses to variables.
+  optional bool packed_struct = 5 [default = false];
 }
 
+// Extensions to protoc 'Descriptor' type in order to define options
+// inside a .proto file.
+//
 // Protocol Buffers extension number registry
 // --------------------------------
 // Project:  Nanopb

generator/nanopb_generator.py

@@ -1,8 +1,9 @@
 '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.'''
-nanopb_version = "nanopb-0.1.7"
+nanopb_version = "nanopb-0.2.3-dev"
 
 try:
     import google.protobuf.descriptor_pb2 as descriptor
+    import google.protobuf.text_format as text_format
 except:
     print
     print "*************************************************************"
@@ -35,28 +36,26 @@ except:
 import time
 import os.path
 
-# Values are tuple (c type, pb ltype)
+# Values are tuple (c type, pb type)
 FieldD = descriptor.FieldDescriptorProto
 datatypes = {
-    FieldD.TYPE_BOOL: ('bool', 'PB_LTYPE_VARINT'),
-    FieldD.TYPE_DOUBLE: ('double', 'PB_LTYPE_FIXED64'),
-    FieldD.TYPE_FIXED32: ('uint32_t', 'PB_LTYPE_FIXED32'),
-    FieldD.TYPE_FIXED64: ('uint64_t', 'PB_LTYPE_FIXED64'),
-    FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED32'),
-    FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT'),
-    FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT'),
-    FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED32'),
-    FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED64'),
-    FieldD.TYPE_SINT32: ('int32_t', 'PB_LTYPE_SVARINT'),
-    FieldD.TYPE_SINT64: ('int64_t', 'PB_LTYPE_SVARINT'),
-    FieldD.TYPE_UINT32: ('uint32_t', 'PB_LTYPE_VARINT'),
-    FieldD.TYPE_UINT64: ('uint64_t', 'PB_LTYPE_VARINT')
+    FieldD.TYPE_BOOL:       ('bool',     'BOOL'),
+    FieldD.TYPE_DOUBLE:     ('double',   'DOUBLE'),
+    FieldD.TYPE_FIXED32:    ('uint32_t', 'FIXED32'),
+    FieldD.TYPE_FIXED64:    ('uint64_t', 'FIXED64'),
+    FieldD.TYPE_FLOAT:      ('float',    'FLOAT'),
+    FieldD.TYPE_INT32:      ('int32_t',  'INT32'),
+    FieldD.TYPE_INT64:      ('int64_t',  'INT64'),
+    FieldD.TYPE_SFIXED32:   ('int32_t',  'SFIXED32'),
+    FieldD.TYPE_SFIXED64:   ('int64_t',  'SFIXED64'),
+    FieldD.TYPE_SINT32:     ('int32_t',  'SINT32'),
+    FieldD.TYPE_SINT64:     ('int64_t',  'SINT64'),
+    FieldD.TYPE_UINT32:     ('uint32_t', 'UINT32'),
+    FieldD.TYPE_UINT64:     ('uint64_t', 'UINT64')
 }
 
 class Names:
-    '''Keeps a set of nested names and formats them to C identifier.
-    You can subclass this with your own implementation.
-    '''
+    '''Keeps a set of nested names and formats them to C identifier.'''
     def __init__(self, parts = ()):
         if isinstance(parts, Names):
             parts = parts.parts
@@ -73,6 +72,9 @@ class Names:
         else:
             raise ValueError("Name parts should be of type str")
     
+    def __eq__(self, other):
+        return isinstance(other, Names) and self.parts == other.parts
+    
 def names_from_type_name(type_name):
     '''Parse Names() from FieldDescriptorProto type_name'''
     if type_name[0] != '.':
@@ -83,12 +85,15 @@ class Enum:
     def __init__(self, names, desc, enum_options):
         '''desc is EnumDescriptorProto'''
         
-        if enum_options.long_names:
+        self.options = enum_options
         self.names = names + desc.name
-        else:
-            self.names = names
         
+        if enum_options.long_names:
             self.values = [(self.names + x.name, x.number) for x in desc.value]            
+        else:
+            self.values = [(names + x.name, x.number) for x in desc.value] 
+        
+        self.value_longnames = [self.names + x.name for x in desc.value]
     
     def __str__(self):
         result = 'typedef enum _%s {\n' % self.names
@@ -117,49 +122,44 @@ class Field:
         if desc.HasField('default_value'):
             self.default = desc.default_value
            
-        # Decide HTYPE
-        # HTYPE is the high-order nibble of nanopb field description,
-        # defining whether value is required/optional/repeated.
+        # Check field rules, i.e. required/optional/repeated.
         can_be_static = True
         if desc.label == FieldD.LABEL_REQUIRED:
-            self.htype = 'PB_HTYPE_REQUIRED'
+            self.rules = 'REQUIRED'
         elif desc.label == FieldD.LABEL_OPTIONAL:
-            self.htype = 'PB_HTYPE_OPTIONAL'
+            self.rules = 'OPTIONAL'
         elif desc.label == FieldD.LABEL_REPEATED:
+            self.rules = 'REPEATED'
             if self.max_count is None:
                 can_be_static = False
             else:
-                self.htype = 'PB_HTYPE_ARRAY'
                 self.array_decl = '[%d]' % self.max_count
         else:
             raise NotImplementedError(desc.label)
         
-        # Decide LTYPE and CTYPE
-        # LTYPE is the low-order nibble of nanopb field description,
-        # defining how to decode an individual value.
-        # CTYPE is the name of the c type to use in the struct.
+        # Decide the C data type to use in the struct.
         if datatypes.has_key(desc.type):
-            self.ctype, self.ltype = datatypes[desc.type]
+            self.ctype, self.pbtype = datatypes[desc.type]
         elif desc.type == FieldD.TYPE_ENUM:
-            self.ltype = 'PB_LTYPE_VARINT'
+            self.pbtype = 'ENUM'
             self.ctype = names_from_type_name(desc.type_name)
             if self.default is not None:
                 self.default = self.ctype + self.default
         elif desc.type == FieldD.TYPE_STRING:
-            self.ltype = 'PB_LTYPE_STRING'
+            self.pbtype = 'STRING'
             if self.max_size is None:
                 can_be_static = False
             else:
                 self.ctype = 'char'
                 self.array_decl += '[%d]' % self.max_size
         elif desc.type == FieldD.TYPE_BYTES:
-            self.ltype = 'PB_LTYPE_BYTES'
+            self.pbtype = 'BYTES'
             if self.max_size is None:
                 can_be_static = False
             else:
                 self.ctype = self.struct_name + self.name + 't'
         elif desc.type == FieldD.TYPE_MESSAGE:
-            self.ltype = 'PB_LTYPE_SUBMESSAGE'
+            self.pbtype = 'MESSAGE'
             self.ctype = self.submsgname = names_from_type_name(desc.type_name)
         else:
             raise NotImplementedError(desc.type)
@@ -173,18 +173,22 @@ class Field:
         if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static:
             raise Exception("Field %s is defined as static, but max_size or max_count is not given." % self.name)
         
-        if field_options.type == nanopb_pb2.FT_CALLBACK:
-            self.htype = 'PB_HTYPE_CALLBACK'
+        if field_options.type == nanopb_pb2.FT_STATIC:
+            self.allocation = 'STATIC'
+        elif field_options.type == nanopb_pb2.FT_CALLBACK:
+            self.allocation = 'CALLBACK'
             self.ctype = 'pb_callback_t'
             self.array_decl = ''
+        else:
+            raise NotImplementedError(field_options.type)
     
     def __cmp__(self, other):
         return cmp(self.tag, other.tag)
     
     def __str__(self):
-        if self.htype == 'PB_HTYPE_OPTIONAL':
+        if self.rules == 'OPTIONAL' and self.allocation == 'STATIC':
             result = '    bool has_' + self.name + ';\n'
-        elif self.htype == 'PB_HTYPE_ARRAY':
+        elif self.rules == 'REPEATED' and self.allocation == 'STATIC':
             result = '    size_t ' + self.name + '_count;\n'
         else:
             result = ''
@@ -193,7 +197,7 @@ class Field:
     
     def types(self):
         '''Return definitions for any special types this field might need.'''
-        if self.ltype == 'PB_LTYPE_BYTES' and self.max_size is not None:
+        if self.pbtype == 'BYTES' and self.allocation == 'STATIC':
             result = 'typedef struct {\n'
             result += '    size_t size;\n'
             result += '    uint8_t bytes[%d];\n' % self.max_size
@@ -207,80 +211,63 @@ class Field:
         if self.default is None:
             return None
 
-        if self.ltype == 'PB_LTYPE_STRING':
-            ctype = 'char'
-            if self.max_size is None:
+        ctype, default = self.ctype, self.default
+        array_decl = ''
+        
+        if self.pbtype == 'STRING':
+            if self.allocation != 'STATIC':
                 return None # Not implemented
-            else:
-                array_decl = '[%d]' % (self.max_size + 1)
+        
+            array_decl = '[%d]' % self.max_size
             default = str(self.default).encode('string_escape')
             default = default.replace('"', '\\"')
             default = '"' + default + '"'
-        elif self.ltype == 'PB_LTYPE_BYTES':
+        elif self.pbtype == 'BYTES':
+            if self.allocation != 'STATIC':
+                return None # Not implemented
+
             data = self.default.decode('string_escape')
             data = ['0x%02x' % ord(c) for c in data]
-            
-            if self.max_size is None:
-                return None # Not implemented
-            else:
-                ctype = self.ctype
-            
             default = '{%d, {%s}}' % (len(data), ','.join(data))
-            array_decl = ''
-        else:
-            ctype, default = self.ctype, self.default
-            array_decl = ''
         
         if declaration_only:
             return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl)
         else:
             return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default)
     
+    def tags(self):
+        '''Return the #define for the tag number of this field.'''
+        identifier = '%s_%s_tag' % (self.struct_name, self.name)
+        return '#define %-40s %d\n' % (identifier, self.tag)
+    
     def pb_field_t(self, prev_field_name):
         '''Return the pb_field_t initializer to use in the constant array.
         prev_field_name is the name of the previous field or None.
         '''
-        result = '    {%d, ' % self.tag
-        result += '(pb_type_t) ((int) ' + self.htype
-        if self.ltype is not None:
-            result += ' | (int) ' + self.ltype
-        result += '),\n'
+        result  = '    PB_FIELD(%3d, ' % self.tag
+        result += '%-8s, ' % self.pbtype
+        result += '%s, ' % self.rules
+        result += '%s, ' % self.allocation
+        result += '%s, ' % self.struct_name
+        result += '%s, ' % self.name
+        result += '%s, ' % (prev_field_name or self.name)
         
-        if prev_field_name is None:
-            result += '    offsetof(%s, %s),' % (self.struct_name, self.name)
+        if self.pbtype == 'MESSAGE':
+            result += '&%s_fields)' % self.submsgname
+        elif self.default is None:
+            result += '0)'
+        elif self.pbtype in ['BYTES', 'STRING'] and self.allocation != 'STATIC':
+            result += '0)' # Arbitrary size default values not implemented
         else:
-            result += '    pb_delta_end(%s, %s, %s),' % (self.struct_name, self.name, prev_field_name)
-        
-        if self.htype == 'PB_HTYPE_OPTIONAL':
-            result += '\n    pb_delta(%s, has_%s, %s),' % (self.struct_name, self.name, self.name)
-        elif self.htype == 'PB_HTYPE_ARRAY':
-            result += '\n    pb_delta(%s, %s_count, %s),' % (self.struct_name, self.name, self.name)
-        else:
-            result += ' 0,'
-        
-        
-        if self.htype == 'PB_HTYPE_ARRAY':
-            result += '\n    pb_membersize(%s, %s[0]),' % (self.struct_name, self.name)
-            result += ('\n    pb_membersize(%s, %s) / pb_membersize(%s, %s[0]),'
-                       % (self.struct_name, self.name, self.struct_name, self.name))
-        else:
-            result += '\n    pb_membersize(%s, %s),' % (self.struct_name, self.name)
-            result += ' 0,'
-        
-        if self.ltype == 'PB_LTYPE_SUBMESSAGE':
-            result += '\n    &%s_fields}' % self.submsgname
-        elif self.default is None or self.htype == 'PB_HTYPE_CALLBACK':
-            result += ' 0}'
-        else:
-            result += '\n    &%s_default}' % (self.struct_name + self.name)
+            result += '&%s_default)' % (self.struct_name + self.name)
         
         return result
     
     def largest_field_value(self):
         '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly.
         Returns numeric value or a C-expression for assert.'''
-        if self.ltype == 'PB_LTYPE_SUBMESSAGE':
-            if self.htype == 'PB_HTYPE_ARRAY':
+        if self.pbtype == 'MESSAGE':
+            if self.rules == 'REPEATED' and self.allocation == 'STATIC':
                 return 'pb_membersize(%s, %s[0])' % (self.struct_name, self.name)
             else:
                 return 'pb_membersize(%s, %s)' % (self.struct_name, self.name)
@@ -288,8 +275,72 @@ class Field:
         return max(self.tag, self.max_size, self.max_count)        
 
 
+class ExtensionRange(Field):
+    def __init__(self, struct_name, range_start, field_options):
+        '''Implements a special pb_extension_t* field in an extensible message
+        structure. The range_start signifies the index at which the extensions
+        start. Not necessarily all tags above this are extensions, it is merely
+        a speed optimization.
+        '''
+        self.tag = range_start
+        self.struct_name = struct_name
+        self.name = 'extensions'
+        self.pbtype = 'EXTENSION'
+        self.rules = 'OPTIONAL'
+        self.allocation = 'CALLBACK'
+        self.ctype = 'pb_extension_t'
+        self.array_decl = ''
+        self.default = None
+        self.max_size = 0
+        self.max_count = 0
         
+    def __str__(self):
+        return '    pb_extension_t *extensions;'
     
+    def types(self):
+        return None
+    
+    def tags(self):
+        return ''
+
+class ExtensionField(Field):
+    def __init__(self, struct_name, desc, field_options):
+        self.fullname = struct_name + desc.name
+        self.extendee_name = names_from_type_name(desc.extendee)
+        Field.__init__(self, self.fullname + 'struct', desc, field_options)
+        
+        if self.rules != 'OPTIONAL':
+            self.skip = True
+        else:
+            self.skip = False
+            self.rules = 'OPTEXT'
+
+    def extension_decl(self):
+        '''Declaration of the extension type in the .pb.h file'''
+        if self.skip:
+            msg = '/* Extension field %s was skipped because only "optional"\n' % self.fullname
+            msg +='   type of extension fields is currently supported. */\n'
+            return msg
+        
+        return 'extern const pb_extension_type_t %s;\n' % self.fullname
+
+    def extension_def(self):
+        '''Definition of the extension type in the .pb.c file'''
+
+        if self.skip:
+            return ''
+
+        result  = 'typedef struct {\n'
+        result += str(self)
+        result += '\n} %s;\n\n' % self.struct_name
+        result += ('static const pb_field_t %s_field = \n  %s;\n\n' %
+                    (self.fullname, self.pb_field_t(None)))
+        result += 'const pb_extension_type_t %s = {\n' % self.fullname
+        result += '    NULL,\n'
+        result += '    NULL,\n'
+        result += '    &%s_field\n' % self.fullname
+        result += '};\n'
+        return result
 
 
 # ---------------------------------------------------------------------------
@@ -300,7 +351,20 @@ class Field:
 class Message:
     def __init__(self, names, desc, message_options):
         self.name = names
-        self.fields = [Field(self.name, f, get_nanopb_suboptions(f, message_options)) for f in desc.field]
+        self.fields = []
+        
+        for f in desc.field:
+            field_options = get_nanopb_suboptions(f, message_options, self.name + f.name)
+            if field_options.type != nanopb_pb2.FT_IGNORE:
+                self.fields.append(Field(self.name, f, field_options))
+        
+        if len(desc.extension_range) > 0:
+            field_options = get_nanopb_suboptions(desc, message_options, self.name + 'extensions')
+            range_start = min([r.start for r in desc.extension_range])
+            if field_options.type != nanopb_pb2.FT_IGNORE:
+                self.fields.append(ExtensionRange(self.name, range_start, field_options))
+        
+        self.packed = message_options.packed_struct
         self.ordered_fields = self.fields[:]
         self.ordered_fields.sort()
 
@@ -310,8 +374,24 @@ class Message:
     
     def __str__(self):
         result = 'typedef struct _%s {\n' % self.name
+
+        if not self.ordered_fields:
+            # Empty structs are not allowed in C standard.
+            # Therefore add a dummy field if an empty message occurs.
+            result += '    uint8_t dummy_field;'
+
         result += '\n'.join([str(f) for f in self.ordered_fields])
-        result += '\n} %s;' % self.name
+        result += '\n}'
+        
+        if self.packed:
+            result += ' pb_packed'
+        
+        result += ' %s;' % self.name
+        
+        if self.packed:
+            result = 'PB_PACKED_STRUCT_START\n' + result
+            result += '\nPB_PACKED_STRUCT_END'
+        
         return result
     
     def types(self):
@@ -340,7 +420,7 @@ class Message:
         prev = None
         for field in self.ordered_fields:
             result += field.pb_field_t(prev)
-            result += ',\n\n'
+            result += ',\n'
             prev = field.name
         
         result += '    PB_LAST_FIELD\n};'
@@ -348,9 +428,6 @@ class Message:
 
 
 
-
-
-
 # ---------------------------------------------------------------------------
 #                    Processing of entire .proto files
 # ---------------------------------------------------------------------------
@@ -370,11 +447,23 @@ def iterate_messages(desc, names = Names()):
         for x in iterate_messages(submsg, sub_names):
             yield x
 
+def iterate_extensions(desc, names = Names()):
+    '''Recursively find all extensions.
+    For each, yield name, FieldDescriptorProto.
+    '''
+    for extension in desc.extension:
+        yield names, extension
+
+    for subname, subdesc in iterate_messages(desc, names):
+        for extension in subdesc.extension:
+            yield subname, extension
+
 def parse_file(fdesc, file_options):
-    '''Takes a FileDescriptorProto and returns tuple (enum, messages).'''
+    '''Takes a FileDescriptorProto and returns tuple (enums, messages, extensions).'''
     
     enums = []
     messages = []
+    extensions = []
     
     if fdesc.package:
         base_name = Names(fdesc.package.split('.'))
@@ -382,16 +471,31 @@ def parse_file(fdesc, file_options):
         base_name = Names()
     
     for enum in fdesc.enum_type:
-        enum_options = get_nanopb_suboptions(enum, file_options)
+        enum_options = get_nanopb_suboptions(enum, file_options, base_name + enum.name)
         enums.append(Enum(base_name, enum, enum_options))
     
     for names, message in iterate_messages(fdesc, base_name):
-        message_options = get_nanopb_suboptions(message, file_options)
+        message_options = get_nanopb_suboptions(message, file_options, names)
         messages.append(Message(names, message, message_options))
         for enum in message.enum_type:
-            enums.append(Enum(names, enum, message_options))
+            enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name)
+            enums.append(Enum(names, enum, enum_options))
     
-    return enums, messages
+    for names, extension in iterate_extensions(fdesc, base_name):
+        field_options = get_nanopb_suboptions(extension, file_options, names)
+        if field_options.type != nanopb_pb2.FT_IGNORE:
+            extensions.append(ExtensionField(names, extension, field_options))
+    
+    # Fix field default values where enum short names are used.
+    for enum in enums:
+        if not enum.options.long_names:
+            for message in messages:
+                for field in message.fields:
+                    if field.default in enum.value_longnames:
+                        idx = enum.value_longnames.index(field.default)
+                        field.default = enum.values[idx][0]
+    
+    return enums, messages, extensions
 
 def toposort2(data):
     '''Topological sort.
@@ -424,7 +528,17 @@ def sort_dependencies(messages):
         if msgname in message_by_name:
             yield message_by_name[msgname]
 
-def generate_header(dependencies, headername, enums, messages):
+def make_identifier(headername):
+    '''Make #ifndef identifier that contains uppercase A-Z and digits 0-9'''
+    result = ""
+    for c in headername.upper():
+        if c.isalnum():
+            result += c
+        else:
+            result += '_'
+    return result
+
+def generate_header(dependencies, headername, enums, messages, extensions, options):
     '''Generate content for a header file.
     Generates strings, which should be concatenated and stored to file.
     '''
@@ -432,14 +546,20 @@ def generate_header(dependencies, headername, enums, messages):
     yield '/* Automatically generated nanopb header */\n'
     yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
     
-    symbol = headername.replace('.', '_').upper()
+    symbol = make_identifier(headername)
     yield '#ifndef _PB_%s_\n' % symbol
     yield '#define _PB_%s_\n' % symbol
-    yield '#include <pb.h>\n\n'
+    try:
+        yield options.libformat % ('pb.h')
+    except TypeError:
+        # no %s specified - use whatever was passed in as options.libformat
+        yield options.libformat
+    yield '\n'
     
     for dependency in dependencies:
         noext = os.path.splitext(dependency)[0]
-        yield '#include "%s.pb.h"\n' % noext
+        yield options.genformat % (noext + '.' + options.extension + '.h')
+        yield '\n'
 
     yield '#ifdef __cplusplus\n'
     yield 'extern "C" {\n'
@@ -454,17 +574,56 @@ def generate_header(dependencies, headername, enums, messages):
         yield msg.types()
         yield str(msg) + '\n\n'
     
+    if extensions:
+        yield '/* Extensions */\n'
+        for extension in extensions:
+            yield extension.extension_decl()
+        yield '\n'
+        
     yield '/* Default values for struct fields */\n'
     for msg in messages:
         yield msg.default_decl(True)
     yield '\n'
     
+    yield '/* Field tags (for use in manual encoding/decoding) */\n'
+    for msg in sort_dependencies(messages):
+        for field in msg.fields:
+            yield field.tags()
+    yield '\n'
+    
     yield '/* Struct field encoding specification for nanopb */\n'
     for msg in messages:
         yield msg.fields_declaration() + '\n'
     
+    yield '\n#ifdef __cplusplus\n'
+    yield '} /* extern "C" */\n'
+    yield '#endif\n'
+    
+    # End of header
+    yield '\n#endif\n'
+
+def generate_source(headername, enums, messages, extensions):
+    '''Generate content for a source file.'''
+    
+    yield '/* Automatically generated nanopb constant definitions */\n'
+    yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
+    yield options.genformat % (headername)
+    yield '\n'
+    
+    for msg in messages:
+        yield msg.default_decl(False)
+    
+    yield '\n\n'
+    
+    for msg in messages:
+        yield msg.fields_definition() + '\n\n'
+    
+    for ext in extensions:
+        yield ext.extension_def() + '\n'
+        
+    # Add checks for numeric limits
     if messages:
-        count_required_fields = lambda m: len([f for f in msg.fields if f.htype == 'PB_HTYPE_REQUIRED'])
+        count_required_fields = lambda m: len([f for f in msg.fields if f.rules == 'REQUIRED'])
         largest_msg = max(messages, key = count_required_fields)
         largest_count = count_required_fields(largest_msg)
         if largest_count > 64:
@@ -477,7 +636,9 @@ def generate_header(dependencies, headername, enums, messages):
     worst = 0
     worst_field = ''
     checks = []
+    checks_msgnames = []
     for msg in messages:
+        checks_msgnames.append(msg.name)
         for field in msg.fields:
             status = field.largest_field_value()
             if isinstance(status, (str, unicode)):
@@ -495,7 +656,8 @@ def generate_header(dependencies, headername, enums, messages):
                 yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field
             else:
                 assertion = ' && '.join(str(c) + ' < 256' for c in checks)
-                yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT)\n' % assertion
+                msgs = '_'.join(str(n) for n in checks_msgnames)
+                yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
             yield '#endif\n\n'
         
         if worst > 65535 or checks:
@@ -504,60 +666,67 @@ def generate_header(dependencies, headername, enums, messages):
                 yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field
             else:
                 assertion = ' && '.join(str(c) + ' < 65536' for c in checks)
-                yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT)\n' % assertion
+                msgs = '_'.join(str(n) for n in checks_msgnames)
+                yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
             yield '#endif\n'
     
-    yield '\n#ifdef __cplusplus\n'
-    yield '} /* extern "C" */\n'
-    yield '#endif\n'
-    
-    # End of header
-    yield '\n#endif\n'
-
-def generate_source(headername, enums, messages):
-    '''Generate content for a source file.'''
-    
-    yield '/* Automatically generated nanopb constant definitions */\n'
-    yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
-    yield '#include "%s"\n\n' % headername
-    
+    # Add check for sizeof(double)
+    has_double = False
     for msg in messages:
-        yield msg.default_decl(False)
-    
-    yield '\n\n'
-    
-    for msg in messages:
-        yield msg.fields_definition() + '\n\n'
+        for field in msg.fields:
+            if field.ctype == 'double':
+                has_double = True
     
+    if has_double:
+        yield '\n'
+        yield '/* On some platforms (such as AVR), double is really float.\n'
+        yield ' * These are not directly supported by nanopb, but see example_avr_double.\n'
+        yield ' * To get rid of this error, remove any double fields from your .proto.\n'
+        yield ' */\n'
+        yield 'STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n'
     
+    yield '\n'
 
 # ---------------------------------------------------------------------------
-#                         Command line interface
+#                    Options parsing for the .proto files
 # ---------------------------------------------------------------------------
 
-import sys
-import os.path    
-from optparse import OptionParser
-import google.protobuf.text_format as text_format
+from fnmatch import fnmatch
 
-optparser = OptionParser(
-    usage = "Usage: nanopb_generator.py [options] file.pb ...",
-    epilog = "Compile file.pb from file.proto by: 'protoc -ofile.pb file.proto'. " +
-             "Output will be written to file.pb.h and file.pb.c.")
-optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", default=[],
-    help="Exclude file from generated #include list.")
-optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False,
-    help="Don't print anything except errors.")
-optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
-    help="Print more information.")
-optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[],
-    help="Set generator option (max_size, max_count etc.).")
+def read_options_file(infile):
+    '''Parse a separate options file to list:
+        [(namemask, options), ...]
+    '''
+    results = []
+    for line in infile:
+        line = line.strip()
+        if not line or line.startswith('//') or line.startswith('#'):
+            continue
         
-def get_nanopb_suboptions(subdesc, options):
+        parts = line.split(None, 1)
+        opts = nanopb_pb2.NanoPBOptions()
+        text_format.Merge(parts[1], opts)
+        results.append((parts[0], opts))
+
+    return results
+
+class Globals:
+    '''Ugly global variables, should find a good way to pass these.'''
+    verbose_options = False
+    separate_options = []
+
+def get_nanopb_suboptions(subdesc, options, name):
     '''Get copy of options, and merge information from subdesc.'''
     new_options = nanopb_pb2.NanoPBOptions()
     new_options.CopyFrom(options)
     
+    # Handle options defined in a separate file
+    dotname = '.'.join(name.parts)
+    for namemask, options in Globals.separate_options:
+        if fnmatch(dotname, namemask):
+            new_options.MergeFrom(options)
+    
+    # Handle options defined in .proto
     if isinstance(subdesc.options, descriptor.FieldOptions):
         ext_type = nanopb_pb2.nanopb
     elif isinstance(subdesc.options, descriptor.FileOptions):
@@ -573,8 +742,44 @@ def get_nanopb_suboptions(subdesc, options):
         ext = subdesc.options.Extensions[ext_type]
         new_options.MergeFrom(ext)
     
+    if Globals.verbose_options:
+        print "Options for " + dotname + ":"
+        print text_format.MessageToString(new_options)
+    
     return new_options
 
+
+# ---------------------------------------------------------------------------
+#                         Command line interface
+# ---------------------------------------------------------------------------
+
+import sys
+import os.path    
+from optparse import OptionParser
+
+optparser = OptionParser(
+    usage = "Usage: nanopb_generator.py [options] file.pb ...",
+    epilog = "Compile file.pb from file.proto by: 'protoc -ofile.pb file.proto'. " +
+             "Output will be written to file.pb.h and file.pb.c.")
+optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", default=[],
+    help="Exclude file from generated #include list.")
+optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default="pb",
+    help="Set extension to use instead of 'pb' for generated files. [default: %default]")
+optparser.add_option("-f", "--options-file", dest="options_file", metavar="FILE", default="%s.options",
+    help="Set name of a separate generator options file.")
+optparser.add_option("-Q", "--generated-include-format", dest="genformat",
+    metavar="FORMAT", default='#include "%s"\n',
+    help="Set format string to use for including other .pb.h files. [default: %default]")
+optparser.add_option("-L", "--library-include-format", dest="libformat",
+    metavar="FORMAT", default='#include <%s>\n',
+    help="Set format string to use for including the nanopb pb.h header. [default: %default]")
+optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False,
+    help="Don't print anything except errors.")
+optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
+    help="Print more information.")
+optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[],
+    help="Set generator option (max_size, max_count etc.).")
+
 def process(filenames, options):
     '''Process the files given on the command line.'''
     
@@ -585,6 +790,8 @@ def process(filenames, options):
     if options.quiet:
         options.verbose = False
 
+    Globals.verbose_options = options.verbose
+    
     toplevel_options = nanopb_pb2.NanoPBOptions()
     for s in options.settings:
         text_format.Merge(s, toplevel_options)
@@ -593,17 +800,28 @@ def process(filenames, options):
         data = open(filename, 'rb').read()
         fdesc = descriptor.FileDescriptorSet.FromString(data)
         
-        file_options = get_nanopb_suboptions(fdesc.file[0], toplevel_options)
+        # Check if any separate options are specified
+        try:
+            optfilename = options.options_file % os.path.splitext(filename)[0]
+        except TypeError:
+            # No %s specified, use the filename as-is
+            optfilename = options.options_file
         
         if options.verbose:
-            print "Options for " + filename + ":"
-            print text_format.MessageToString(file_options)
+            print 'Reading options from ' + optfilename
         
-        enums, messages = parse_file(fdesc.file[0], file_options)
+        if os.path.isfile(optfilename):
+            Globals.separate_options = read_options_file(open(optfilename, "rU"))
+        else:
+            Globals.separate_options = []
+        
+        # Parse the file
+        file_options = get_nanopb_suboptions(fdesc.file[0], toplevel_options, Names([filename]))
+        enums, messages, extensions = parse_file(fdesc.file[0], file_options)
         
         noext = os.path.splitext(filename)[0]
-        headername = noext + '.pb.h'
-        sourcename = noext + '.pb.c'
+        headername = noext + '.' + options.extension + '.h'
+        sourcename = noext + '.' + options.extension + '.c'
         headerbasename = os.path.basename(headername)
         
         if not options.quiet:
@@ -615,11 +833,12 @@ def process(filenames, options):
         dependencies = [d for d in fdesc.file[0].dependency if d not in excludes]
         
         header = open(headername, 'w')
-        for part in generate_header(dependencies, headerbasename, enums, messages):
+        for part in generate_header(dependencies, headerbasename, enums,
+                                    messages, extensions, options):
             header.write(part)
 
         source = open(sourcename, 'w')
-        for part in generate_source(headerbasename, enums, messages):
+        for part in generate_source(headerbasename, enums, messages, extensions):
             source.write(part)
 
     return True

generator/nanopb_pb2.py

@@ -12,7 +12,7 @@ import google.protobuf.descriptor_pb2
 DESCRIPTOR = descriptor.FileDescriptor(
   name='nanopb.proto',
   package='',
-  serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"t\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true*;\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions')
+  serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*J\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions')
 
 _FIELDTYPE = descriptor.EnumDescriptor(
   name='FieldType',
@@ -32,17 +32,22 @@ _FIELDTYPE = descriptor.EnumDescriptor(
       name='FT_STATIC', index=2, number=2,
       options=None,
       type=None),
+    descriptor.EnumValueDescriptor(
+      name='FT_IGNORE', index=3, number=3,
+      options=None,
+      type=None),
   ],
   containing_type=None,
   options=None,
-  serialized_start=168,
-  serialized_end=227,
+  serialized_start=199,
+  serialized_end=273,
 )
 
 
 FT_DEFAULT = 0
 FT_CALLBACK = 1
 FT_STATIC = 2
+FT_IGNORE = 3
 
 NANOPB_FILEOPT_FIELD_NUMBER = 1010
 nanopb_fileopt = descriptor.FieldDescriptor(
@@ -113,6 +118,13 @@ _NANOPBOPTIONS = descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
+    descriptor.FieldDescriptor(
+      name='packed_struct', full_name='NanoPBOptions.packed_struct', index=4,
+      number=5, type=8, cpp_type=7, label=1,
+      has_default_value=True, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
   ],
   extensions=[
   ],
@@ -122,8 +134,8 @@ _NANOPBOPTIONS = descriptor.Descriptor(
   options=None,
   is_extendable=False,
   extension_ranges=[],
-  serialized_start=50,
-  serialized_end=166,
+  serialized_start=51,
+  serialized_end=197,
 )
 
 _NANOPBOPTIONS.fields_by_name['type'].enum_type = _FIELDTYPE

pb.h

@@ -1,21 +1,92 @@
+/* Common parts of the nanopb library. Most of these are quite low-level
+ * stuff. For the high-level interface, see pb_encode.h and pb_decode.h.
+ */
+
 #ifndef _PB_H_
 #define _PB_H_
 
-/* pb.h: Common parts for nanopb library.
- * Most of these are quite low-level stuff. For the high-level interface,
- * see pb_encode.h or pb_decode.h
+/*****************************************************************
+ * Nanopb compilation time options. You can change these here by *
+ * uncommenting the lines, or on the compiler command line.      *
+ *****************************************************************/
+
+/* Define this if your CPU architecture is big endian, i.e. it
+ * stores the most-significant byte first. */
+/* #define __BIG_ENDIAN__ 1 */
+
+/* Increase the number of required fields that are tracked.
+ * A compiler warning will tell if you need this. */
+/* #define PB_MAX_REQUIRED_FIELDS 256 */
+
+/* Add support for tag numbers > 255 and fields larger than 255 bytes. */
+/* #define PB_FIELD_16BIT 1 */
+
+/* Add support for tag numbers > 65536 and fields larger than 65536 bytes. */
+/* #define PB_FIELD_32BIT 1 */
+
+/* Disable support for error messages in order to save some code space. */
+/* #define PB_NO_ERRMSG 1 */
+
+/* Disable support for custom streams (support only memory buffers). */
+/* #define PB_BUFFER_ONLY 1 */
+
+/* Switch back to the old-style callback function signature.
+ * This was the default until nanopb-0.2.1. */
+/* #define PB_OLD_CALLBACK_STYLE */
+
+
+/******************************************************************
+ * You usually don't need to change anything below this line.     *
+ * Feel free to look around and use the defined macros, though.   *
+ ******************************************************************/
+
+
+/* Version of the nanopb library. Just in case you want to check it in
+ * your own program. */
+#define NANOPB_VERSION nanopb-0.2.3-dev
+
+/* Include all the system headers needed by nanopb. You will need the
+ * definitions of the following:
+ * - strlen, memcpy, memset functions
+ * - [u]int8_t, [u]int16_t, [u]int32_t, [u]int64_t
+ * - size_t
+ * - bool
+ *
+ * If you don't have the standard header files, you can instead provide
+ * a custom header that defines or includes all this. In that case,
+ * define PB_SYSTEM_HEADER to the path of this file.
  */
-
-#define NANOPB_VERSION nanopb-0.1.7
-
+#ifdef PB_SYSTEM_HEADER
+#include PB_SYSTEM_HEADER
+#else
 #include <stdint.h>
 #include <stddef.h>
 #include <stdbool.h>
+#include <string.h>
+#endif
 
-#ifdef __GNUC__
-/* This just reduces memory requirements, but is not required. */
+/* Macro for defining packed structures (compiler dependent).
+ * This just reduces memory requirements, but is not required.
+ */
+#if defined(__GNUC__) || defined(__clang__)
+    /* For GCC and clang */
+#   define PB_PACKED_STRUCT_START
+#   define PB_PACKED_STRUCT_END
 #   define pb_packed __attribute__((packed))
+#elif defined(__ICCARM__)
+    /* For IAR ARM compiler */
+#   define PB_PACKED_STRUCT_START _Pragma("pack(push, 1)")
+#   define PB_PACKED_STRUCT_END _Pragma("pack(pop)")
+#   define pb_packed
+#elif defined(_MSC_VER) && (_MSC_VER >= 1500)
+    /* For Microsoft Visual C++ */
+#   define PB_PACKED_STRUCT_START __pragma(pack(push, 1))
+#   define PB_PACKED_STRUCT_END __pragma(pack(pop))
+#   define pb_packed
 #else
+    /* Unknown compiler */
+#   define PB_PACKED_STRUCT_START
+#   define PB_PACKED_STRUCT_END
 #   define pb_packed
 #endif
 
@@ -24,13 +95,16 @@
 #define UNUSED(x) (void)(x)
 #endif
 
-/* Compile-time assertion, used for checking compatible compilation options. */
+/* Compile-time assertion, used for checking compatible compilation options.
+ * If this fails on your compiler for some reason, use #define STATIC_ASSERT
+ * to disable it. */
 #ifndef STATIC_ASSERT
-#define STATIC_ASSERT(COND,MSG) typedef char static_assertion_##MSG[(COND)?1:-1];
+#define STATIC_ASSERT(COND,MSG) typedef char STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND)?1:-1];
+#define STATIC_ASSERT_MSG(MSG, LINE, COUNTER) STATIC_ASSERT_MSG_(MSG, LINE, COUNTER)
+#define STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) static_assertion_##MSG##LINE##COUNTER
 #endif
 
-/* Number of required fields to keep track of
- * (change here or on compiler command line). */
+/* Number of required fields to keep track of. */
 #ifndef PB_MAX_REQUIRED_FIELDS
 #define PB_MAX_REQUIRED_FIELDS 64
 #endif
@@ -49,60 +123,53 @@
  * SINT* is different, though, because it is zig-zag coded.
  */
 
-typedef enum {
-    /************************
-     * Field contents types *
-     ************************/
+typedef uint8_t pb_type_t;
+
+/**** Field data types ****/
 
 /* Numeric types */
-    PB_LTYPE_VARINT = 0x00, /* int32, uint32, int64, uint64, bool, enum */
-    PB_LTYPE_SVARINT = 0x01, /* sint32, sint64 */
-    PB_LTYPE_FIXED32 = 0x02, /* fixed32, sfixed32, float */
-    PB_LTYPE_FIXED64 = 0x03, /* fixed64, sfixed64, double */
+#define PB_LTYPE_VARINT 0x00 /* int32, uint32, int64, uint64, bool, enum */
+#define PB_LTYPE_SVARINT 0x01 /* sint32, sint64 */
+#define PB_LTYPE_FIXED32 0x02 /* fixed32, sfixed32, float */
+#define PB_LTYPE_FIXED64 0x03 /* fixed64, sfixed64, double */
 
 /* Marker for last packable field type. */
-    PB_LTYPE_LAST_PACKABLE = 0x03,
+#define PB_LTYPE_LAST_PACKABLE 0x03
 
 /* Byte array with pre-allocated buffer.
  * data_size is the length of the allocated PB_BYTES_ARRAY structure. */
-    PB_LTYPE_BYTES = 0x04,
+#define PB_LTYPE_BYTES 0x04
 
 /* String with pre-allocated buffer.
  * data_size is the maximum length. */
-    PB_LTYPE_STRING = 0x05,
+#define PB_LTYPE_STRING 0x05
 
 /* Submessage
  * submsg_fields is pointer to field descriptions */
-    PB_LTYPE_SUBMESSAGE = 0x06,
+#define PB_LTYPE_SUBMESSAGE 0x06
+
+/* Extension pseudo-field
+ * The field contains a pointer to pb_extension_t */
+#define PB_LTYPE_EXTENSION 0x07
 
 /* Number of declared LTYPES */
-    PB_LTYPES_COUNT = 7,
-    PB_LTYPE_MASK = 0x0F,
+#define PB_LTYPES_COUNT 8
+#define PB_LTYPE_MASK 0x0F
 
-    /******************
-     * Modifier flags *
-     ******************/
+/**** Field repetition rules ****/
 
-    /* Just the basic, write data at data_offset */
-    PB_HTYPE_REQUIRED = 0x00,
+#define PB_HTYPE_REQUIRED 0x00
+#define PB_HTYPE_OPTIONAL 0x10
+#define PB_HTYPE_REPEATED 0x20
+#define PB_HTYPE_MASK     0x30
 
-    /* Write true at size_offset */
-    PB_HTYPE_OPTIONAL = 0x10,
+/**** Field allocation types ****/
  
-    /* Read to pre-allocated array
-     * Maximum number of entries is array_size,
-     * actual number is stored at size_offset */
-    PB_HTYPE_ARRAY = 0x20,
-    
-    /* Works for all required/optional/repeated fields.
-     * data_offset points to pb_callback_t structure.
-     * LTYPE should be 0 (it is ignored, but sometimes
-     * used to speculatively index an array). */
-    PB_HTYPE_CALLBACK = 0x30,
-    
-    PB_HTYPE_MASK = 0xF0
-} pb_packed pb_type_t;
+#define PB_ATYPE_STATIC   0x00
+#define PB_ATYPE_CALLBACK 0x40
+#define PB_ATYPE_MASK     0xC0
 
+#define PB_ATYPE(x) ((x) & PB_ATYPE_MASK)
 #define PB_HTYPE(x) ((x) & PB_HTYPE_MASK)
 #define PB_LTYPE(x) ((x) & PB_LTYPE_MASK)
 
@@ -114,6 +181,7 @@ typedef enum {
  * structures. Fix that by defining PB_FIELD_16BIT or
  * PB_FIELD_32BIT.
  */
+PB_PACKED_STRUCT_START
 typedef struct _pb_field_t pb_field_t;
 struct _pb_field_t {
 
@@ -145,6 +213,18 @@ struct _pb_field_t {
      * If null, then field will zeroed. */
     const void *ptr;
 } pb_packed;
+PB_PACKED_STRUCT_END
+
+/* Make sure that the standard integer types are of the expected sizes.
+ * All kinds of things may break otherwise.. atleast all fixed* types. */
+STATIC_ASSERT(sizeof(int8_t) == 1, INT8_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(uint8_t) == 1, UINT8_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(int16_t) == 2, INT16_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(uint16_t) == 2, UINT16_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(int32_t) == 4, INT32_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(uint32_t) == 4, UINT32_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(int64_t) == 8, INT64_T_WRONG_SIZE)
+STATIC_ASSERT(sizeof(uint64_t) == 8, UINT64_T_WRONG_SIZE)
 
 /* This structure is used for 'bytes' arrays.
  * It has the number of bytes in the beginning, and after that an array.
@@ -179,10 +259,19 @@ typedef struct _pb_istream_t pb_istream_t;
 typedef struct _pb_ostream_t pb_ostream_t;
 typedef struct _pb_callback_t pb_callback_t;
 struct _pb_callback_t {
+#ifdef PB_OLD_CALLBACK_STYLE
+    /* Deprecated since nanopb-0.2.1 */
     union {
         bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg);
         bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, const void *arg);
     } funcs;
+#else
+    /* New function signature, which allows modifying arg contents in callback. */
+    union {
+        bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void **arg);
+        bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, void * const *arg);
+    } funcs;
+#endif    
     
     /* Free arg for use by callback */
     void *arg;
@@ -196,13 +285,146 @@ typedef enum {
     PB_WT_32BIT  = 5
 } pb_wire_type_t;
 
+/* Structure for defining the handling of unknown/extension fields.
+ * Usually the pb_extension_type_t structure is automatically generated,
+ * while the pb_extension_t structure is created by the user. However,
+ * if you want to catch all unknown fields, you can also create a custom
+ * pb_extension_type_t with your own callback.
+ */
+typedef struct _pb_extension_type_t pb_extension_type_t;
+typedef struct _pb_extension_t pb_extension_t;
+struct _pb_extension_type_t {
+    /* Called for each unknown field in the message.
+     * If you handle the field, read off all of its data and return true.
+     * If you do not handle the field, do not read anything and return true.
+     * If you run into an error, return false.
+     * Set to NULL for default handler.
+     */
+    bool (*decode)(pb_istream_t *stream, pb_extension_t *extension,
+                   uint32_t tag, pb_wire_type_t wire_type);
+    
+    /* Called once after all regular fields have been encoded.
+     * If you have something to write, do so and return true.
+     * If you do not have anything to write, just return true.
+     * If you run into an error, return false.
+     * Set to NULL for default handler.
+     */
+    bool (*encode)(pb_ostream_t *stream, const pb_extension_t *extension);
+    
+    /* Free field for use by the callback. */
+    const void *arg;
+};
+
+struct _pb_extension_t {
+    /* Type describing the extension field. Usually you'll initialize
+     * this to a pointer to the automatically generated structure. */
+    const pb_extension_type_t *type;
+    
+    /* Destination for the decoded data. This must match the datatype
+     * of the extension field. */
+    void *dest;
+    
+    /* Pointer to the next extension handler, or NULL.
+     * If this extension does not match a field, the next handler is
+     * automatically called. */
+    pb_extension_t *next;
+};
+
 /* These macros are used to declare pb_field_t's in the constant array. */
 #define pb_membersize(st, m) (sizeof ((st*)0)->m)
 #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0]))
 #define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2))
-#define pb_delta_end(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2))
+#define pb_delta_end(st, m1, m2) (int)(offsetof(st, m1) == offsetof(st, m2) \
+                                  ? offsetof(st, m1) \
+                                  : offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2))
 #define PB_LAST_FIELD {0,(pb_type_t) 0,0,0,0,0,0}
 
+/* Required fields are the simplest. They just have delta (padding) from
+ * previous field end, and the size of the field. Pointer is used for
+ * submessages and default values.
+ */
+#define PB_REQUIRED_STATIC(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | ltype, \
+    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+
+/* Optional fields add the delta to the has_ variable. */
+#define PB_OPTIONAL_STATIC(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \
+    pb_delta_end(st, m, pm), \
+    pb_delta(st, has_ ## m, m), \
+    pb_membersize(st, m), 0, ptr}
+
+/* Repeated fields have a _count field and also the maximum number of entries. */
+#define PB_REPEATED_STATIC(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | ltype, \
+    pb_delta_end(st, m, pm), \
+    pb_delta(st, m ## _count, m), \
+    pb_membersize(st, m[0]), \
+    pb_arraysize(st, m), ptr}
+
+/* Callbacks are much like required fields except with special datatype. */
+#define PB_REQUIRED_CALLBACK(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REQUIRED | ltype, \
+    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+
+#define PB_OPTIONAL_CALLBACK(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \
+    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+    
+#define PB_REPEATED_CALLBACK(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REPEATED | ltype, \
+    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+
+/* Optional extensions don't have the has_ field, as that would be redundant. */
+#define PB_OPTEXT_STATIC(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \
+    0, \
+    0, \
+    pb_membersize(st, m), 0, ptr}
+
+#define PB_OPTEXT_CALLBACK(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \
+    0, 0, pb_membersize(st, m), 0, ptr}
+
+/* The mapping from protobuf types to LTYPEs is done using these macros. */
+#define PB_LTYPE_MAP_BOOL       PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_BYTES      PB_LTYPE_BYTES
+#define PB_LTYPE_MAP_DOUBLE     PB_LTYPE_FIXED64
+#define PB_LTYPE_MAP_ENUM       PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_FIXED32    PB_LTYPE_FIXED32
+#define PB_LTYPE_MAP_FIXED64    PB_LTYPE_FIXED64
+#define PB_LTYPE_MAP_FLOAT      PB_LTYPE_FIXED32
+#define PB_LTYPE_MAP_INT32      PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_INT64      PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_MESSAGE    PB_LTYPE_SUBMESSAGE
+#define PB_LTYPE_MAP_SFIXED32   PB_LTYPE_FIXED32
+#define PB_LTYPE_MAP_SFIXED64   PB_LTYPE_FIXED64
+#define PB_LTYPE_MAP_SINT32     PB_LTYPE_SVARINT
+#define PB_LTYPE_MAP_SINT64     PB_LTYPE_SVARINT
+#define PB_LTYPE_MAP_STRING     PB_LTYPE_STRING
+#define PB_LTYPE_MAP_UINT32     PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_UINT64     PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_EXTENSION  PB_LTYPE_EXTENSION
+
+/* This is the actual macro used in field descriptions.
+ * It takes these arguments:
+ * - Field tag number
+ * - Field type:   BOOL, BYTES, DOUBLE, ENUM, FIXED32, FIXED64,
+ *                 FLOAT, INT32, INT64, MESSAGE, SFIXED32, SFIXED64
+ *                 SINT32, SINT64, STRING, UINT32, UINT64 or EXTENSION
+ * - Field rules:  REQUIRED, OPTIONAL or REPEATED
+ * - Allocation:   STATIC or CALLBACK
+ * - Message name
+ * - Field name
+ * - Previous field name (or field name again for first field)
+ * - Pointer to default value or submsg fields.
+ */
+
+#define PB_FIELD(tag, type, rules, allocation, message, field, prevfield, ptr) \
+    PB_ ## rules ## _ ## allocation(tag, message, field, prevfield, \
+                                    PB_LTYPE_MAP_ ## type, ptr)
+
+
 /* These macros are used for giving out error messages.
  * They are mostly a debugging aid; the main error information
  * is the true/false return value from functions.

pb_decode.c

@@ -14,7 +14,6 @@
 #define NANOPB_INTERNALS
 #include "pb.h"
 #include "pb_decode.h"
-#include <string.h>
 
 typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn;
 
@@ -29,7 +28,8 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = {
     
     &pb_dec_bytes,
     &pb_dec_string,
-    &pb_dec_submessage
+    &pb_dec_submessage,
+    NULL /* extensions */
 };
 
 /**************
@@ -39,16 +39,20 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = {
 static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count)
 {
     uint8_t *source = (uint8_t*)stream->state;
+    stream->state = source + count;
     
     if (buf != NULL)
-        memcpy(buf, source, count);
+    {
+        while (count--)
+            *buf++ = *source++;
+    }
     
-    stream->state = source + count;
     return true;
 }
 
 bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count)
 {
+#ifndef PB_BUFFER_ONLY
 	if (buf == NULL && stream->callback != buf_read)
 	{
 		/* Skip input bytes */
@@ -63,12 +67,18 @@ bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count)
 		
 		return pb_read(stream, tmp, count);
 	}
+#endif
 
     if (stream->bytes_left < count)
         PB_RETURN_ERROR(stream, "end-of-stream");
     
+#ifndef PB_BUFFER_ONLY
     if (!stream->callback(stream, buf, count))
         PB_RETURN_ERROR(stream, "io error");
+#else
+    if (!buf_read(stream, buf, count))
+        return false;
+#endif
     
     stream->bytes_left -= count;
     return true;
@@ -77,9 +87,16 @@ bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count)
 pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize)
 {
     pb_istream_t stream;
+#ifdef PB_BUFFER_ONLY
+    stream.callback = NULL;
+#else
     stream.callback = &buf_read;
+#endif
     stream.state = buf;
     stream.bytes_left = bufsize;
+#ifndef PB_NO_ERRMSG
+    stream.errmsg = NULL;
+#endif
     return stream;
 }
 
@@ -89,28 +106,60 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize)
 
 static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest)
 {
-    uint64_t temp;
-    bool status = pb_decode_varint(stream, &temp);
-    *dest = (uint32_t)temp;
-    return status;
+    uint8_t byte;
+    uint32_t result;
+    
+    if (!pb_read(stream, &byte, 1))
+        return false;
+    
+    if (!(byte & 0x80))
+    {
+        /* Quick case, 1 byte value */
+        result = byte;
+    }
+    else
+    {
+        /* Multibyte case */
+        uint8_t bitpos = 7;
+        result = byte & 0x7F;
+        
+        do
+        {
+            if (bitpos >= 32)
+                PB_RETURN_ERROR(stream, "varint overflow");
+            
+            if (!pb_read(stream, &byte, 1))
+                return false;
+            
+            result |= (uint32_t)(byte & 0x7F) << bitpos;
+            bitpos = (uint8_t)(bitpos + 7);
+        } while (byte & 0x80);
+   }
+   
+   *dest = result;
+   return true;
 }
 
 bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest)
 {
     uint8_t byte;
-    int bitpos = 0;
-    *dest = 0;
+    uint8_t bitpos = 0;
+    uint64_t result = 0;
     
-    while (bitpos < 64 && pb_read(stream, &byte, 1))
+    do
     {
-        *dest |= (uint64_t)(byte & 0x7F) << bitpos;
-        bitpos += 7;
-        
-        if (!(byte & 0x80))
-            return true;
-    }
-    
+        if (bitpos >= 64)
             PB_RETURN_ERROR(stream, "varint overflow");
+        
+        if (!pb_read(stream, &byte, 1))
+            return false;
+
+        result |= (uint64_t)(byte & 0x7F) << bitpos;
+        bitpos = (uint8_t)(bitpos + 7);
+    } while (byte & 0x80);
+    
+    *dest = result;
+    return true;
 }
 
 bool checkreturn pb_skip_varint(pb_istream_t *stream)
@@ -222,14 +271,18 @@ bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *su
 void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream)
 {
     stream->state = substream->state;
+
+#ifndef PB_NO_ERRMSG
+    stream->errmsg = substream->errmsg;
+#endif
 }
 
 /* Iterator for pb_field_t list */
 typedef struct {
     const pb_field_t *start; /* Start of the pb_field_t array */
-    const pb_field_t *current; /* Current position of the iterator */
-    int field_index; /* Zero-based index of the field. */
-    int required_field_index; /* Zero-based index that counts only the required fields */
+    const pb_field_t *pos; /* Current position of the iterator */
+    unsigned field_index; /* Zero-based index of the field. */
+    unsigned required_field_index; /* Zero-based index that counts only the required fields */
     void *dest_struct; /* Pointer to the destination structure to decode to */
     void *pData; /* Pointer where to store current field value */
     void *pSize; /* Pointer where to store the size of current array field */
@@ -237,30 +290,36 @@ typedef struct {
 
 static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct)
 {
-    iter->start = iter->current = fields;
+    iter->start = iter->pos = fields;
     iter->field_index = 0;
     iter->required_field_index = 0;
-    iter->pData = (char*)dest_struct + iter->current->data_offset;
-    iter->pSize = (char*)iter->pData + iter->current->size_offset;
+    iter->pData = (char*)dest_struct + iter->pos->data_offset;
+    iter->pSize = (char*)iter->pData + iter->pos->size_offset;
     iter->dest_struct = dest_struct;
 }
 
 static bool pb_field_next(pb_field_iterator_t *iter)
 {
     bool notwrapped = true;
-    size_t prev_size = iter->current->data_size;
+    size_t prev_size = iter->pos->data_size;
     
-    if (PB_HTYPE(iter->current->type) == PB_HTYPE_ARRAY)
-        prev_size *= iter->current->array_size;
+    if (PB_ATYPE(iter->pos->type) == PB_ATYPE_STATIC &&
+        PB_HTYPE(iter->pos->type) == PB_HTYPE_REPEATED)
+    {
+        prev_size *= iter->pos->array_size;
+    }
     
-    if (PB_HTYPE(iter->current->type) == PB_HTYPE_REQUIRED)
+    if (iter->pos->tag == 0)
+        return false; /* Only happens with empty message types */
+    
+    if (PB_HTYPE(iter->pos->type) == PB_HTYPE_REQUIRED)
         iter->required_field_index++;
     
-    iter->current++;
+    iter->pos++;
     iter->field_index++;
-    if (iter->current->tag == 0)
+    if (iter->pos->tag == 0)
     {
-        iter->current = iter->start;
+        iter->pos = iter->start;
         iter->field_index = 0;
         iter->required_field_index = 0;
         iter->pData = iter->dest_struct;
@@ -268,18 +327,21 @@ static bool pb_field_next(pb_field_iterator_t *iter)
         notwrapped = false;
     }
     
-    iter->pData = (char*)iter->pData + prev_size + iter->current->data_offset;
-    iter->pSize = (char*)iter->pData + iter->current->size_offset;
+    iter->pData = (char*)iter->pData + prev_size + iter->pos->data_offset;
+    iter->pSize = (char*)iter->pData + iter->pos->size_offset;
     return notwrapped;
 }
 
 static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag)
 {
-    int start = iter->field_index;
+    unsigned start = iter->field_index;
     
     do {
-        if (iter->current->tag == tag)
+        if (iter->pos->tag == tag &&
+            PB_LTYPE(iter->pos->type) != PB_LTYPE_EXTENSION)
+        {
             return true;
+        }
         pb_field_next(iter);
     } while (iter->field_index != start);
     
@@ -290,57 +352,78 @@ static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag)
  * Decode a single field *
  *************************/
 
-static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
+static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
 {
-    pb_decoder_t func = PB_DECODERS[PB_LTYPE(iter->current->type)];
+    pb_type_t type;
+    pb_decoder_t func;
     
-    switch (PB_HTYPE(iter->current->type))
+    type = iter->pos->type;
+    func = PB_DECODERS[PB_LTYPE(type)];
+
+    switch (PB_HTYPE(type))
     {
         case PB_HTYPE_REQUIRED:
-            return func(stream, iter->current, iter->pData);
+            return func(stream, iter->pos, iter->pData);
             
         case PB_HTYPE_OPTIONAL:
             *(bool*)iter->pSize = true;
-            return func(stream, iter->current, iter->pData);
+            return func(stream, iter->pos, iter->pData);
     
-        case PB_HTYPE_ARRAY:
+        case PB_HTYPE_REPEATED:
             if (wire_type == PB_WT_STRING
-                && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE)
+                && PB_LTYPE(type) <= PB_LTYPE_LAST_PACKABLE)
             {
                 /* Packed array */
-                bool status;
+                bool status = true;
                 size_t *size = (size_t*)iter->pSize;
                 pb_istream_t substream;
                 if (!pb_make_string_substream(stream, &substream))
                     return false;
                 
-                while (substream.bytes_left && *size < iter->current->array_size)
+                while (substream.bytes_left && *size < iter->pos->array_size)
                 {
-                    void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size);
-                    if (!func(&substream, iter->current, pItem))
-                        return false;
+                    void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size);
+                    if (!func(&substream, iter->pos, pItem))
+                    {
+                        status = false;
+                        break;
+                    }
                     (*size)++;
                 }
-                status = (substream.bytes_left == 0);
                 pb_close_string_substream(stream, &substream);
+                
+                if (substream.bytes_left != 0)
+                    PB_RETURN_ERROR(stream, "array overflow");
+                
                 return status;
             }
             else
             {
                 /* Repeated field */
                 size_t *size = (size_t*)iter->pSize;
-                void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size);
-                if (*size >= iter->current->array_size)
+                void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size);
+                if (*size >= iter->pos->array_size)
                     PB_RETURN_ERROR(stream, "array overflow");
                 
                 (*size)++;
-                return func(stream, iter->current, pItem);
+                return func(stream, iter->pos, pItem);
             }
 
-        case PB_HTYPE_CALLBACK:
+        default:
+            PB_RETURN_ERROR(stream, "invalid field type");
+    }
+}
+
+static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
 {
     pb_callback_t *pCallback = (pb_callback_t*)iter->pData;
     
+#ifdef PB_OLD_CALLBACK_STYLE
+    void *arg = pCallback->arg;
+#else
+    void **arg = &(pCallback->arg);
+#endif
+    
     if (pCallback->funcs.decode == NULL)
         return pb_skip_field(stream, wire_type);
     
@@ -351,11 +434,11 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
         if (!pb_make_string_substream(stream, &substream))
             return false;
         
-                while (substream.bytes_left)
+        do
         {
-                    if (!pCallback->funcs.decode(&substream, iter->current, pCallback->arg))
+            if (!pCallback->funcs.decode(&substream, iter->pos, arg))
                 PB_RETURN_ERROR(stream, "callback failed");
-                }
+        } while (substream.bytes_left);
         
         pb_close_string_substream(stream, &substream);
         return true;
@@ -374,15 +457,89 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
             return false;
         substream = pb_istream_from_buffer(buffer, size);
         
-                return pCallback->funcs.decode(&substream, iter->current, pCallback->arg);
+        return pCallback->funcs.decode(&substream, iter->pos, arg);
     }
 }
 
+static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
+{
+    switch (PB_ATYPE(iter->pos->type))
+    {
+        case PB_ATYPE_STATIC:
+            return decode_static_field(stream, wire_type, iter);
+        
+        case PB_ATYPE_CALLBACK:
+            return decode_callback_field(stream, wire_type, iter);
+        
         default:
             PB_RETURN_ERROR(stream, "invalid field type");
     }
 }
 
+/* Default handler for extension fields. Expects a pb_field_t structure
+ * in extension->type->arg. */
+static bool checkreturn default_extension_handler(pb_istream_t *stream,
+    pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type)
+{
+    const pb_field_t *field = (const pb_field_t*)extension->type->arg;
+    pb_field_iterator_t iter;
+    bool dummy;
+    
+    if (field->tag != tag)
+        return true;
+    
+    iter.start = field;
+    iter.pos = field;
+    iter.field_index = 0;
+    iter.required_field_index = 0;
+    iter.dest_struct = extension->dest;
+    iter.pData = extension->dest;
+    iter.pSize = &dummy;
+    
+    return decode_field(stream, wire_type, &iter);
+}
+
+/* Try to decode an unknown field as an extension field. Tries each extension
+ * decoder in turn, until one of them handles the field or loop ends. */
+static bool checkreturn decode_extension(pb_istream_t *stream,
+    uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
+{
+    pb_extension_t *extension = *(pb_extension_t* const *)iter->pData;
+    size_t pos = stream->bytes_left;
+    
+    while (extension && pos == stream->bytes_left)
+    {
+        bool status;
+        if (extension->type->decode)
+            status = extension->type->decode(stream, extension, tag, wire_type);
+        else
+            status = default_extension_handler(stream, extension, tag, wire_type);
+
+        if (!status)
+            return false;
+        
+        extension = extension->next;
+    }
+    
+    return true;
+}
+
+/* Step through the iterator until an extension field is found or until all
+ * entries have been checked. There can be only one extension field per
+ * message. Returns false if no extension field is found. */
+static bool checkreturn find_extension_field(pb_field_iterator_t *iter)
+{
+    unsigned start = iter->field_index;
+    
+    do {
+        if (PB_LTYPE(iter->pos->type) == PB_LTYPE_EXTENSION)
+            return true;
+        pb_field_next(iter);
+    } while (iter->field_index != start);
+    
+    return false;
+}
+
 /* Initialize message fields to default values, recursively */
 static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct)
 {
@@ -392,36 +549,42 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str
     /* Initialize size/has fields and apply default values */
     do
     {
-        if (iter.current->tag == 0)
+        pb_type_t type;
+        type = iter.pos->type;
+    
+        if (iter.pos->tag == 0)
             continue;
         
+        if (PB_ATYPE(type) == PB_ATYPE_STATIC)
+        {
             /* Initialize the size field for optional/repeated fields to 0. */
-        if (PB_HTYPE(iter.current->type) == PB_HTYPE_OPTIONAL)
+            if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL)
             {
                 *(bool*)iter.pSize = false;
             }
-        else if (PB_HTYPE(iter.current->type) == PB_HTYPE_ARRAY)
+            else if (PB_HTYPE(type) == PB_HTYPE_REPEATED)
             {
                 *(size_t*)iter.pSize = 0;
                 continue; /* Array is empty, no need to initialize contents */
             }
             
             /* Initialize field contents to default value */
-        if (PB_HTYPE(iter.current->type) == PB_HTYPE_CALLBACK)
+            if (PB_LTYPE(iter.pos->type) == PB_LTYPE_SUBMESSAGE)
             {
-            continue; /* Don't overwrite callback */
+                pb_message_set_to_defaults((const pb_field_t *) iter.pos->ptr, iter.pData);
             }
-        else if (PB_LTYPE(iter.current->type) == PB_LTYPE_SUBMESSAGE)
+            else if (iter.pos->ptr != NULL)
             {
-            pb_message_set_to_defaults((const pb_field_t *) iter.current->ptr, iter.pData);
-        }
-        else if (iter.current->ptr != NULL)
-        {
-            memcpy(iter.pData, iter.current->ptr, iter.current->data_size);
+                memcpy(iter.pData, iter.pos->ptr, iter.pos->data_size);
             }
             else
             {
-            memset(iter.pData, 0, iter.current->data_size);
+                memset(iter.pData, 0, iter.pos->data_size);
+            }
+        }
+        else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK)
+        {
+            continue; /* Don't overwrite callback */
         }
     } while (pb_field_next(&iter));
 }
@@ -433,6 +596,7 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str
 bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct)
 {
     uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0}; /* Used to check for required fields */
+    uint32_t extension_range_start = 0;
     pb_field_iterator_t iter;
     
     pb_field_init(&iter, fields, dest_struct);
@@ -453,13 +617,36 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[
         
         if (!pb_field_find(&iter, tag))
         {
+            /* No match found, check if it matches an extension. */
+            if (tag >= extension_range_start)
+            {
+                if (!find_extension_field(&iter))
+                    extension_range_start = (uint32_t)-1;
+                else
+                    extension_range_start = iter.pos->tag;
+                
+                if (tag >= extension_range_start)
+                {
+                    size_t pos = stream->bytes_left;
+                
+                    if (!decode_extension(stream, tag, wire_type, &iter))
+                        return false;
+                    
+                    if (pos != stream->bytes_left)
+                    {
+                        /* The field was handled */
+                        continue;                    
+                    }
+                }
+            }
+        
             /* No match found, skip data */
             if (!pb_skip_field(stream, wire_type))
                 return false;
             continue;
         }
         
-        if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED
+        if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REQUIRED
             && iter.required_field_index < PB_MAX_REQUIRED_FIELDS)
         {
             fields_seen[iter.required_field_index >> 3] |= (uint8_t)(1 << (iter.required_field_index & 7));
@@ -470,15 +657,34 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[
     }
     
     /* Check that all required fields were present. */
-    pb_field_init(&iter, fields, dest_struct);
-    do {
-        if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED &&
-            iter.required_field_index < PB_MAX_REQUIRED_FIELDS &&
-            !(fields_seen[iter.required_field_index >> 3] & (1 << (iter.required_field_index & 7))))
     {
+        /* First figure out the number of required fields by
+         * seeking to the end of the field array. Usually we
+         * are already close to end after decoding.
+         */
+        unsigned req_field_count;
+        pb_type_t last_type;
+        unsigned i;
+        do {
+            req_field_count = iter.required_field_index;
+            last_type = iter.pos->type;
+        } while (pb_field_next(&iter));
+        
+        /* Fixup if last field was also required. */
+        if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag)
+            req_field_count++;
+        
+        /* Check the whole bytes */
+        for (i = 0; i < (req_field_count >> 3); i++)
+        {
+            if (fields_seen[i] != 0xFF)
+                PB_RETURN_ERROR(stream, "missing required field");
+        }
+        
+        /* Check the remaining bits */
+        if (fields_seen[req_field_count >> 3] != (0xFF >> (8 - (req_field_count & 7))))
             PB_RETURN_ERROR(stream, "missing required field");
     }
-    } while (pb_field_next(&iter));
     
     return true;
 }
@@ -489,6 +695,19 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void
     return pb_decode_noinit(stream, fields, dest_struct);
 }
 
+bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct)
+{
+    pb_istream_t substream;
+    bool status;
+    
+    if (!pb_make_string_substream(stream, &substream))
+        return false;
+    
+    status = pb_decode(&substream, fields, dest_struct);
+    pb_close_string_substream(stream, &substream);
+    return status;
+}
+
 /* Field decoders */
 
 bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest)
@@ -550,7 +769,8 @@ bool pb_decode_fixed64(pb_istream_t *stream, void *dest)
 bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest)
 {
     uint64_t value;
-    bool status = pb_decode_varint(stream, &value);
+    if (!pb_decode_varint(stream, &value))
+        return false;
     
     switch (field->data_size)
     {
@@ -561,13 +781,14 @@ bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, vo
         default: PB_RETURN_ERROR(stream, "invalid data_size");
     }
     
-    return status;
+    return true;
 }
 
 bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest)
 {
     int64_t value;
-    bool status = pb_decode_svarint(stream, &value);
+    if (!pb_decode_svarint(stream, &value))
+        return false;
     
     switch (field->data_size)
     {
@@ -576,7 +797,7 @@ bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, v
         default: PB_RETURN_ERROR(stream, "invalid data_size");
     }
     
-    return status;
+    return true;
 }
 
 bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest)
@@ -637,7 +858,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field
     
     /* New array entries need to be initialized, while required and optional
      * submessages have already been initialized in the top-level pb_decode. */
-    if (PB_HTYPE(field->type) == PB_HTYPE_ARRAY)
+    if (PB_HTYPE(field->type) == PB_HTYPE_REPEATED)
         status = pb_decode(&substream, submsg_fields, dest);
     else
         status = pb_decode_noinit(&substream, submsg_fields, dest);

pb_decode.h

@@ -1,38 +1,42 @@
+/* pb_decode.h: Functions to decode protocol buffers. Depends on pb_decode.c.
+ * The main function is pb_decode. You also need an input stream, and the
+ * field descriptions created by nanopb_generator.py.
+ */
+
 #ifndef _PB_DECODE_H_
 #define _PB_DECODE_H_
 
-/* pb_decode.h: Functions to decode protocol buffers. Depends on pb_decode.c.
- * The main function is pb_decode. You will also need to create an input
- * stream, which is easiest to do with pb_istream_from_buffer().
- * 
- * You also need structures and their corresponding pb_field_t descriptions.
- * These are usually generated from .proto-files with a script.
- */
-
-#include <stdbool.h>
 #include "pb.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/* Lightweight input stream.
- * You can provide a callback function for reading or use
- * pb_istream_from_buffer.
+/* Structure for defining custom input streams. You will need to provide
+ * a callback function to read the bytes from your storage, which can be
+ * for example a file or a network socket.
+ * 
+ * The callback must conform to these rules:
  *
- * Rules for callback:
  * 1) Return false on IO errors. This will cause decoding to abort.
- * 
  * 2) You can use state to store your own data (e.g. buffer pointer),
  *    and rely on pb_read to verify that no-body reads past bytes_left.
- * 
  * 3) Your callback may be used with substreams, in which case bytes_left
  *    is different than from the main stream. Don't use bytes_left to compute
  *    any pointers.
  */
 struct _pb_istream_t
 {
+#ifdef PB_BUFFER_ONLY
+    /* Callback pointer is not used in buffer-only configuration.
+     * Having an int pointer here allows binary compatibility but
+     * gives an error if someone tries to assign callback function.
+     */
+    int *callback;
+#else
     bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count);
+#endif
+
     void *state; /* Free field for use by callback implementation */
     size_t bytes_left;
     
@@ -41,24 +45,64 @@ struct _pb_istream_t
 #endif
 };
 
-pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize);
-bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count);
+/***************************
+ * Main decoding functions *
+ ***************************/
  
-/* Decode from stream to destination struct.
+/* Decode a single protocol buffers message from input stream into a C structure.
  * Returns true on success, false on any failure.
  * The actual struct pointed to by dest must match the description in fields.
+ * Callback fields of the destination structure must be initialized by caller.
+ * All other fields will be initialized by this function.
+ *
+ * Example usage:
+ *    MyMessage msg = {};
+ *    uint8_t buffer[64];
+ *    pb_istream_t stream;
+ *    
+ *    // ... read some data into buffer ...
+ *
+ *    stream = pb_istream_from_buffer(buffer, count);
+ *    pb_decode(&stream, MyMessage_fields, &msg);
  */
 bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct);
 
 /* Same as pb_decode, except does not initialize the destination structure
  * to default values. This is slightly faster if you need no default values
  * and just do memset(struct, 0, sizeof(struct)) yourself.
+ *
+ * This can also be used for 'merging' two messages, i.e. update only the
+ * fields that exist in the new message.
  */
 bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct);
 
-/* --- Helper functions ---
- * You may want to use these from your caller or callbacks.
+/* Same as pb_decode, except expects the stream to start with the message size
+ * encoded as varint. Corresponds to parseDelimitedFrom() in Google's
+ * protobuf API.
  */
+bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct);
+
+
+/**************************************
+ * Functions for manipulating streams *
+ **************************************/
+
+/* Create an input stream for reading from a memory buffer.
+ *
+ * Alternatively, you can use a custom stream that reads directly from e.g.
+ * a file or a network socket.
+ */
+pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize);
+
+/* Function to read from a pb_istream_t. You can use this if you need to
+ * read some custom header data, or to read data in field callbacks.
+ */
+bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count);
+
+
+/************************************************
+ * Helper functions for writing field callbacks *
+ ************************************************/
 
 /* Decode the tag for the next field in the stream. Gives the wire type and
  * field tag. At end of the message, returns false and sets eof to true. */
@@ -87,10 +131,10 @@ bool pb_decode_fixed64(pb_istream_t *stream, void *dest);
 bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream);
 void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream);
 
-/* --- Internal functions ---
- * These functions are not terribly useful for the average library user, but
- * are exported to make the unit testing and extending nanopb easier.
- */
+
+/*******************************
+ * Internal / legacy functions *
+ *******************************/
 
 #ifdef NANOPB_INTERNALS
 bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest);

pb_encode.c

@@ -6,7 +6,6 @@
 #define NANOPB_INTERNALS
 #include "pb.h"
 #include "pb_encode.h"
-#include <string.h>
 
 /* The warn_unused_result attribute appeared first in gcc-3.4.0 */
 #if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4)
@@ -29,7 +28,8 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = {
     
     &pb_enc_bytes,
     &pb_enc_string,
-    &pb_enc_submessage
+    &pb_enc_submessage,
+    NULL /* extensions */
 };
 
 /* pb_ostream_t implementation */
@@ -37,18 +37,28 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = {
 static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count)
 {
     uint8_t *dest = (uint8_t*)stream->state;
-    memcpy(dest, buf, count);
     stream->state = dest + count;
+    
+    while (count--)
+        *dest++ = *buf++;
+    
     return true;
 }
 
 pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize)
 {
     pb_ostream_t stream;
+#ifdef PB_BUFFER_ONLY
+    stream.callback = (void*)1; /* Just some marker value */
+#else
     stream.callback = &buf_write;
+#endif
     stream.state = buf;
     stream.max_size = bufsize;
     stream.bytes_written = 0;
+#ifndef PB_NO_ERRMSG
+    stream.errmsg = NULL;
+#endif
     return stream;
 }
 
@@ -57,10 +67,15 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count
     if (stream->callback != NULL)
     {
         if (stream->bytes_written + count > stream->max_size)
-            return false;
+            PB_RETURN_ERROR(stream, "stream full");
 
+#ifdef PB_BUFFER_ONLY
+        if (!buf_write(stream, buf, count))
+            PB_RETURN_ERROR(stream, "io error");
+#else        
         if (!stream->callback(stream, buf, count))
-            return false;
+            PB_RETURN_ERROR(stream, "io error");
+#endif
     }
     
     stream->bytes_written += count;
@@ -69,10 +84,7 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count
 
 /* Main encoding stuff */
 
-/* Callbacks don't need this function because they usually know the data type
- * without examining the field structure.
- * Therefore it is static for now.
- */
+/* Encode a static array. Handles the size calculations and possible packing. */
 static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field,
                          const void *pData, size_t count, pb_encoder_t func)
 {
@@ -83,6 +95,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
     if (count == 0)
         return true;
     
+    /* We always pack arrays if the datatype allows it. */
     if (PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE)
     {
         if (!pb_encode_tag(stream, PB_WT_STRING, field->tag))
@@ -99,7 +112,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
         }
         else
         { 
-            pb_ostream_t sizestream = {0,0,0,0};
+            pb_ostream_t sizestream = PB_OSTREAM_SIZING;
             p = pData;
             for (i = 0; i < count; i++)
             {
@@ -141,22 +154,21 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
     return true;
 }
 
-bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct)
+/* Encode a field with static allocation, i.e. one whose data is stored
+ * in the structure itself. */
+static bool checkreturn encode_static_field(pb_ostream_t *stream,
+    const pb_field_t *field, const void *pData)
 {
-    const pb_field_t *field = fields;
-    const void *pData = src_struct;
+    pb_encoder_t func;
     const void *pSize;
-    size_t prev_size = 0;
+    bool dummy = true;
     
-    while (field->tag != 0)
-    {
-        pb_encoder_t func = PB_ENCODERS[PB_LTYPE(field->type)];
-        pData = (const char*)pData + prev_size + field->data_offset;
+    func = PB_ENCODERS[PB_LTYPE(field->type)];
+    
+    if (field->size_offset)
         pSize = (const char*)pData + field->size_offset;
-        
-        prev_size = field->data_size;
-        if (PB_HTYPE(field->type) == PB_HTYPE_ARRAY)
-            prev_size *= field->array_size;
+    else
+        pSize = &dummy;
     
     switch (PB_HTYPE(field->type))
     {
@@ -178,21 +190,119 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons
             }
             break;
         
-            case PB_HTYPE_ARRAY:
+        case PB_HTYPE_REPEATED:
             if (!encode_array(stream, field, pData, *(const size_t*)pSize, func))
                 return false;
             break;
         
-            case PB_HTYPE_CALLBACK:
+        default:
+            PB_RETURN_ERROR(stream, "invalid field type");
+    }
+    
+    return true;
+}
+
+/* Encode a field with callback semantics. This means that a user function is
+ * called to provide and encode the actual data. */
+static bool checkreturn encode_callback_field(pb_ostream_t *stream,
+    const pb_field_t *field, const void *pData)
 {
     const pb_callback_t *callback = (const pb_callback_t*)pData;
+    
+#ifdef PB_OLD_CALLBACK_STYLE
+    const void *arg = callback->arg;
+#else
+    void * const *arg = &(callback->arg);
+#endif    
+    
     if (callback->funcs.encode != NULL)
     {
-                    if (!callback->funcs.encode(stream, field, callback->arg))
+        if (!callback->funcs.encode(stream, field, arg))
+            PB_RETURN_ERROR(stream, "callback error");
+    }
+    return true;
+}
+
+/* Encode a single field of any callback or static type. */
+static bool checkreturn encode_field(pb_ostream_t *stream,
+    const pb_field_t *field, const void *pData)
+{
+    switch (PB_ATYPE(field->type))
+    {
+        case PB_ATYPE_STATIC:
+            return encode_static_field(stream, field, pData);
+        
+        case PB_ATYPE_CALLBACK:
+            return encode_callback_field(stream, field, pData);
+        
+        default:
+            PB_RETURN_ERROR(stream, "invalid field type");
+    }
+}
+
+/* Default handler for extension fields. Expects to have a pb_field_t
+ * pointer in the extension->type->arg field. */
+static bool checkreturn default_extension_handler(pb_ostream_t *stream,
+    const pb_extension_t *extension)
+{
+    const pb_field_t *field = (const pb_field_t*)extension->type->arg;
+    return encode_field(stream, field, extension->dest);
+}
+
+/* Walk through all the registered extensions and give them a chance
+ * to encode themselves. */
+static bool checkreturn encode_extension_field(pb_ostream_t *stream,
+    const pb_field_t *field, const void *pData)
+{
+    const pb_extension_t *extension = *(const pb_extension_t* const *)pData;
+    UNUSED(field);
+    
+    while (extension)
+    {
+        bool status;
+        if (extension->type->encode)
+            status = extension->type->encode(stream, extension);
+        else
+            status = default_extension_handler(stream, extension);
+
+        if (!status)
+            return false;
+        
+        extension = extension->next;
+    }
+    
+    return true;
+}
+
+bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct)
+{
+    const pb_field_t *field = fields;
+    const void *pData = src_struct;
+    size_t prev_size = 0;
+    
+    while (field->tag != 0)
+    {
+        pData = (const char*)pData + prev_size + field->data_offset;
+        prev_size = field->data_size;
+        
+        /* Special case for static arrays */
+        if (PB_ATYPE(field->type) == PB_ATYPE_STATIC &&
+            PB_HTYPE(field->type) == PB_HTYPE_REPEATED)
+        {
+            prev_size *= field->array_size;
+        }
+        
+        if (PB_LTYPE(field->type) == PB_LTYPE_EXTENSION)
+        {
+            /* Special case for the extension field placeholder */
+            if (!encode_extension_field(stream, field, pData))
                 return false;
         }
-                break;
-            }
+        else
+        {
+            /* Regular field */
+            if (!encode_field(stream, field, pData))
+                return false;
         }
     
         field++;
@@ -201,6 +311,11 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons
     return true;
 }
 
+bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct)
+{
+    return pb_encode_submessage(stream, fields, src_struct);
+}
+
 /* Helper functions */
 bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value)
 {
@@ -297,7 +412,7 @@ bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t
             break;
         
         default:
-            return false;
+            PB_RETURN_ERROR(stream, "invalid field type");
     }
     
     return pb_encode_tag(stream, wiretype, field->tag);
@@ -314,7 +429,7 @@ bool checkreturn pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, s
 bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct)
 {
     /* First calculate the message size using a non-writing substream. */
-    pb_ostream_t substream = {0,0,0,0};
+    pb_ostream_t substream = PB_OSTREAM_SIZING;
     size_t size;
     bool status;
     
@@ -330,7 +445,7 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie
         return pb_write(stream, NULL, size); /* Just sizing */
     
     if (stream->bytes_written + size > stream->max_size)
-        return false;
+        PB_RETURN_ERROR(stream, "stream full");
         
     /* Use a substream to verify that a callback doesn't write more than
      * what it did the first time. */
@@ -338,14 +453,20 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie
     substream.state = stream->state;
     substream.max_size = size;
     substream.bytes_written = 0;
+#ifndef PB_NO_ERRMSG
+    substream.errmsg = NULL;
+#endif
     
     status = pb_encode(&substream, fields, src_struct);
     
     stream->bytes_written += substream.bytes_written;
     stream->state = substream.state;
+#ifndef PB_NO_ERRMSG
+    stream->errmsg = substream.errmsg;
+#endif
     
     if (substream.bytes_written != size)
-        return false;
+        PB_RETURN_ERROR(stream, "submsg size changed");
     
     return status;
 }
@@ -362,7 +483,7 @@ bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, co
         case 2: value = *(const uint16_t*)src; break;
         case 4: value = *(const uint32_t*)src; break;
         case 8: value = *(const uint64_t*)src; break;
-        default: return false;
+        default: PB_RETURN_ERROR(stream, "invalid data_size");
     }
     
     return pb_encode_varint(stream, value);
@@ -376,7 +497,7 @@ bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, c
     {
         case 4: value = *(const int32_t*)src; break;
         case 8: value = *(const int64_t*)src; break;
-        default: return false;
+        default: PB_RETURN_ERROR(stream, "invalid data_size");
     }
     
     return pb_encode_svarint(stream, value);
@@ -403,14 +524,22 @@ bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, con
 
 bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
-    UNUSED(field);
-    return pb_encode_string(stream, (const uint8_t*)src, strlen((const char*)src));
+    /* strnlen() is not always available, so just use a for-loop */
+    size_t size = 0;
+    const char *p = (const char*)src;
+    while (size < field->data_size && *p != '\0')
+    {
+        size++;
+        p++;
+    }
+
+    return pb_encode_string(stream, (const uint8_t*)src, size);
 }
 
 bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
     if (field->ptr == NULL)
-        return false;
+        PB_RETURN_ERROR(stream, "invalid field descriptor");
     
     return pb_encode_submessage(stream, (const pb_field_t*)field->ptr, src);
 }

pb_encode.h

@@ -1,63 +1,120 @@
+/* pb_encode.h: Functions to encode protocol buffers. Depends on pb_encode.c.
+ * The main function is pb_encode. You also need an output stream, and the
+ * field descriptions created by nanopb_generator.py.
+ */
+
 #ifndef _PB_ENCODE_H_
 #define _PB_ENCODE_H_
 
-/* pb_encode.h: Functions to encode protocol buffers. Depends on pb_encode.c.
- * The main function is pb_encode. You also need an output stream, structures
- * and their field descriptions (just like with pb_decode).
- */
-
-#include <stdbool.h>
 #include "pb.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/* Lightweight output stream.
- * You can provide callback for writing or use pb_ostream_from_buffer.
+/* Structure for defining custom output streams. You will need to provide
+ * a callback function to write the bytes to your storage, which can be
+ * for example a file or a network socket.
  *
- * Alternatively, callback can be NULL in which case the stream will just
- * count the number of bytes that would have been written. In this case
- * max_size is not checked.
+ * The callback must conform to these rules:
  *
- * Rules for callback:
  * 1) Return false on IO errors. This will cause encoding to abort.
- * 
  * 2) You can use state to store your own data (e.g. buffer pointer).
- * 
  * 3) pb_write will update bytes_written after your callback runs.
- * 
- * 4) Substreams will modify max_size and bytes_written. Don't use them to
- * calculate any pointers.
+ * 4) Substreams will modify max_size and bytes_written. Don't use them
+ *    to calculate any pointers.
  */
 struct _pb_ostream_t
 {
+#ifdef PB_BUFFER_ONLY
+    /* Callback pointer is not used in buffer-only configuration.
+     * Having an int pointer here allows binary compatibility but
+     * gives an error if someone tries to assign callback function.
+     * Also, NULL pointer marks a 'sizing stream' that does not
+     * write anything.
+     */
+    int *callback;
+#else
     bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count);
-    void *state; /* Free field for use by callback implementation */
+#endif
+    void *state;          /* Free field for use by callback implementation. */
     size_t max_size;      /* Limit number of output bytes written (or use SIZE_MAX). */
-    size_t bytes_written;
+    size_t bytes_written; /* Number of bytes written so far. */
+    
+#ifndef PB_NO_ERRMSG
+    const char *errmsg;
+#endif
 };
 
-pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize);
-bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count);
+/***************************
+ * Main encoding functions *
+ ***************************/
 
-/* Encode struct to given output stream.
+/* Encode a single protocol buffers message from C structure into a stream.
  * Returns true on success, false on any failure.
  * The actual struct pointed to by src_struct must match the description in fields.
  * All required fields in the struct are assumed to have been filled in.
+ *
+ * Example usage:
+ *    MyMessage msg = {};
+ *    uint8_t buffer[64];
+ *    pb_ostream_t stream;
+ *
+ *    msg.field1 = 42;
+ *    stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+ *    pb_encode(&stream, MyMessage_fields, &msg);
  */
 bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct);
 
-/* --- Helper functions ---
- * You may want to use these from your caller or callbacks.
+/* Same as pb_encode, but prepends the length of the message as a varint.
+ * Corresponds to writeDelimitedTo() in Google's protobuf API.
  */
+bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct);
 
-/* Encode field header based on LTYPE and field number defined in the field structure.
- * Call this from the callback before writing out field contents. */
+/**************************************
+ * Functions for manipulating streams *
+ **************************************/
+
+/* Create an output stream for writing into a memory buffer.
+ * The number of bytes written can be found in stream.bytes_written after
+ * encoding the message.
+ *
+ * Alternatively, you can use a custom stream that writes directly to e.g.
+ * a file or a network socket.
+ */
+pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize);
+
+/* Pseudo-stream for measuring the size of a message without actually storing
+ * the encoded data.
+ * 
+ * Example usage:
+ *    MyMessage msg = {};
+ *    pb_ostream_t stream = PB_OSTREAM_SIZING;
+ *    pb_encode(&stream, MyMessage_fields, &msg);
+ *    printf("Message size is %d\n", stream.bytes_written);
+ */
+#ifndef PB_NO_ERRMSG
+#define PB_OSTREAM_SIZING {0,0,0,0,0}
+#else
+#define PB_OSTREAM_SIZING {0,0,0,0}
+#endif
+
+/* Function to write into a pb_ostream_t stream. You can use this if you need
+ * to append or prepend some custom headers to the message.
+ */
+bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count);
+
+
+/************************************************
+ * Helper functions for writing field callbacks *
+ ************************************************/
+
+/* Encode field header based on type and field number defined in the field
+ * structure. Call this from the callback before writing out field contents. */
 bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field);
 
-/* Encode field header by manually specifing wire type. You need to use this if
- * you want to write out packed arrays from a callback field. */
+/* Encode field header by manually specifing wire type. You need to use this
+ * if you want to write out packed arrays from a callback field. */
 bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number);
 
 /* Encode an integer in the varint format.
@@ -80,15 +137,16 @@ bool pb_encode_fixed32(pb_ostream_t *stream, const void *value);
 bool pb_encode_fixed64(pb_ostream_t *stream, const void *value);
 
 /* Encode a submessage field.
- * You need to pass the pb_field_t array and pointer to struct, just like with pb_encode().
- * This internally encodes the submessage twice, first to calculate message size and then to actually write it out.
+ * You need to pass the pb_field_t array and pointer to struct, just like
+ * with pb_encode(). This internally encodes the submessage twice, first to
+ * calculate message size and then to actually write it out.
  */
 bool pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct);
 
-/* --- Internal functions ---
- * These functions are not terribly useful for the average library user, but
- * are exported to make the unit testing and extending nanopb easier.
- */
+
+/*******************************
+ * Internal / legacy functions *
+ *******************************/
 
 #ifdef NANOPB_INTERNALS
 bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src);

tests/Makefile

@@ -1,94 +1,6 @@
-CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage
-LDFLAGS=--coverage
-DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h missing_fields.pb.h
-TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages
-
-# More strict checks for the core part of nanopb
-CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op -Wconversion
-
-all: breakpoints $(TESTS) run_unittests
+all:
+	scons
 
 clean:
-	rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno
+	scons -c
 
-%.pb.o: %.pb.c %.pb.h
-	$(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-
-%.o: %.c
-%.o: %.c $(DEPS)
-	$(CC) $(CFLAGS) -c -o $@ $<
-
-pb_encode.o: ../pb_encode.c $(DEPS)
-	$(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-pb_decode.o: ../pb_decode.c $(DEPS)
-	$(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-
-pb_encode.cxx.o: ../pb_encode.c $(DEPS)
-	$(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-pb_decode.cxx.o: ../pb_decode.c $(DEPS)
-	$(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
-
-test_cxxcompile: pb_encode.cxx.o pb_decode.cxx.o
-test_decode1: test_decode1.o pb_decode.o person.pb.o
-test_decode2: test_decode2.o pb_decode.o person.pb.o
-test_decode3: test_decode3.o pb_decode.o alltypes.pb.o
-test_encode1: test_encode1.o pb_encode.o person.pb.o
-test_encode2: test_encode2.o pb_encode.o person.pb.o
-test_encode3: test_encode3.o pb_encode.o alltypes.pb.o
-test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o
-test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o
-test_missing_fields: test_missing_fields.o pb_encode.o pb_decode.o missing_fields.pb.o
-decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o
-encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o
-test_no_messages: no_messages.pb.h no_messages.pb.c no_messages.pb.o
-
-%.pb: %.proto
-	protoc -I. -I../generator -I/usr/include -o$@ $<
-
-%.pb.c %.pb.h: %.pb ../generator/nanopb_generator.py
-	python ../generator/nanopb_generator.py $<
-
-breakpoints: ../*.c *.c
-	grep -n 'return false;' $^ | cut -d: -f-2 | xargs -n 1 echo b > $@
-
-coverage: run_unittests
-	gcov pb_encode.gcda
-	gcov pb_decode.gcda
-
-run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields test_options
-	rm -f *.gcda
-	
-	./decode_unittests > /dev/null
-	./encode_unittests > /dev/null
-	
-	[ "`./test_encode1 | ./test_decode1`" = \
-	"`./test_encode1 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ]
-
-	[ "`./test_encode2 | ./test_decode1`" = \
-	"`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ]
-
-	[ "`./test_encode2 | ./test_decode2`" = \
-	"`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ]
-	
-	[ "`./test_decode2 < person_with_extra_field.pb`" = \
-	"`./test_encode2 | ./test_decode2`" ]
-	
-	[ "`./test_encode_callbacks | ./test_decode_callbacks`" = \
-	"`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ]
-
-	./test_encode3 | ./test_decode3
-	./test_encode3 1 | ./test_decode3 1
-	./test_encode3 1 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null
-	
-	./test_missing_fields
-
-test_options: options.pb.h options.expected
-	cat options.expected | while read -r p; do \
-	    if ! grep -q "$$p" $<; then \
-	        echo Expected: "$$p"; \
-	        exit 1; \
-	    fi \
-	done
-
-run_fuzztest: test_decode2
-	bash -c 'I=1; while true; do cat /dev/urandom | ./test_decode2 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done'

tests/SConstruct

@@ -0,0 +1,109 @@
+Help('''
+Type 'scons' to build and run all the available test cases.
+It will automatically detect your platform and C compiler and
+build appropriately.
+
+You can modify the behavious using following options:
+CC          Name of C compiler
+CXX         Name of C++ compiler
+CCFLAGS     Flags to pass to the C compiler
+CXXFLAGS    Flags to pass to the C++ compiler
+
+For example, for a clang build, use:
+scons CC=clang CXX=clang++
+''')
+
+import os
+env = Environment(ENV = os.environ)
+
+# Allow overriding the compiler with scons CC=???
+if 'CC' in ARGUMENTS: env.Replace(CC = ARGUMENTS['CC'])
+if 'CXX' in ARGUMENTS: env.Replace(CXX = ARGUMENTS['CXX'])
+if 'CFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CFLAGS'])
+if 'CXXFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CXXFLAGS'])
+
+# Add the builders defined in site_init.py
+add_nanopb_builders(env)
+
+# Path to the files shared by tests, and to the nanopb core.
+env.Append(CPPPATH = ["#../", "#common"])
+
+# Path for finding nanopb.proto
+env.Append(PROTOCPATH = '#../generator')
+
+# Check the compilation environment, unless we are just cleaning up.
+if not env.GetOption('clean'):
+    conf = Configure(env)
+
+    # If the platform doesn't support C99, use our own header file instead.
+    stdbool = conf.CheckCHeader('stdbool.h')
+    stdint = conf.CheckCHeader('stdint.h')
+    stddef = conf.CheckCHeader('stddef.h')
+    string = conf.CheckCHeader('string.h')
+    if not stdbool or not stdint or not stddef or not string:
+        conf.env.Append(CPPDEFINES = {'PB_SYSTEM_HEADER': '\\"pb_syshdr.h\\"'})
+        conf.env.Append(CPPPATH = "#../compat")
+        
+        if stdbool: conf.env.Append(CPPDEFINES = {'HAVE_STDBOOL_H': 1})
+        if stdint: conf.env.Append(CPPDEFINES = {'HAVE_STDINT_H': 1})
+        if stddef: conf.env.Append(CPPDEFINES = {'HAVE_STDDEF_H': 1})
+        if string: conf.env.Append(CPPDEFINES = {'HAVE_STRING_H': 1})
+        
+    # Check if we can use pkg-config to find protobuf include path
+    status, output = conf.TryAction('pkg-config protobuf --variable=includedir > $TARGET')
+    if status:
+        conf.env.Append(PROTOCPATH = output.strip())
+    else:
+        conf.env.Append(PROTOCPATH = '/usr/include')
+    
+    # Check if libmudflap is available (only with GCC)
+    if 'gcc' in env['CC']:
+        if conf.CheckLib('mudflap'):
+            conf.env.Append(CCFLAGS = '-fmudflap')
+            conf.env.Append(LINKFLAGS = '-lmudflap -fmudflap')
+    
+    # End the config stuff
+    env = conf.Finish()
+
+# Initialize the CCFLAGS according to the compiler
+if 'gcc' in env['CC']:
+    # GNU Compiler Collection
+    
+    # Debug info, warnings as errors
+    env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror --coverage -fstack-protector-all')
+    env.Append(LINKFLAGS = '--coverage')
+    
+    # More strict checks on the nanopb core
+    env.Append(CORECFLAGS = '-Wextra -Wcast-qual -Wlogical-op -Wconversion')
+elif 'clang' in env['CC']:
+    # CLang
+    env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror')
+    env.Append(CORECFLAGS = ' -Wextra -Wcast-qual -Wconversion')
+elif 'cl' in env['CC']:
+    # Microsoft Visual C++
+    
+    # Debug info on, warning level 2 for tests, warnings as errors
+    env.Append(CFLAGS = '/Zi /W2 /WX')
+    env.Append(LINKFLAGS = '/DEBUG')
+    
+    # More strict checks on the nanopb core
+    env.Append(CORECFLAGS = '/W4')
+    
+    # PB_RETURN_ERROR triggers C4127 because of while(0)
+    env.Append(CFLAGS = '/wd4127')
+elif 'tcc' in env['CC']:
+    # Tiny C Compiler
+    env.Append(CFLAGS = '-Wall -Werror -g')
+
+env.SetDefault(CORECFLAGS = '')
+
+if 'clang++' in env['CXX']:
+    env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers')
+elif 'g++' in env['CXX']:
+    env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers')
+elif 'cl' in env['CXX']:
+    env.Append(CXXFLAGS = '/Zi /W2 /WX')
+    
+# Now include the SConscript files from all subdirectories
+SConscript(Glob('*/SConscript'), exports = 'env')
+

tests/alltypes/SConscript

@@ -0,0 +1,12 @@
+# Build and run a test that encodes and decodes a message that contains
+# all of the Protocol Buffers data types.
+
+Import("env")
+
+env.NanopbProto("alltypes")
+enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "#common/pb_encode.o"])
+dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "#common/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_alltypes.output"])
+

tests/alltypes/alltypes.options

@@ -0,0 +1,3 @@
+* max_size:16
+* max_count:5
+

tests/alltypes/alltypes.proto

@@ -1,11 +1,13 @@
-import "nanopb.proto";
-
 message SubMessage {
-    required string substuff1 = 1 [(nanopb).max_size = 16, default = "1"];
+    required string substuff1 = 1 [default = "1"];
     required int32 substuff2 = 2 [default = 2];
     optional fixed32 substuff3 = 3 [default = 3];
 }
 
+message EmptyMessage {
+
+}
+
 enum MyEnum {
     Zero = 0;
     First = 1;
@@ -30,32 +32,34 @@ message AllTypes {
     required sfixed64   req_sfixed64= 12;
     required double     req_double  = 13;
     
-    required string     req_string  = 14 [(nanopb).max_size = 16];
-    required bytes      req_bytes   = 15 [(nanopb).max_size = 16];
+    required string     req_string  = 14;
+    required bytes      req_bytes   = 15;
     required SubMessage req_submsg  = 16;
     required MyEnum     req_enum    = 17;
+    required EmptyMessage req_emptymsg = 18;
     
     
-    repeated int32      rep_int32   = 21 [(nanopb).max_count = 5];
-    repeated int64      rep_int64   = 22 [(nanopb).max_count = 5];
-    repeated uint32     rep_uint32  = 23 [(nanopb).max_count = 5];
-    repeated uint64     rep_uint64  = 24 [(nanopb).max_count = 5];
-    repeated sint32     rep_sint32  = 25 [(nanopb).max_count = 5];
-    repeated sint64     rep_sint64  = 26 [(nanopb).max_count = 5];
-    repeated bool       rep_bool    = 27 [(nanopb).max_count = 5];
+    repeated int32      rep_int32   = 21;
+    repeated int64      rep_int64   = 22;
+    repeated uint32     rep_uint32  = 23;
+    repeated uint64     rep_uint64  = 24;
+    repeated sint32     rep_sint32  = 25;
+    repeated sint64     rep_sint64  = 26;
+    repeated bool       rep_bool    = 27;
     
-    repeated fixed32    rep_fixed32 = 28 [(nanopb).max_count = 5];
-    repeated sfixed32   rep_sfixed32= 29 [(nanopb).max_count = 5];
-    repeated float      rep_float   = 30 [(nanopb).max_count = 5];
+    repeated fixed32    rep_fixed32 = 28;
+    repeated sfixed32   rep_sfixed32= 29;
+    repeated float      rep_float   = 30;
     
-    repeated fixed64    rep_fixed64 = 31 [(nanopb).max_count = 5];
-    repeated sfixed64   rep_sfixed64= 32 [(nanopb).max_count = 5];
-    repeated double     rep_double  = 33 [(nanopb).max_count = 5];
+    repeated fixed64    rep_fixed64 = 31;
+    repeated sfixed64   rep_sfixed64= 32;
+    repeated double     rep_double  = 33;
     
-    repeated string     rep_string  = 34 [(nanopb).max_size = 16, (nanopb).max_count = 5];
-    repeated bytes      rep_bytes   = 35 [(nanopb).max_size = 16, (nanopb).max_count = 5];
-    repeated SubMessage rep_submsg  = 36 [(nanopb).max_count = 5];
-    repeated MyEnum     rep_enum    = 37 [(nanopb).max_count = 5];
+    repeated string     rep_string  = 34;
+    repeated bytes      rep_bytes   = 35;
+    repeated SubMessage rep_submsg  = 36;
+    repeated MyEnum     rep_enum    = 37;
+    repeated EmptyMessage rep_emptymsg = 38;
     
     optional int32      opt_int32   = 41 [default = 4041];
     optional int64      opt_int64   = 42 [default = 4042];
@@ -73,13 +77,17 @@ message AllTypes {
     optional sfixed64   opt_sfixed64= 52 [default = 4052];
     optional double     opt_double  = 53 [default = 4053];
     
-    optional string     opt_string  = 54 [(nanopb).max_size = 16, default = "4054"];
-    optional bytes      opt_bytes   = 55 [(nanopb).max_size = 16, default = "4055"];
+    optional string     opt_string  = 54 [default = "4054"];
+    optional bytes      opt_bytes   = 55 [default = "4055"];
     optional SubMessage opt_submsg  = 56;
     optional MyEnum     opt_enum    = 57 [default = Second];
+    optional EmptyMessage opt_emptymsg = 58;
 
     // Just to make sure that the size of the fields has been calculated
     // properly, i.e. otherwise a bug in last field might not be detected.
     required int32      end = 99;
+
+
+    extensions 200 to 255;
 }
 

tests/alltypes/decode_alltypes.c

@@ -0,0 +1,202 @@
+/* Tests the decoding of all types.
+ * This is the counterpart of test_encode3.
+ * Run e.g. ./test_encode3 | ./test_decode3
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <pb_decode.h>
+#include "alltypes.pb.h"
+#include "test_helpers.h"
+
+#define TEST(x) if (!(x)) { \
+    printf("Test " #x " failed.\n"); \
+    return false; \
+    }
+
+/* This function is called once from main(), it handles
+   the decoding and checks the fields. */
+bool check_alltypes(pb_istream_t *stream, int mode)
+{
+    AllTypes alltypes;
+    
+    /* Fill with garbage to better detect initialization errors */
+    memset(&alltypes, 0xAA, sizeof(alltypes));
+    
+    if (!pb_decode(stream, AllTypes_fields, &alltypes))
+        return false;
+    
+    TEST(alltypes.req_int32         == -1001);
+    TEST(alltypes.req_int64         == -1002);
+    TEST(alltypes.req_uint32        == 1003);
+    TEST(alltypes.req_uint64        == 1004);
+    TEST(alltypes.req_sint32        == -1005);
+    TEST(alltypes.req_sint64        == -1006);
+    TEST(alltypes.req_bool          == true);
+    
+    TEST(alltypes.req_fixed32       == 1008);
+    TEST(alltypes.req_sfixed32      == -1009);
+    TEST(alltypes.req_float         == 1010.0f);
+    
+    TEST(alltypes.req_fixed64       == 1011);
+    TEST(alltypes.req_sfixed64      == -1012);
+    TEST(alltypes.req_double        == 1013.0f);
+    
+    TEST(strcmp(alltypes.req_string, "1014") == 0);
+    TEST(alltypes.req_bytes.size == 4);
+    TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0);
+    TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0);
+    TEST(alltypes.req_submsg.substuff2 == 1016);
+    TEST(alltypes.req_submsg.substuff3 == 3);
+    TEST(alltypes.req_enum == MyEnum_Truth);
+    
+    TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0);
+    TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0);
+    TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0);
+    TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0);
+    TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0);
+    TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0);
+    TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false);
+    
+    TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0);
+    TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0);
+    TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f);
+    
+    TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0);
+    TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0);
+    TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0);
+    
+    TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0');
+    TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0);
+    TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0);
+
+    TEST(alltypes.rep_submsg_count == 5);
+    TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0');
+    TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0);
+    TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3);
+    
+    TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero);
+    TEST(alltypes.rep_emptymsg_count == 5);
+    
+    if (mode == 0)
+    {
+        /* Expect default values */
+        TEST(alltypes.has_opt_int32     == false);
+        TEST(alltypes.opt_int32         == 4041);
+        TEST(alltypes.has_opt_int64     == false);
+        TEST(alltypes.opt_int64         == 4042);
+        TEST(alltypes.has_opt_uint32    == false);
+        TEST(alltypes.opt_uint32        == 4043);
+        TEST(alltypes.has_opt_uint64    == false);
+        TEST(alltypes.opt_uint64        == 4044);
+        TEST(alltypes.has_opt_sint32    == false);
+        TEST(alltypes.opt_sint32        == 4045);
+        TEST(alltypes.has_opt_sint64    == false);
+        TEST(alltypes.opt_sint64        == 4046);
+        TEST(alltypes.has_opt_bool      == false);
+        TEST(alltypes.opt_bool          == false);
+        
+        TEST(alltypes.has_opt_fixed32   == false);
+        TEST(alltypes.opt_fixed32       == 4048);
+        TEST(alltypes.has_opt_sfixed32  == false);
+        TEST(alltypes.opt_sfixed32      == 4049);
+        TEST(alltypes.has_opt_float     == false);
+        TEST(alltypes.opt_float         == 4050.0f);
+        
+        TEST(alltypes.has_opt_fixed64   == false);
+        TEST(alltypes.opt_fixed64       == 4051);
+        TEST(alltypes.has_opt_sfixed64  == false);
+        TEST(alltypes.opt_sfixed64      == 4052);
+        TEST(alltypes.has_opt_double    == false);
+        TEST(alltypes.opt_double        == 4053.0);
+        
+        TEST(alltypes.has_opt_string    == false);
+        TEST(strcmp(alltypes.opt_string, "4054") == 0);
+        TEST(alltypes.has_opt_bytes     == false);
+        TEST(alltypes.opt_bytes.size == 4);
+        TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0);
+        TEST(alltypes.has_opt_submsg    == false);
+        TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0);
+        TEST(alltypes.opt_submsg.substuff2 == 2);
+        TEST(alltypes.opt_submsg.substuff3 == 3);
+        TEST(alltypes.has_opt_enum     == false);
+        TEST(alltypes.opt_enum == MyEnum_Second);
+        TEST(alltypes.has_opt_emptymsg == false);
+    }
+    else
+    {
+        /* Expect filled-in values */
+        TEST(alltypes.has_opt_int32     == true);
+        TEST(alltypes.opt_int32         == 3041);
+        TEST(alltypes.has_opt_int64     == true);
+        TEST(alltypes.opt_int64         == 3042);
+        TEST(alltypes.has_opt_uint32    == true);
+        TEST(alltypes.opt_uint32        == 3043);
+        TEST(alltypes.has_opt_uint64    == true);
+        TEST(alltypes.opt_uint64        == 3044);
+        TEST(alltypes.has_opt_sint32    == true);
+        TEST(alltypes.opt_sint32        == 3045);
+        TEST(alltypes.has_opt_sint64    == true);
+        TEST(alltypes.opt_sint64        == 3046);
+        TEST(alltypes.has_opt_bool      == true);
+        TEST(alltypes.opt_bool          == true);
+        
+        TEST(alltypes.has_opt_fixed32   == true);
+        TEST(alltypes.opt_fixed32       == 3048);
+        TEST(alltypes.has_opt_sfixed32  == true);
+        TEST(alltypes.opt_sfixed32      == 3049);
+        TEST(alltypes.has_opt_float     == true);
+        TEST(alltypes.opt_float         == 3050.0f);
+        
+        TEST(alltypes.has_opt_fixed64   == true);
+        TEST(alltypes.opt_fixed64       == 3051);
+        TEST(alltypes.has_opt_sfixed64  == true);
+        TEST(alltypes.opt_sfixed64      == 3052);
+        TEST(alltypes.has_opt_double    == true);
+        TEST(alltypes.opt_double        == 3053.0);
+        
+        TEST(alltypes.has_opt_string    == true);
+        TEST(strcmp(alltypes.opt_string, "3054") == 0);
+        TEST(alltypes.has_opt_bytes     == true);
+        TEST(alltypes.opt_bytes.size == 4);
+        TEST(memcmp(alltypes.opt_bytes.bytes, "3055", 4) == 0);
+        TEST(alltypes.has_opt_submsg    == true);
+        TEST(strcmp(alltypes.opt_submsg.substuff1, "3056") == 0);
+        TEST(alltypes.opt_submsg.substuff2 == 3056);
+        TEST(alltypes.opt_submsg.substuff3 == 3);
+        TEST(alltypes.has_opt_enum      == true);
+        TEST(alltypes.opt_enum == MyEnum_Truth);
+        TEST(alltypes.has_opt_emptymsg  == true);
+    }
+    
+    TEST(alltypes.end == 1099);
+    
+    return true;
+}
+
+int main(int argc, char **argv)
+{
+    uint8_t buffer[1024];
+    size_t count;
+    pb_istream_t stream;
+
+    /* Whether to expect the optional values or the default values. */
+    int mode = (argc > 1) ? atoi(argv[1]) : 0;
+    
+    /* Read the data into buffer */
+    SET_BINARY_MODE(stdin);
+    count = fread(buffer, 1, sizeof(buffer), stdin);
+    
+    /* Construct a pb_istream_t for reading from the buffer */
+    stream = pb_istream_from_buffer(buffer, count);
+    
+    /* Decode and print out the stuff */
+    if (!check_alltypes(&stream, mode))
+    {
+        printf("Parsing failed: %s\n", PB_GET_ERROR(&stream));
+        return 1;
+    } else {
+        return 0;
+    }
+}

tests/alltypes/encode_alltypes.c

@@ -6,6 +6,7 @@
 #include <string.h>
 #include <pb_encode.h>
 #include "alltypes.pb.h"
+#include "test_helpers.h"
 
 int main(int argc, char **argv)
 {
@@ -64,6 +65,7 @@ int main(int argc, char **argv)
     alltypes.rep_submsg[4].substuff3 = 2016;
     
     alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth;
+    alltypes.rep_emptymsg_count = 5;
     
     if (mode != 0)
     {
@@ -107,22 +109,26 @@ int main(int argc, char **argv)
         alltypes.opt_submsg.substuff2 = 3056;
         alltypes.has_opt_enum = true;
         alltypes.opt_enum = MyEnum_Truth;
+        alltypes.has_opt_emptymsg = true;
     }
     
     alltypes.end = 1099;
     
+    {
         uint8_t buffer[1024];
         pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
         
         /* Now encode it and check if we succeeded. */
         if (pb_encode(&stream, AllTypes_fields, &alltypes))
         {
+            SET_BINARY_MODE(stdout);
             fwrite(buffer, 1, stream.bytes_written, stdout);
             return 0; /* Success */
         }
         else
         {
-        fprintf(stderr, "Encoding failed!\n");
+            fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
             return 1; /* Failure */
         }
     }
+}

tests/backwards_compatibility/SConscript

@@ -0,0 +1,11 @@
+# Check that the old generated .pb.c/.pb.h files are still compatible with the
+# current version of nanopb.
+
+Import("env")
+
+enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "#common/pb_encode.o"])
+dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "#common/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_legacy.output"])
+

tests/backwards_compatibility/alltypes_legacy.c

@@ -0,0 +1,93 @@
+/* Automatically generated nanopb constant definitions */
+/* Generated by 0.2.0-dev at Sun Feb 17 00:09:53 2013. */
+/* This is a file generated using nanopb-0.2.0-dev.
+ * It is used as a part of test suite in order to detect any
+ * incompatible changes made to the generator in future versions.
+ */
+
+#include "alltypes_legacy.h"
+
+const char SubMessage_substuff1_default[16] = "1";
+const int32_t SubMessage_substuff2_default = 2;
+const uint32_t SubMessage_substuff3_default = 3;
+const int32_t AllTypes_opt_int32_default = 4041;
+const int64_t AllTypes_opt_int64_default = 4042;
+const uint32_t AllTypes_opt_uint32_default = 4043;
+const uint64_t AllTypes_opt_uint64_default = 4044;
+const int32_t AllTypes_opt_sint32_default = 4045;
+const int64_t AllTypes_opt_sint64_default = 4046;
+const bool AllTypes_opt_bool_default = false;
+const uint32_t AllTypes_opt_fixed32_default = 4048;
+const int32_t AllTypes_opt_sfixed32_default = 4049;
+const float AllTypes_opt_float_default = 4050;
+const uint64_t AllTypes_opt_fixed64_default = 4051;
+const int64_t AllTypes_opt_sfixed64_default = 4052;
+const double AllTypes_opt_double_default = 4053;
+const char AllTypes_opt_string_default[16] = "4054";
+const AllTypes_opt_bytes_t AllTypes_opt_bytes_default = {4, {0x34,0x30,0x35,0x35}};
+const MyEnum AllTypes_opt_enum_default = MyEnum_Second;
+
+
+const pb_field_t SubMessage_fields[4] = {
+    PB_FIELD(  1, STRING  , REQUIRED, STATIC, SubMessage, substuff1, substuff1, &SubMessage_substuff1_default),
+    PB_FIELD(  2, INT32   , REQUIRED, STATIC, SubMessage, substuff2, substuff1, &SubMessage_substuff2_default),
+    PB_FIELD(  3, FIXED32 , OPTIONAL, STATIC, SubMessage, substuff3, substuff2, &SubMessage_substuff3_default),
+    PB_LAST_FIELD
+};
+
+const pb_field_t AllTypes_fields[53] = {
+    PB_FIELD(  1, INT32   , REQUIRED, STATIC, AllTypes, req_int32, req_int32, 0),
+    PB_FIELD(  2, INT64   , REQUIRED, STATIC, AllTypes, req_int64, req_int32, 0),
+    PB_FIELD(  3, UINT32  , REQUIRED, STATIC, AllTypes, req_uint32, req_int64, 0),
+    PB_FIELD(  4, UINT64  , REQUIRED, STATIC, AllTypes, req_uint64, req_uint32, 0),
+    PB_FIELD(  5, SINT32  , REQUIRED, STATIC, AllTypes, req_sint32, req_uint64, 0),
+    PB_FIELD(  6, SINT64  , REQUIRED, STATIC, AllTypes, req_sint64, req_sint32, 0),
+    PB_FIELD(  7, BOOL    , REQUIRED, STATIC, AllTypes, req_bool, req_sint64, 0),
+    PB_FIELD(  8, FIXED32 , REQUIRED, STATIC, AllTypes, req_fixed32, req_bool, 0),
+    PB_FIELD(  9, SFIXED32, REQUIRED, STATIC, AllTypes, req_sfixed32, req_fixed32, 0),
+    PB_FIELD( 10, FLOAT   , REQUIRED, STATIC, AllTypes, req_float, req_sfixed32, 0),
+    PB_FIELD( 11, FIXED64 , REQUIRED, STATIC, AllTypes, req_fixed64, req_float, 0),
+    PB_FIELD( 12, SFIXED64, REQUIRED, STATIC, AllTypes, req_sfixed64, req_fixed64, 0),
+    PB_FIELD( 13, DOUBLE  , REQUIRED, STATIC, AllTypes, req_double, req_sfixed64, 0),
+    PB_FIELD( 14, STRING  , REQUIRED, STATIC, AllTypes, req_string, req_double, 0),
+    PB_FIELD( 15, BYTES   , REQUIRED, STATIC, AllTypes, req_bytes, req_string, 0),
+    PB_FIELD( 16, MESSAGE , REQUIRED, STATIC, AllTypes, req_submsg, req_bytes, &SubMessage_fields),
+    PB_FIELD( 17, ENUM    , REQUIRED, STATIC, AllTypes, req_enum, req_submsg, 0),
+    PB_FIELD( 21, INT32   , REPEATED, STATIC, AllTypes, rep_int32, req_enum, 0),
+    PB_FIELD( 22, INT64   , REPEATED, STATIC, AllTypes, rep_int64, rep_int32, 0),
+    PB_FIELD( 23, UINT32  , REPEATED, STATIC, AllTypes, rep_uint32, rep_int64, 0),
+    PB_FIELD( 24, UINT64  , REPEATED, STATIC, AllTypes, rep_uint64, rep_uint32, 0),
+    PB_FIELD( 25, SINT32  , REPEATED, STATIC, AllTypes, rep_sint32, rep_uint64, 0),
+    PB_FIELD( 26, SINT64  , REPEATED, STATIC, AllTypes, rep_sint64, rep_sint32, 0),
+    PB_FIELD( 27, BOOL    , REPEATED, STATIC, AllTypes, rep_bool, rep_sint64, 0),
+    PB_FIELD( 28, FIXED32 , REPEATED, STATIC, AllTypes, rep_fixed32, rep_bool, 0),
+    PB_FIELD( 29, SFIXED32, REPEATED, STATIC, AllTypes, rep_sfixed32, rep_fixed32, 0),
+    PB_FIELD( 30, FLOAT   , REPEATED, STATIC, AllTypes, rep_float, rep_sfixed32, 0),
+    PB_FIELD( 31, FIXED64 , REPEATED, STATIC, AllTypes, rep_fixed64, rep_float, 0),
+    PB_FIELD( 32, SFIXED64, REPEATED, STATIC, AllTypes, rep_sfixed64, rep_fixed64, 0),
+    PB_FIELD( 33, DOUBLE  , REPEATED, STATIC, AllTypes, rep_double, rep_sfixed64, 0),
+    PB_FIELD( 34, STRING  , REPEATED, STATIC, AllTypes, rep_string, rep_double, 0),
+    PB_FIELD( 35, BYTES   , REPEATED, STATIC, AllTypes, rep_bytes, rep_string, 0),
+    PB_FIELD( 36, MESSAGE , REPEATED, STATIC, AllTypes, rep_submsg, rep_bytes, &SubMessage_fields),
+    PB_FIELD( 37, ENUM    , REPEATED, STATIC, AllTypes, rep_enum, rep_submsg, 0),
+    PB_FIELD( 41, INT32   , OPTIONAL, STATIC, AllTypes, opt_int32, rep_enum, &AllTypes_opt_int32_default),
+    PB_FIELD( 42, INT64   , OPTIONAL, STATIC, AllTypes, opt_int64, opt_int32, &AllTypes_opt_int64_default),
+    PB_FIELD( 43, UINT32  , OPTIONAL, STATIC, AllTypes, opt_uint32, opt_int64, &AllTypes_opt_uint32_default),
+    PB_FIELD( 44, UINT64  , OPTIONAL, STATIC, AllTypes, opt_uint64, opt_uint32, &AllTypes_opt_uint64_default),
+    PB_FIELD( 45, SINT32  , OPTIONAL, STATIC, AllTypes, opt_sint32, opt_uint64, &AllTypes_opt_sint32_default),
+    PB_FIELD( 46, SINT64  , OPTIONAL, STATIC, AllTypes, opt_sint64, opt_sint32, &AllTypes_opt_sint64_default),
+    PB_FIELD( 47, BOOL    , OPTIONAL, STATIC, AllTypes, opt_bool, opt_sint64, &AllTypes_opt_bool_default),
+    PB_FIELD( 48, FIXED32 , OPTIONAL, STATIC, AllTypes, opt_fixed32, opt_bool, &AllTypes_opt_fixed32_default),
+    PB_FIELD( 49, SFIXED32, OPTIONAL, STATIC, AllTypes, opt_sfixed32, opt_fixed32, &AllTypes_opt_sfixed32_default),
+    PB_FIELD( 50, FLOAT   , OPTIONAL, STATIC, AllTypes, opt_float, opt_sfixed32, &AllTypes_opt_float_default),
+    PB_FIELD( 51, FIXED64 , OPTIONAL, STATIC, AllTypes, opt_fixed64, opt_float, &AllTypes_opt_fixed64_default),
+    PB_FIELD( 52, SFIXED64, OPTIONAL, STATIC, AllTypes, opt_sfixed64, opt_fixed64, &AllTypes_opt_sfixed64_default),
+    PB_FIELD( 53, DOUBLE  , OPTIONAL, STATIC, AllTypes, opt_double, opt_sfixed64, &AllTypes_opt_double_default),
+    PB_FIELD( 54, STRING  , OPTIONAL, STATIC, AllTypes, opt_string, opt_double, &AllTypes_opt_string_default),
+    PB_FIELD( 55, BYTES   , OPTIONAL, STATIC, AllTypes, opt_bytes, opt_string, &AllTypes_opt_bytes_default),
+    PB_FIELD( 56, MESSAGE , OPTIONAL, STATIC, AllTypes, opt_submsg, opt_bytes, &SubMessage_fields),
+    PB_FIELD( 57, ENUM    , OPTIONAL, STATIC, AllTypes, opt_enum, opt_submsg, &AllTypes_opt_enum_default),
+    PB_FIELD( 99, INT32   , REQUIRED, STATIC, AllTypes, end, opt_enum, 0),
+    PB_LAST_FIELD
+};
+

tests/backwards_compatibility/alltypes_legacy.h

@@ -0,0 +1,178 @@
+/* Automatically generated nanopb header */
+/* This is a file generated using nanopb-0.2.0-dev.
+ * It is used as a part of test suite in order to detect any
+ * incompatible changes made to the generator in future versions.
+ */
+
+#ifndef _PB_ALLTYPES_PB_H_
+#define _PB_ALLTYPES_PB_H_
+#include <pb.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Enum definitions */
+typedef enum _MyEnum {
+    MyEnum_Zero = 0,
+    MyEnum_First = 1,
+    MyEnum_Second = 2,
+    MyEnum_Truth = 42
+} MyEnum;
+
+/* Struct definitions */
+typedef struct _SubMessage {
+    char substuff1[16];
+    int32_t substuff2;
+    bool has_substuff3;
+    uint32_t substuff3;
+} SubMessage;
+
+typedef struct {
+    size_t size;
+    uint8_t bytes[16];
+} AllTypes_req_bytes_t;
+
+typedef struct {
+    size_t size;
+    uint8_t bytes[16];
+} AllTypes_rep_bytes_t;
+
+typedef struct {
+    size_t size;
+    uint8_t bytes[16];
+} AllTypes_opt_bytes_t;
+
+typedef struct _AllTypes {
+    int32_t req_int32;
+    int64_t req_int64;
+    uint32_t req_uint32;
+    uint64_t req_uint64;
+    int32_t req_sint32;
+    int64_t req_sint64;
+    bool req_bool;
+    uint32_t req_fixed32;
+    int32_t req_sfixed32;
+    float req_float;
+    uint64_t req_fixed64;
+    int64_t req_sfixed64;
+    double req_double;
+    char req_string[16];
+    AllTypes_req_bytes_t req_bytes;
+    SubMessage req_submsg;
+    MyEnum req_enum;
+    size_t rep_int32_count;
+    int32_t rep_int32[5];
+    size_t rep_int64_count;
+    int64_t rep_int64[5];
+    size_t rep_uint32_count;
+    uint32_t rep_uint32[5];
+    size_t rep_uint64_count;
+    uint64_t rep_uint64[5];
+    size_t rep_sint32_count;
+    int32_t rep_sint32[5];
+    size_t rep_sint64_count;
+    int64_t rep_sint64[5];
+    size_t rep_bool_count;
+    bool rep_bool[5];
+    size_t rep_fixed32_count;
+    uint32_t rep_fixed32[5];
+    size_t rep_sfixed32_count;
+    int32_t rep_sfixed32[5];
+    size_t rep_float_count;
+    float rep_float[5];
+    size_t rep_fixed64_count;
+    uint64_t rep_fixed64[5];
+    size_t rep_sfixed64_count;
+    int64_t rep_sfixed64[5];
+    size_t rep_double_count;
+    double rep_double[5];
+    size_t rep_string_count;
+    char rep_string[5][16];
+    size_t rep_bytes_count;
+    AllTypes_rep_bytes_t rep_bytes[5];
+    size_t rep_submsg_count;
+    SubMessage rep_submsg[5];
+    size_t rep_enum_count;
+    MyEnum rep_enum[5];
+    bool has_opt_int32;
+    int32_t opt_int32;
+    bool has_opt_int64;
+    int64_t opt_int64;
+    bool has_opt_uint32;
+    uint32_t opt_uint32;
+    bool has_opt_uint64;
+    uint64_t opt_uint64;
+    bool has_opt_sint32;
+    int32_t opt_sint32;
+    bool has_opt_sint64;
+    int64_t opt_sint64;
+    bool has_opt_bool;
+    bool opt_bool;
+    bool has_opt_fixed32;
+    uint32_t opt_fixed32;
+    bool has_opt_sfixed32;
+    int32_t opt_sfixed32;
+    bool has_opt_float;
+    float opt_float;
+    bool has_opt_fixed64;
+    uint64_t opt_fixed64;
+    bool has_opt_sfixed64;
+    int64_t opt_sfixed64;
+    bool has_opt_double;
+    double opt_double;
+    bool has_opt_string;
+    char opt_string[16];
+    bool has_opt_bytes;
+    AllTypes_opt_bytes_t opt_bytes;
+    bool has_opt_submsg;
+    SubMessage opt_submsg;
+    bool has_opt_enum;
+    MyEnum opt_enum;
+    int32_t end;
+} AllTypes;
+
+/* Default values for struct fields */
+extern const char SubMessage_substuff1_default[16];
+extern const int32_t SubMessage_substuff2_default;
+extern const uint32_t SubMessage_substuff3_default;
+extern const int32_t AllTypes_opt_int32_default;
+extern const int64_t AllTypes_opt_int64_default;
+extern const uint32_t AllTypes_opt_uint32_default;
+extern const uint64_t AllTypes_opt_uint64_default;
+extern const int32_t AllTypes_opt_sint32_default;
+extern const int64_t AllTypes_opt_sint64_default;
+extern const bool AllTypes_opt_bool_default;
+extern const uint32_t AllTypes_opt_fixed32_default;
+extern const int32_t AllTypes_opt_sfixed32_default;
+extern const float AllTypes_opt_float_default;
+extern const uint64_t AllTypes_opt_fixed64_default;
+extern const int64_t AllTypes_opt_sfixed64_default;
+extern const double AllTypes_opt_double_default;
+extern const char AllTypes_opt_string_default[16];
+extern const AllTypes_opt_bytes_t AllTypes_opt_bytes_default;
+extern const MyEnum AllTypes_opt_enum_default;
+
+/* Struct field encoding specification for nanopb */
+extern const pb_field_t SubMessage_fields[4];
+extern const pb_field_t AllTypes_fields[53];
+
+/* Check that field information fits in pb_field_t */
+#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
+STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 256 && pb_membersize(AllTypes, rep_submsg[0]) < 256 && pb_membersize(AllTypes, opt_submsg) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_SubMessage_AllTypes)
+#endif
+
+#if !defined(PB_FIELD_32BIT)
+STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 65536 && pb_membersize(AllTypes, rep_submsg[0]) < 65536 && pb_membersize(AllTypes, opt_submsg) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_SubMessage_AllTypes)
+#endif
+
+/* On some platforms (such as AVR), double is really float.
+ * These are not directly supported by nanopb, but see example_avr_double.
+ */
+STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif

tests/backwards_compatibility/decode_legacy.c

@@ -1,13 +1,16 @@
 /* Tests the decoding of all types.
- * This is the counterpart of test_encode3.
- * Run e.g. ./test_encode3 | ./test_decode3
+ * This is a backwards-compatibility test, using alltypes_legacy.h.
+ * It is similar to decode_alltypes, but duplicated in order to allow
+ * decode_alltypes to test any new features introduced later.
+ *
+ * Run e.g. ./encode_legacy | ./decode_legacy
  */
 
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <pb_decode.h>
-#include "alltypes.pb.h"
+#include "alltypes_legacy.h"
 
 #define TEST(x) if (!(x)) { \
     printf("Test " #x " failed.\n"); \

tests/backwards_compatibility/encode_legacy.c

@@ -0,0 +1,135 @@
+/* Attempts to test all the datatypes supported by ProtoBuf.
+ * This is a backwards-compatibility test, using alltypes_legacy.h.
+ * It is similar to encode_alltypes, but duplicated in order to allow
+ * encode_alltypes to test any new features introduced later.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pb_encode.h>
+#include "alltypes_legacy.h"
+#include "test_helpers.h"
+
+int main(int argc, char **argv)
+{
+    int mode = (argc > 1) ? atoi(argv[1]) : 0;
+    
+    /* Initialize the structure with constants */
+    AllTypes alltypes = {0};
+    
+    alltypes.req_int32         = -1001;
+    alltypes.req_int64         = -1002;
+    alltypes.req_uint32        = 1003;
+    alltypes.req_uint64        = 1004;
+    alltypes.req_sint32        = -1005;
+    alltypes.req_sint64        = -1006;
+    alltypes.req_bool          = true;
+    
+    alltypes.req_fixed32       = 1008;
+    alltypes.req_sfixed32      = -1009;
+    alltypes.req_float         = 1010.0f;
+    
+    alltypes.req_fixed64       = 1011;
+    alltypes.req_sfixed64      = -1012;
+    alltypes.req_double        = 1013.0;
+    
+    strcpy(alltypes.req_string, "1014");
+    alltypes.req_bytes.size = 4;
+    memcpy(alltypes.req_bytes.bytes, "1015", 4);
+    strcpy(alltypes.req_submsg.substuff1, "1016");
+    alltypes.req_submsg.substuff2 = 1016;
+    alltypes.req_enum = MyEnum_Truth;
+    
+    alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001;
+    alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002;
+    alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003;
+    alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004;
+    alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005;
+    alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006;
+    alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true;
+    
+    alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008;
+    alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009;
+    alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f;
+    
+    alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011;
+    alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012;
+    alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0;
+    
+    alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014");
+    alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4;
+    memcpy(alltypes.rep_bytes[4].bytes, "2015", 4);
+
+    alltypes.rep_submsg_count = 5;
+    strcpy(alltypes.rep_submsg[4].substuff1, "2016");
+    alltypes.rep_submsg[4].substuff2 = 2016;
+    alltypes.rep_submsg[4].has_substuff3 = true;
+    alltypes.rep_submsg[4].substuff3 = 2016;
+    
+    alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth;
+    
+    if (mode != 0)
+    {
+        /* Fill in values for optional fields */
+        alltypes.has_opt_int32 = true;
+        alltypes.opt_int32         = 3041;
+        alltypes.has_opt_int64 = true;
+        alltypes.opt_int64         = 3042;
+        alltypes.has_opt_uint32 = true;
+        alltypes.opt_uint32        = 3043;
+        alltypes.has_opt_uint64 = true;
+        alltypes.opt_uint64        = 3044;
+        alltypes.has_opt_sint32 = true;
+        alltypes.opt_sint32        = 3045;
+        alltypes.has_opt_sint64 = true;
+        alltypes.opt_sint64        = 3046;
+        alltypes.has_opt_bool = true;
+        alltypes.opt_bool          = true;
+        
+        alltypes.has_opt_fixed32 = true;
+        alltypes.opt_fixed32       = 3048;
+        alltypes.has_opt_sfixed32 = true;
+        alltypes.opt_sfixed32      = 3049;
+        alltypes.has_opt_float = true;
+        alltypes.opt_float         = 3050.0f;
+        
+        alltypes.has_opt_fixed64 = true;
+        alltypes.opt_fixed64       = 3051;
+        alltypes.has_opt_sfixed64 = true;
+        alltypes.opt_sfixed64      = 3052;
+        alltypes.has_opt_double = true;
+        alltypes.opt_double        = 3053.0;
+        
+        alltypes.has_opt_string = true;
+        strcpy(alltypes.opt_string, "3054");
+        alltypes.has_opt_bytes = true;
+        alltypes.opt_bytes.size = 4;
+        memcpy(alltypes.opt_bytes.bytes, "3055", 4);
+        alltypes.has_opt_submsg = true;
+        strcpy(alltypes.opt_submsg.substuff1, "3056");
+        alltypes.opt_submsg.substuff2 = 3056;
+        alltypes.has_opt_enum = true;
+        alltypes.opt_enum = MyEnum_Truth;
+    }
+    
+    alltypes.end = 1099;
+
+    {    
+        uint8_t buffer[1024];
+        pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        
+        /* Now encode it and check if we succeeded. */
+        if (pb_encode(&stream, AllTypes_fields, &alltypes))
+        {
+            SET_BINARY_MODE(stdout);
+            fwrite(buffer, 1, stream.bytes_written, stdout);
+            return 0; /* Success */
+        }
+        else
+        {
+            fprintf(stderr, "Encoding failed!\n");
+            return 1; /* Failure */
+        }
+    }
+}

tests/basic_buffer/SConscript

@@ -0,0 +1,12 @@
+# Build and run a basic round-trip test using memory buffer encoding.
+
+Import("env")
+
+enc = env.Program(["encode_buffer.c", "#common/person.pb.c", "#common/pb_encode.o"])
+dec = env.Program(["decode_buffer.c", "#common/person.pb.c", "#common/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_buffer.output"])
+env.Decode(["encode_buffer.output", "#common/person.proto"], MESSAGE = "Person")
+env.Compare(["decode_buffer.output", "encode_buffer.decoded"])
+

tests/basic_buffer/decode_buffer.c

@@ -9,6 +9,7 @@
 #include <stdio.h>
 #include <pb_decode.h>
 #include "person.pb.h"
+#include "test_helpers.h"
 
 /* This function is called once from main(), it handles
    the decoding and printing. */
@@ -59,9 +60,13 @@ bool print_person(pb_istream_t *stream)
 
 int main()
 {
-    /* Read the data into buffer */
     uint8_t buffer[512];
-    size_t count = fread(buffer, 1, sizeof(buffer), stdin);
+    pb_istream_t stream;
+    size_t count;
+    
+    /* Read the data into buffer */
+    SET_BINARY_MODE(stdin);
+    count = fread(buffer, 1, sizeof(buffer), stdin);
     
     if (!feof(stdin))
     {
@@ -70,7 +75,7 @@ int main()
     }
     
     /* Construct a pb_istream_t for reading from the buffer */
-    pb_istream_t stream = pb_istream_from_buffer(buffer, count);
+    stream = pb_istream_from_buffer(buffer, count);
     
     /* Decode and print out the stuff */
     if (!print_person(&stream))

tests/basic_buffer/encode_buffer.c

@@ -6,9 +6,13 @@
 #include <stdio.h>
 #include <pb_encode.h>
 #include "person.pb.h"
+#include "test_helpers.h"
 
 int main()
 {
+    uint8_t buffer[512];
+    pb_ostream_t stream;
+    
     /* Initialize the structure with constants */
     Person person = {"Test Person 99", 99, true, "test@person.com",
         3, {{"555-12345678", true, Person_PhoneType_MOBILE},
@@ -16,17 +20,19 @@ int main()
             {"1234-5678", true, Person_PhoneType_WORK},
         }};
 
-    uint8_t buffer[512];
-    pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+    stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
     
     /* Now encode it and check if we succeeded. */
     if (pb_encode(&stream, Person_fields, &person))
     {    
+        /* Write the result data to stdout */
+        SET_BINARY_MODE(stdout);
         fwrite(buffer, 1, stream.bytes_written, stdout);
         return 0; /* Success */
     }
     else
     {
+        fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
         return 1; /* Failure */
     }
 }

tests/basic_stream/SConscript

@@ -0,0 +1,12 @@
+# Build and run a basic round-trip test using direct stream encoding.
+
+Import("env")
+
+enc = env.Program(["encode_stream.c", "#common/person.pb.c", "#common/pb_encode.o"])
+dec = env.Program(["decode_stream.c", "#common/person.pb.c", "#common/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_stream.output"])
+env.Decode(["encode_stream.output", "#common/person.proto"], MESSAGE = "Person")
+env.Compare(["decode_stream.output", "encode_stream.decoded"])
+

tests/basic_stream/decode_stream.c

@@ -4,6 +4,7 @@
 #include <stdio.h>
 #include <pb_decode.h>
 #include "person.pb.h"
+#include "test_helpers.h"
 
 /* This function is called once from main(), it handles
    the decoding and printing.
@@ -69,10 +70,10 @@ bool callback(pb_istream_t *stream, uint8_t *buf, size_t count)
 
 int main()
 {
-    /* Maximum size is specified to prevent infinite length messages from
-     * hanging this in the fuzz test.
-     */
-    pb_istream_t stream = {&callback, stdin, 10000};
+    pb_istream_t stream = {&callback, NULL, SIZE_MAX};
+    stream.state = stdin;
+    SET_BINARY_MODE(stdin);
+
     if (!print_person(&stream))
     {
         printf("Parsing failed: %s\n", PB_GET_ERROR(&stream));

tests/basic_stream/encode_stream.c

@@ -4,6 +4,7 @@
 #include <stdio.h>
 #include <pb_encode.h>
 #include "person.pb.h"
+#include "test_helpers.h"
 
 /* This binds the pb_ostream_t into the stdout stream */
 bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count)
@@ -22,11 +23,18 @@ int main()
         }};
     
     /* Prepare the stream, output goes directly to stdout */
-    pb_ostream_t stream = {&streamcallback, stdout, SIZE_MAX, 0};
+    pb_ostream_t stream = {&streamcallback, NULL, SIZE_MAX, 0};
+    stream.state = stdout;
+    SET_BINARY_MODE(stdout);
     
     /* Now encode it and check if we succeeded. */
     if (pb_encode(&stream, Person_fields, &person))
+    {
         return 0; /* Success */
+    }
     else
+    {
+        fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
         return 1; /* Failure */
     }
+}

tests/buffer_only/SConscript

@@ -0,0 +1,23 @@
+# Run the alltypes test case, but compile with PB_BUFFER_ONLY=1
+
+Import("env")
+
+# Take copy of the files for custom build.
+c = Copy("$TARGET", "$SOURCE")
+env.Command("pb_encode.c", "#../pb_encode.c", c)
+env.Command("pb_decode.c", "#../pb_decode.c", c)
+env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c)
+env.Command("alltypes.pb.c", "#alltypes/alltypes.pb.c", c)
+env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+
+# Define the compilation options
+opts = env.Clone()
+opts.Append(CPPDEFINES = {'PB_BUFFER_ONLY': 1})
+
+# Now build and run the test normally.
+enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"])
+dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_alltypes.output"])

tests/callbacks/SConscript

@@ -0,0 +1,14 @@
+# Test the functionality of the callback fields.
+
+Import("env")
+
+env.NanopbProto("callbacks")
+enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "#common/pb_encode.o"])
+dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "#common/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_callbacks.output"])
+
+env.Decode(["encode_callbacks.output", "callbacks.proto"], MESSAGE = "TestMessage")
+env.Compare(["decode_callbacks.output", "encode_callbacks.decoded"])
+

tests/callbacks/callbacks.proto

@@ -11,5 +11,6 @@ message TestMessage {
     repeated fixed32 fixed32value = 3;
     repeated fixed64 fixed64value = 4;
     optional SubMessage submsg = 5;
+    repeated string repeatedstring = 6;
 }
 

tests/callbacks/decode_callbacks.c

@@ -5,8 +5,9 @@
 #include <stdio.h>
 #include <pb_decode.h>
 #include "callbacks.pb.h"
+#include "test_helpers.h"
 
-bool print_string(pb_istream_t *stream, const pb_field_t *field, void *arg)
+bool print_string(pb_istream_t *stream, const pb_field_t *field, void **arg)
 {
     uint8_t buffer[1024] = {0};
     
@@ -20,51 +21,54 @@ bool print_string(pb_istream_t *stream, const pb_field_t *field, void *arg)
     /* Print the string, in format comparable with protoc --decode.
      * Format comes from the arg defined in main().
      */
-    printf((char*)arg, buffer);
+    printf((char*)*arg, buffer);
     return true;
 }
 
-bool print_int32(pb_istream_t *stream, const pb_field_t *field, void *arg)
+bool print_int32(pb_istream_t *stream, const pb_field_t *field, void **arg)
 {
     uint64_t value;
     if (!pb_decode_varint(stream, &value))
         return false;
     
-    printf((char*)arg, (long)value);
+    printf((char*)*arg, (long)value);
     return true;
 }
 
-bool print_fixed32(pb_istream_t *stream, const pb_field_t *field, void *arg)
+bool print_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg)
 {
     uint32_t value;
     if (!pb_decode_fixed32(stream, &value))
         return false;
     
-    printf((char*)arg, (long)value);
+    printf((char*)*arg, (long)value);
     return true;
 }
 
-bool print_fixed64(pb_istream_t *stream, const pb_field_t *field, void *arg)
+bool print_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg)
 {
     uint64_t value;
     if (!pb_decode_fixed64(stream, &value))
         return false;
     
-    printf((char*)arg, (long long)value);
+    printf((char*)*arg, (long)value);
     return true;
 }
 
 int main()
 {
     uint8_t buffer[1024];
-    size_t length = fread(buffer, 1, 1024, stdin);
-    pb_istream_t stream = pb_istream_from_buffer(buffer, length);
-    
+    size_t length;
+    pb_istream_t stream;
     /* Note: empty initializer list initializes the struct with all-0.
      * This is recommended so that unused callbacks are set to NULL instead
      * of crashing at runtime.
      */
-    TestMessage testmessage = {};
+    TestMessage testmessage = {{{NULL}}};
+    
+    SET_BINARY_MODE(stdin);
+    length = fread(buffer, 1, 1024, stdin);
+    stream = pb_istream_from_buffer(buffer, length);    
     
     testmessage.submsg.stringvalue.funcs.decode = &print_string;
     testmessage.submsg.stringvalue.arg = "submsg {\n  stringvalue: \"%s\"\n";
@@ -73,7 +77,7 @@ int main()
     testmessage.submsg.fixed32value.funcs.decode = &print_fixed32;
     testmessage.submsg.fixed32value.arg = "  fixed32value: %ld\n";
     testmessage.submsg.fixed64value.funcs.decode = &print_fixed64;
-    testmessage.submsg.fixed64value.arg = "  fixed64value: %lld\n}\n";
+    testmessage.submsg.fixed64value.arg = "  fixed64value: %ld\n}\n";
     
     testmessage.stringvalue.funcs.decode = &print_string;
     testmessage.stringvalue.arg = "stringvalue: \"%s\"\n";
@@ -82,7 +86,9 @@ int main()
     testmessage.fixed32value.funcs.decode = &print_fixed32;
     testmessage.fixed32value.arg = "fixed32value: %ld\n";
     testmessage.fixed64value.funcs.decode = &print_fixed64;
-    testmessage.fixed64value.arg = "fixed64value: %lld\n";
+    testmessage.fixed64value.arg = "fixed64value: %ld\n";
+    testmessage.repeatedstring.funcs.decode = &print_string;
+    testmessage.repeatedstring.arg = "repeatedstring: \"%s\"\n";
     
     if (!pb_decode(&stream, TestMessage_fields, &testmessage))
         return 1;

tests/callbacks/encode_callbacks.c

@@ -4,8 +4,9 @@
 #include <string.h>
 #include <pb_encode.h>
 #include "callbacks.pb.h"
+#include "test_helpers.h"
 
-bool encode_string(pb_ostream_t *stream, const pb_field_t *field, const void *arg)
+bool encode_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
 {
     char *str = "Hello world!";
     
@@ -15,7 +16,7 @@ bool encode_string(pb_ostream_t *stream, const pb_field_t *field, const void *ar
     return pb_encode_string(stream, (uint8_t*)str, strlen(str));
 }
 
-bool encode_int32(pb_ostream_t *stream, const pb_field_t *field, const void *arg)
+bool encode_int32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
 {
     if (!pb_encode_tag_for_field(stream, field))
         return false;
@@ -23,29 +24,49 @@ bool encode_int32(pb_ostream_t *stream, const pb_field_t *field, const void *arg
     return pb_encode_varint(stream, 42);
 }
 
-bool encode_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *arg)
+bool encode_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
 {
+    uint32_t value = 42;
+
     if (!pb_encode_tag_for_field(stream, field))
         return false;
     
-    uint32_t value = 42;
     return pb_encode_fixed32(stream, &value);
 }
 
-bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *arg)
+bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    uint64_t value = 42;
+
+    if (!pb_encode_tag_for_field(stream, field))
+        return false;
+    
+    return pb_encode_fixed64(stream, &value);
+}
+
+bool encode_repeatedstring(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
+{
+    char *str[4] = {"Hello world!", "", "Test", "Test2"};
+    int i;
+    
+    for (i = 0; i < 4; i++)
     {
         if (!pb_encode_tag_for_field(stream, field))
             return false;
         
-    uint64_t value = 42;
-    return pb_encode_fixed64(stream, &value);
+        if (!pb_encode_string(stream, (uint8_t*)str[i], strlen(str[i])))
+            return false;
+    }
+    return true;
 }
 
 int main()
 {
     uint8_t buffer[1024];
-    pb_ostream_t stream = pb_ostream_from_buffer(buffer, 1024);
-    TestMessage testmessage = {};
+    pb_ostream_t stream;
+    TestMessage testmessage = {{{NULL}}};
+    
+    stream = pb_ostream_from_buffer(buffer, 1024);
     
     testmessage.stringvalue.funcs.encode = &encode_string;
     testmessage.int32value.funcs.encode = &encode_int32;
@@ -58,9 +79,12 @@ int main()
     testmessage.submsg.fixed32value.funcs.encode = &encode_fixed32;
     testmessage.submsg.fixed64value.funcs.encode = &encode_fixed64;
 
+    testmessage.repeatedstring.funcs.encode = &encode_repeatedstring;
+    
     if (!pb_encode(&stream, TestMessage_fields, &testmessage))
         return 1;
     
+    SET_BINARY_MODE(stdout);
     if (fwrite(buffer, stream.bytes_written, 1, stdout) != 1)
         return 2;
     

tests/common/SConscript

@@ -0,0 +1,17 @@
+# Build the common files needed by multiple test cases
+
+Import('env')
+
+# Protocol definitions for the encode/decode_unittests
+env.NanopbProto("unittestproto")
+
+# Protocol definitions for basic_buffer/stream tests
+env.NanopbProto("person")
+
+# Binaries of the pb_decode.c and pb_encode.c
+# These are built using more strict warning flags.
+strict = env.Clone()
+strict.Append(CFLAGS = strict['CORECFLAGS'])
+strict.Object("pb_decode.o", "#../pb_decode.c")
+strict.Object("pb_encode.o", "#../pb_encode.c")
+

tests/common/test_helpers.h

@@ -0,0 +1,17 @@
+/* Compatibility helpers for the test programs. */
+
+#ifndef _TEST_HELPERS_H_
+#define _TEST_HELPERS_H_
+
+#ifdef _WIN32
+#include <io.h>
+#include <fcntl.h>
+#define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
+
+#else
+#define SET_BINARY_MODE(file)
+
+#endif
+
+
+#endif

tests/common/unittestproto.proto

@@ -8,6 +8,10 @@ message FloatArray {
     repeated float data = 1 [(nanopb).max_count = 10];
 }
 
+message StringMessage {
+    required string data = 1 [(nanopb).max_size = 10];
+}
+
 message CallbackArray {
     // We cheat a bit and use this message for testing other types, too.
     // Nanopb does not care about the actual defined data type for callback

tests/cxx_main_program/SConscript

@@ -0,0 +1,20 @@
+# Run the alltypes test case, but compile it as C++ instead.
+# In fact, compile the entire nanopb using C++ compiler.
+
+Import("env")
+
+# Copy the files to .cxx extension in order to force C++ build.
+c = Copy("$TARGET", "$SOURCE")
+env.Command("pb_encode.cxx", "#../pb_encode.c", c)
+env.Command("pb_decode.cxx", "#../pb_decode.c", c)
+env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c)
+env.Command("alltypes.pb.cxx", "#alltypes/alltypes.pb.c", c)
+env.Command("encode_alltypes.cxx", "#alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.cxx", "#alltypes/decode_alltypes.c", c)
+
+# Now build and run the test normally.
+enc = env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx"])
+dec = env.Program(["decode_alltypes.cxx", "alltypes.pb.cxx", "pb_decode.cxx"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_alltypes.output"])

tests/decode_unittests/SConscript

@@ -0,0 +1,4 @@
+Import('env')
+p = env.Program(["decode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_decode.o"])
+env.RunTest(p)
+

tests/decode_unittests/decode_unittests.c

@@ -19,11 +19,11 @@ bool stream_callback(pb_istream_t *stream, uint8_t *buf, size_t count)
 }
 
 /* Verifies that the stream passed to callback matches the byte array pointed to by arg. */
-bool callback_check(pb_istream_t *stream, const pb_field_t *field, void *arg)
+bool callback_check(pb_istream_t *stream, const pb_field_t *field, void **arg)
 {
     int i;
     uint8_t byte;
-    pb_bytes_array_t *ref = (pb_bytes_array_t*) arg;
+    pb_bytes_array_t *ref = (pb_bytes_array_t*) *arg;
     
     for (i = 0; i < ref->size; i++)
     {
@@ -289,6 +289,16 @@ int main()
         TEST((s = S("\x08"), !pb_decode(&s, IntegerArray_fields, &dest)))
     }
     
+    {
+        pb_istream_t s;
+        IntegerContainer dest = {{0}};
+        
+        COMMENT("Testing pb_decode_delimited")
+        TEST((s = S("\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05"),
+              pb_decode_delimited(&s, IntegerContainer_fields, &dest)) &&
+              dest.submsg.data_count == 5)
+    }
+    
     if (status != 0)
         fprintf(stdout, "\n\nSome tests FAILED!\n");
     

tests/encode_unittests/SConscript

@@ -0,0 +1,5 @@
+# Build and run the stand-alone unit tests for the nanopb encoder part.
+
+Import('env')
+p = env.Program(["encode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_encode.o"])
+env.RunTest(p)

tests/encode_unittests/encode_unittests.c

@@ -17,7 +17,7 @@ bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count)
     return true;
 }
 
-bool fieldcallback(pb_ostream_t *stream, const pb_field_t *field, const void *arg)
+bool fieldcallback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
 {
     int value = 0x55;
     if (!pb_encode_tag_for_field(stream, field))
@@ -25,7 +25,7 @@ bool fieldcallback(pb_ostream_t *stream, const pb_field_t *field, const void *ar
     return pb_encode_varint(stream, value);
 }
 
-bool crazyfieldcallback(pb_ostream_t *stream, const pb_field_t *field, const void *arg)
+bool crazyfieldcallback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg)
 {
     /* This callback writes different amount of data the second time. */
     uint32_t *state = (uint32_t*)arg;
@@ -180,12 +180,14 @@ int main()
     {
         uint8_t buffer[30];
         pb_ostream_t s;
-        char value[] = "xyzzy";
+        char value[30] = "xyzzy";
         
         COMMENT("Test pb_enc_string")
-        TEST(WRITES(pb_enc_string(&s, NULL, &value), "\x05xyzzy"))
+        TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x05xyzzy"))
         value[0] = '\0';
-        TEST(WRITES(pb_enc_string(&s, NULL, &value), "\x00"))
+        TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x00"))
+        memset(value, 'x', 30);
+        TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x0Axxxxxxxxxx"))
     }
     
     {
@@ -242,6 +244,16 @@ int main()
                     "\x0A\x07\x0A\x05\x01\x02\x03\x04\x05"))
     }
     
+    {
+        uint8_t buffer[20];
+        pb_ostream_t s;
+        IntegerContainer msg = {{5, {1,2,3,4,5}}};
+        
+        COMMENT("Test pb_encode_delimited.")
+        TEST(WRITES(pb_encode_delimited(&s, IntegerContainer_fields, &msg),
+                    "\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05"))
+    }
+    
     {
         uint8_t buffer[10];
         pb_ostream_t s;

tests/extensions/SConscript

@@ -0,0 +1,16 @@
+# Test the support for extension fields.
+
+Import("env")
+
+# We use the files from the alltypes test case
+incpath = env.Clone()
+incpath.Append(PROTOCPATH = '#alltypes')
+incpath.Append(CPPPATH = '#alltypes')
+
+incpath.NanopbProto("extensions")
+enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb$OBJSUFFIX", "#common/pb_encode.o"])
+dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb$OBJSUFFIX", "#common/pb_decode.o"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_extensions.output"])
+

tests/extensions/decode_extensions.c

@@ -0,0 +1,58 @@
+/* Test decoding of extension fields. */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <pb_decode.h>
+#include "alltypes.pb.h"
+#include "extensions.pb.h"
+#include "test_helpers.h"
+
+#define TEST(x) if (!(x)) { \
+    printf("Test " #x " failed.\n"); \
+    return 2; \
+    }
+
+int main(int argc, char **argv)
+{
+    uint8_t buffer[1024];
+    size_t count;
+    pb_istream_t stream;
+    
+    AllTypes alltypes = {0};
+    int32_t extensionfield1;
+    pb_extension_t ext1;
+    ExtensionMessage extensionfield2;
+    pb_extension_t ext2;
+    
+    /* Read the message data */
+    SET_BINARY_MODE(stdin);
+    count = fread(buffer, 1, sizeof(buffer), stdin);
+    stream = pb_istream_from_buffer(buffer, count);
+    
+    /* Add the extensions */
+    alltypes.extensions = &ext1;        
+
+    ext1.type = &AllTypes_extensionfield1;
+    ext1.dest = &extensionfield1;
+    ext1.next = &ext2;
+    
+    ext2.type = &ExtensionMessage_AllTypes_extensionfield2;
+    ext2.dest = &extensionfield2;
+    ext2.next = NULL;
+    
+    /* Decode the message */
+    if (!pb_decode(&stream, AllTypes_fields, &alltypes))
+    {
+        printf("Parsing failed: %s\n", PB_GET_ERROR(&stream));
+        return 1;
+    }
+
+    /* Check that the extensions decoded properly */
+    TEST(extensionfield1 == 12345)
+    TEST(strcmp(extensionfield2.test1, "test") == 0)
+    TEST(extensionfield2.test2 == 54321)
+    
+    return 0;
+}
+

tests/extensions/encode_extensions.c

@@ -0,0 +1,50 @@
+/* Tests extension fields.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pb_encode.h>
+#include "alltypes.pb.h"
+#include "extensions.pb.h"
+#include "test_helpers.h"
+
+int main(int argc, char **argv)
+{
+    uint8_t buffer[1024];
+    pb_ostream_t stream;
+
+    AllTypes alltypes = {0};
+    int32_t extensionfield1 = 12345;
+    pb_extension_t ext1;
+    ExtensionMessage extensionfield2 = {"test", 54321};
+    pb_extension_t ext2;
+
+    /* Set up the extensions */
+    alltypes.extensions = &ext1;
+
+    ext1.type = &AllTypes_extensionfield1;
+    ext1.dest = &extensionfield1;
+    ext1.next = &ext2;
+    
+    ext2.type = &ExtensionMessage_AllTypes_extensionfield2;
+    ext2.dest = &extensionfield2;
+    ext2.next = NULL;
+
+    /* Set up the output stream */
+    stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+    
+    /* Now encode the message and check if we succeeded. */
+    if (pb_encode(&stream, AllTypes_fields, &alltypes))
+    {
+        SET_BINARY_MODE(stdout);
+        fwrite(buffer, 1, stream.bytes_written, stdout);
+        return 0; /* Success */
+    }
+    else
+    {
+        fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
+        return 1; /* Failure */
+    }
+}
+

tests/extensions/extensions.options

@@ -0,0 +1 @@
+* max_size:16

tests/extensions/extensions.proto

@@ -0,0 +1,17 @@
+import 'alltypes.proto';
+
+extend AllTypes {
+    optional int32 AllTypes_extensionfield1 = 255;
+}
+
+message ExtensionMessage {
+    extend AllTypes {
+        optional ExtensionMessage AllTypes_extensionfield2 = 254;
+	required ExtensionMessage AllTypes_extensionfield3 = 253;
+	repeated ExtensionMessage AllTypes_extensionfield4 = 252;
+    }
+    
+    required string test1 = 1;
+    required int32 test2 = 2;
+}
+

tests/extra_fields/SConscript

@@ -0,0 +1,10 @@
+# Test that the decoder properly handles unknown fields in the input.
+
+Import("env")
+
+dec = env.GetBuildPath('#basic_buffer/${PROGPREFIX}decode_buffer${PROGSUFFIX}')
+env.RunTest('person_with_extra_field.output', [dec, "person_with_extra_field.pb"])
+env.Compare(["person_with_extra_field.output", "person_with_extra_field.expected"])
+
+dec2 = env.GetBuildPath('#alltypes/${PROGPREFIX}decode_alltypes${PROGSUFFIX}')
+env.RunTest('alltypes_with_extra_fields.output', [dec2, 'alltypes_with_extra_fields.pb'])

tests/extra_fields/person_with_extra_field.expected

@@ -0,0 +1,14 @@
+name: "Test Person 99"
+id: 99
+email: "test@person.com"
+phone {
+  number: "555-12345678"
+  type: MOBILE
+}
+phone {
+  number: "99-2342"
+}
+phone {
+  number: "1234-5678"
+  type: WORK
+}

tests/field_size_16/SConscript

@@ -0,0 +1,24 @@
+# Run the alltypes test case, but compile with PB_FIELD_16BIT=1.
+# Also the .proto file has been modified to have high indexes.
+
+Import("env")
+
+# Take copy of the files for custom build.
+c = Copy("$TARGET", "$SOURCE")
+env.Command("pb_encode.c", "#../pb_encode.c", c)
+env.Command("pb_decode.c", "#../pb_decode.c", c)
+env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+
+env.NanopbProto("alltypes")
+
+# Define the compilation options
+opts = env.Clone()
+opts.Append(CPPDEFINES = {'PB_FIELD_16BIT': 1})
+
+# Now build and run the test normally.
+enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"])
+dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_alltypes.output"])

tests/field_size_16/alltypes.options

@@ -0,0 +1,3 @@
+* max_size:16
+* max_count:5
+

tests/field_size_16/alltypes.proto

@@ -0,0 +1,90 @@
+message SubMessage {
+    required string substuff1 = 1 [default = "1"];
+    required int32 substuff2 = 2 [default = 2];
+    optional fixed32 substuff3 = 65535 [default = 3];
+}
+
+message EmptyMessage {
+
+}
+
+enum MyEnum {
+    Zero = 0;
+    First = 1;
+    Second = 2;
+    Truth = 42;
+}
+
+message AllTypes {
+    required int32      req_int32   = 1;
+    required int64      req_int64   = 2;
+    required uint32     req_uint32  = 3;
+    required uint64     req_uint64  = 4;
+    required sint32     req_sint32  = 5;
+    required sint64     req_sint64  = 6;
+    required bool       req_bool    = 7;
+    
+    required fixed32    req_fixed32 = 8;
+    required sfixed32   req_sfixed32= 9;
+    required float      req_float   = 10;
+    
+    required fixed64    req_fixed64 = 11;
+    required sfixed64   req_sfixed64= 12;
+    required double     req_double  = 13;
+    
+    required string     req_string  = 14;
+    required bytes      req_bytes   = 15;
+    required SubMessage req_submsg  = 16;
+    required MyEnum     req_enum    = 17;
+    required EmptyMessage req_emptymsg = 18;
+    
+    
+    repeated int32      rep_int32   = 21;
+    repeated int64      rep_int64   = 22;
+    repeated uint32     rep_uint32  = 23;
+    repeated uint64     rep_uint64  = 24;
+    repeated sint32     rep_sint32  = 25;
+    repeated sint64     rep_sint64  = 26;
+    repeated bool       rep_bool    = 27;
+    
+    repeated fixed32    rep_fixed32 = 28;
+    repeated sfixed32   rep_sfixed32= 29;
+    repeated float      rep_float   = 30;
+    
+    repeated fixed64    rep_fixed64 = 10031;
+    repeated sfixed64   rep_sfixed64= 10032;
+    repeated double     rep_double  = 10033;
+    
+    repeated string     rep_string  = 10034;
+    repeated bytes      rep_bytes   = 10035;
+    repeated SubMessage rep_submsg  = 10036;
+    repeated MyEnum     rep_enum    = 10037;
+    repeated EmptyMessage rep_emptymsg = 10038;
+    
+    optional int32      opt_int32   = 10041 [default = 4041];
+    optional int64      opt_int64   = 10042 [default = 4042];
+    optional uint32     opt_uint32  = 10043 [default = 4043];
+    optional uint64     opt_uint64  = 10044 [default = 4044];
+    optional sint32     opt_sint32  = 10045 [default = 4045];
+    optional sint64     opt_sint64  = 10046 [default = 4046];
+    optional bool       opt_bool    = 10047 [default = false];
+    
+    optional fixed32    opt_fixed32 = 10048 [default = 4048];
+    optional sfixed32   opt_sfixed32= 10049 [default = 4049];
+    optional float      opt_float   = 10050 [default = 4050];
+    
+    optional fixed64    opt_fixed64 = 10051 [default = 4051];
+    optional sfixed64   opt_sfixed64= 10052 [default = 4052];
+    optional double     opt_double  = 10053 [default = 4053];
+    
+    optional string     opt_string  = 10054 [default = "4054"];
+    optional bytes      opt_bytes   = 10055 [default = "4055"];
+    optional SubMessage opt_submsg  = 10056;
+    optional MyEnum     opt_enum    = 10057 [default = Second];
+    optional EmptyMessage opt_emptymsg = 10058;
+
+    // Just to make sure that the size of the fields has been calculated
+    // properly, i.e. otherwise a bug in last field might not be detected.
+    required int32      end = 10099;
+}
+

tests/field_size_32/SConscript

@@ -0,0 +1,24 @@
+# Run the alltypes test case, but compile with PB_FIELD_32BIT=1.
+# Also the .proto file has been modified to have high indexes.
+
+Import("env")
+
+# Take copy of the files for custom build.
+c = Copy("$TARGET", "$SOURCE")
+env.Command("pb_encode.c", "#../pb_encode.c", c)
+env.Command("pb_decode.c", "#../pb_decode.c", c)
+env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+
+env.NanopbProto("alltypes")
+
+# Define the compilation options
+opts = env.Clone()
+opts.Append(CPPDEFINES = {'PB_FIELD_32BIT': 1})
+
+# Now build and run the test normally.
+enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"])
+dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_alltypes.output"])

tests/field_size_32/alltypes.options

@@ -0,0 +1,3 @@
+* max_size:16
+* max_count:5
+

tests/field_size_32/alltypes.proto

@@ -0,0 +1,90 @@
+message SubMessage {
+    required string substuff1 = 1 [default = "1"];
+    required int32 substuff2 = 2 [default = 2];
+    optional fixed32 substuff3 = 12365535 [default = 3];
+}
+
+message EmptyMessage {
+
+}
+
+enum MyEnum {
+    Zero = 0;
+    First = 1;
+    Second = 2;
+    Truth = 42;
+}
+
+message AllTypes {
+    required int32      req_int32   = 1;
+    required int64      req_int64   = 2;
+    required uint32     req_uint32  = 3;
+    required uint64     req_uint64  = 4;
+    required sint32     req_sint32  = 5;
+    required sint64     req_sint64  = 6;
+    required bool       req_bool    = 7;
+    
+    required fixed32    req_fixed32 = 8;
+    required sfixed32   req_sfixed32= 9;
+    required float      req_float   = 10;
+    
+    required fixed64    req_fixed64 = 11;
+    required sfixed64   req_sfixed64= 12;
+    required double     req_double  = 13;
+    
+    required string     req_string  = 14;
+    required bytes      req_bytes   = 15;
+    required SubMessage req_submsg  = 16;
+    required MyEnum     req_enum    = 17;
+    required EmptyMessage req_emptymsg = 18;
+    
+    
+    repeated int32      rep_int32   = 21;
+    repeated int64      rep_int64   = 22;
+    repeated uint32     rep_uint32  = 23;
+    repeated uint64     rep_uint64  = 24;
+    repeated sint32     rep_sint32  = 25;
+    repeated sint64     rep_sint64  = 26;
+    repeated bool       rep_bool    = 27;
+    
+    repeated fixed32    rep_fixed32 = 28;
+    repeated sfixed32   rep_sfixed32= 29;
+    repeated float      rep_float   = 30;
+    
+    repeated fixed64    rep_fixed64 = 10031;
+    repeated sfixed64   rep_sfixed64= 10032;
+    repeated double     rep_double  = 10033;
+    
+    repeated string     rep_string  = 10034;
+    repeated bytes      rep_bytes   = 10035;
+    repeated SubMessage rep_submsg  = 10036;
+    repeated MyEnum     rep_enum    = 10037;
+    repeated EmptyMessage rep_emptymsg = 10038;
+    
+    optional int32      opt_int32   = 10041 [default = 4041];
+    optional int64      opt_int64   = 10042 [default = 4042];
+    optional uint32     opt_uint32  = 10043 [default = 4043];
+    optional uint64     opt_uint64  = 10044 [default = 4044];
+    optional sint32     opt_sint32  = 10045 [default = 4045];
+    optional sint64     opt_sint64  = 10046 [default = 4046];
+    optional bool       opt_bool    = 10047 [default = false];
+    
+    optional fixed32    opt_fixed32 = 10048 [default = 4048];
+    optional sfixed32   opt_sfixed32= 10049 [default = 4049];
+    optional float      opt_float   = 10050 [default = 4050];
+    
+    optional fixed64    opt_fixed64 = 10051 [default = 4051];
+    optional sfixed64   opt_sfixed64= 10052 [default = 4052];
+    optional double     opt_double  = 10053 [default = 4053];
+    
+    optional string     opt_string  = 10054 [default = "4054"];
+    optional bytes      opt_bytes   = 10055 [default = "4055"];
+    optional SubMessage opt_submsg  = 10056;
+    optional MyEnum     opt_enum    = 10057 [default = Second];
+    optional EmptyMessage opt_emptymsg = 10058;
+
+    // Just to make sure that the size of the fields has been calculated
+    // properly, i.e. otherwise a bug in last field might not be detected.
+    required int32      end = 13432099;
+}
+

tests/missing_fields/SConscript

@@ -0,0 +1,8 @@
+# Check that the decoder properly detects when required fields are missing.
+
+Import("env")
+
+env.NanopbProto("missing_fields")
+test = env.Program(["missing_fields.c", "missing_fields.pb.c", "#common/pb_encode.o", "#common/pb_decode.o"])
+env.RunTest(test)
+

tests/missing_fields/missing_fields.c

@@ -7,12 +7,13 @@
 
 int main()
 {
-    uint8_t buffer[512] = {};
+    uint8_t buffer[512];
     
     /* Create a message with one missing field */
     {
-        MissingField msg = {};
+        MissingField msg = {0};
         pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        
         if (!pb_encode(&stream, MissingField_fields, &msg))
         {
             printf("Encode failed.\n");
@@ -22,7 +23,7 @@ int main()
 
     /* Test that it decodes properly if we don't require that field */
     {
-        MissingField msg = {};
+        MissingField msg = {0};
         pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer));
         
         if (!pb_decode(&stream, MissingField_fields, &msg))
@@ -34,7 +35,7 @@ int main()
     
     /* Test that it does *not* decode properly if we require the field */
     {
-        AllFields msg = {};
+        AllFields msg = {0};
         pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer));
         
         if (pb_decode(&stream, AllFields_fields, &msg))

tests/multiple_files/SConscript

@@ -0,0 +1,13 @@
+# Test that multiple .proto files don't cause name collisions.
+
+Import("env")
+
+incpath = env.Clone()
+incpath.Append(PROTOCPATH = '#multiple_files')
+
+incpath.NanopbProto("callbacks")
+incpath.NanopbProto("callbacks2")
+test = incpath.Program(["test_multiple_files.c", "callbacks.pb.c", "callbacks2.pb.c"])
+
+env.RunTest(test)
+

tests/multiple_files/callbacks.proto

@@ -0,0 +1,16 @@
+message SubMessage {
+    optional string stringvalue = 1;
+    repeated int32 int32value = 2;
+    repeated fixed32 fixed32value = 3;
+    repeated fixed64 fixed64value = 4;
+}
+
+message TestMessage {
+    optional string stringvalue = 1;
+    repeated int32 int32value = 2;
+    repeated fixed32 fixed32value = 3;
+    repeated fixed64 fixed64value = 4;
+    optional SubMessage submsg = 5;
+    repeated string repeatedstring = 6;
+}
+

tests/multiple_files/callbacks2.proto

@@ -0,0 +1,9 @@
+// Test if including generated header file for this file + implicit include of
+// callbacks.pb.h still compiles. Used with test_compiles.c.
+import "callbacks.proto";
+
+message Callback2Message {
+    required TestMessage tstmsg = 1;
+    required SubMessage submsg = 2;
+}
+

tests/multiple_files/test_multiple_files.c

@@ -0,0 +1,12 @@
+/*
+ * Tests if this still compiles when multiple .proto files are involved.
+ */
+
+#include <stdio.h>
+#include <pb_encode.h>
+#include "callbacks2.pb.h"
+
+int main()
+{
+	return 0;
+}

tests/no_errmsg/SConscript

@@ -0,0 +1,23 @@
+# Run the alltypes test case, but compile with PB_NO_ERRMSG=1
+
+Import("env")
+
+# Take copy of the files for custom build.
+c = Copy("$TARGET", "$SOURCE")
+env.Command("pb_encode.c", "#../pb_encode.c", c)
+env.Command("pb_decode.c", "#../pb_decode.c", c)
+env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c)
+env.Command("alltypes.pb.c", "#alltypes/alltypes.pb.c", c)
+env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
+env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+
+# Define the compilation options
+opts = env.Clone()
+opts.Append(CPPDEFINES = {'PB_NO_ERRMSG': 1})
+
+# Now build and run the test normally.
+enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"])
+dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"])
+
+env.RunTest(enc)
+env.RunTest([dec, "encode_alltypes.output"])

tests/no_messages/SConscript

@@ -0,0 +1,7 @@
+# Test that a .proto file without any messages compiles fine.
+
+Import("env")
+
+env.NanopbProto("no_messages")
+env.Object('no_messages.pb.c')
+

tests/options/SConscript

@@ -0,0 +1,9 @@
+# Test that the generator options work as expected.
+
+Import("env")
+
+env.NanopbProto("options")
+env.Object('options.pb.c')
+
+env.Match(['options.pb.h', 'options.expected'])
+

tests/options/options.expected

@@ -2,4 +2,6 @@ char filesize\[20\];
 char msgsize\[30\];
 char fieldsize\[40\];
 pb_callback_t int32_callback;
-[^_]EnumValue1 = 1
+\sEnumValue1 = 1
+Message5_EnumValue1
+} pb_packed my_packed_struct;

tests/options/options.proto

@@ -36,4 +36,38 @@ enum Enum1
 {
     option (nanopb_enumopt).long_names = false;
     EnumValue1 = 1;
+    EnumValue2 = 2;
 }
+
+message EnumTest
+{
+    required Enum1 field = 1 [default = EnumValue2];
+}
+
+// Short enum names inside message
+message Message5
+{
+    enum Enum2
+    {
+       option (nanopb_enumopt).long_names = false;
+       EnumValue1 = 1;
+    }
+    required Enum2 field = 1 [default = EnumValue1];
+}
+
+// Packed structure
+message my_packed_struct
+{
+    option (nanopb_msgopt).packed_struct = true;
+    optional int32 myfield = 1;
+}
+
+// Message with ignored field
+// Note: doesn't really test if the field is missing in the output,
+// but atleast tests that the output compiles.
+message Message6
+{
+    required int32 field1 = 1;
+    optional int32 field2 = 2 [(nanopb).type = FT_IGNORE];
+}
+

tests/site_scons/site_init.py

@@ -0,0 +1,111 @@
+import subprocess
+import sys
+import re
+
+try:
+    # Make terminal colors work on windows
+    import colorama
+    colorama.init()
+except ImportError:
+    pass
+
+def add_nanopb_builders(env):
+    '''Add the necessary builder commands for nanopb tests.'''
+    
+    # Build command for building .pb from .proto using protoc
+    def proto_actions(source, target, env, for_signature):
+        esc = env['ESCAPE']
+        dirs = ' '.join(['-I' + esc(env.GetBuildPath(d)) for d in env['PROTOCPATH']])
+        return '$PROTOC $PROTOCFLAGS %s -o%s %s' % (dirs, esc(str(target[0])), esc(str(source[0])))
+
+    proto_file_builder = Builder(generator = proto_actions,
+                                 suffix = '.pb',
+                                 src_suffix = '.proto')
+    env.Append(BUILDERS = {'Proto': proto_file_builder})
+    env.SetDefault(PROTOC = 'protoc')
+    env.SetDefault(PROTOCPATH = ['.'])
+
+    # Build command for running nanopb generator
+    import os.path
+    def nanopb_targets(target, source, env):
+        basename = os.path.splitext(str(source[0]))[0]
+        target.append(basename + '.pb.h')
+        return target, source
+
+    nanopb_file_builder = Builder(action = '$NANOPB_GENERATOR $NANOPB_FLAGS $SOURCE',
+                                  suffix = '.pb.c',
+                                  src_suffix = '.pb',
+                                  emitter = nanopb_targets)
+    env.Append(BUILDERS = {'Nanopb': nanopb_file_builder})
+    env.SetDefault(NANOPB_GENERATOR = 'python ' + env.GetBuildPath("#../generator/nanopb_generator.py"))
+    env.SetDefault(NANOPB_FLAGS = '-q')
+
+    # Combined method to run both protoc and nanopb generator
+    def run_protoc_and_nanopb(env, source):
+        b1 = env.Proto(source)
+        b2 = env.Nanopb(source)
+        return b1 + b2
+    env.AddMethod(run_protoc_and_nanopb, "NanopbProto")
+
+    # Build command that runs a test program and saves the output
+    def run_test(target, source, env):
+        if len(source) > 1:
+            infile = open(str(source[1]))
+        else:
+            infile = None
+        
+        pipe = subprocess.Popen(str(source[0]),
+                                stdin = infile,
+                                stdout = open(str(target[0]), 'w'),
+                                stderr = sys.stderr)
+        result = pipe.wait()
+        if result == 0:
+            print '\033[32m[ OK ]\033[0m   Ran ' + str(source[0])
+        else:
+            print '\033[31m[FAIL]\033[0m   Program ' + str(source[0]) + ' returned ' + str(result)
+        return result
+        
+    run_test_builder = Builder(action = run_test,
+                               suffix = '.output')
+    env.Append(BUILDERS = {'RunTest': run_test_builder})
+
+    # Build command that decodes a message using protoc
+    def decode_actions(source, target, env, for_signature):
+        dirs = ' '.join(['-I' + env.GetBuildPath(d) for d in env['PROTOCPATH']])
+        return '$PROTOC $PROTOCFLAGS %s --decode=%s %s <%s >%s' % (dirs, env['MESSAGE'], source[1], source[0], target[0])
+
+    decode_builder = Builder(generator = decode_actions,
+                             suffix = '.decoded')
+    env.Append(BUILDERS = {'Decode': decode_builder})    
+
+    # Build command that asserts that two files be equal
+    def compare_files(target, source, env):
+        data1 = open(str(source[0]), 'rb').read()
+        data2 = open(str(source[1]), 'rb').read()
+        if data1 == data2:
+            print '\033[32m[ OK ]\033[0m   Files equal: ' + str(source[0]) + ' and ' + str(source[1])
+            return 0
+        else:
+            print '\033[31m[FAIL]\033[0m   Files differ: ' + str(source[0]) + ' and ' + str(source[1])
+            return 1
+
+    compare_builder = Builder(action = compare_files,
+                              suffix = '.equal')
+    env.Append(BUILDERS = {'Compare': compare_builder})
+
+    # Build command that checks that each pattern in source2 is found in source1.
+    def match_files(target, source, env):
+        data = open(str(source[0]), 'rU').read()
+        patterns = open(str(source[1]))
+        for pattern in patterns:
+            if pattern.strip() and not re.search(pattern.strip(), data, re.MULTILINE):
+                print '\033[31m[FAIL]\033[0m   Pattern not found in ' + str(source[0]) + ': ' + pattern
+                return 1
+        else:
+            print '\033[32m[ OK ]\033[0m   All patterns found in ' + str(source[0])
+            return 0
+
+    match_builder = Builder(action = match_files, suffix = '.matched')
+    env.Append(BUILDERS = {'Match': match_builder})
+    
+

tests/special_characters/SConscript

@@ -0,0 +1,7 @@
+# Test that special characters in .proto filenames work.
+
+Import('env')
+
+env.Proto("funny-proto+name has.characters.proto")
+env.Nanopb("funny-proto+name has.characters.pb.c", "funny-proto+name has.characters.pb")
+env.Object("funny-proto+name has.characters.pb.c")

tests/testperson.pb

@@ -1,3 +0,0 @@
-
-Test Person7foobar@foobar.com"
-555-12345678