Publishing nanopb-0.2.0

Patch courtesy of Michael Haberler.

.gitignore

@@ -0,0 +1,39 @@
+*.gcda
+*.gcno
+*.gcov
+*.o
+*.pb.c
+*.pb.h
+*.pb
+*~
+*.tar.gz
+julkaisu.txt
+docs/*.html
+docs/generator_flow.png
+example/client
+example/server
+example_avr_double/decode_double
+example_avr_double/encode_double
+example_avr_double/test_conversions
+example_unions/decode
+example_unions/encode
+generator/nanopb_pb2.pyc
+tests/decode_unittests
+tests/encode_unittests
+tests/test_compiles
+tests/test_decode1
+tests/test_decode2
+tests/test_decode3
+tests/test_decode3_buf
+tests/test_decode_callbacks
+tests/test_encode1
+tests/test_encode2
+tests/test_encode3
+tests/test_encode3_buf
+tests/test_encode_callbacks
+tests/test_missing_fields
+tests/test_multiple_files
+tests/bc_decode
+tests/bc_encode
+tests/breakpoints
+

CHANGELOG

@@ -1,3 +1,43 @@
+nanopb-0.2.0
+ NOTE: This release requires you to regenerate all .pb.c
+       files. Files generated by older versions will not
+       compile anymore.
+
+ Reformat generated .pb.c files using macros (issue 58)
+ Rename PB_HTYPE_ARRAY -> PB_HTYPE_REPEATED
+ Separate PB_HTYPE to PB_ATYPE and PB_HTYPE
+ Move STATIC_ASSERTs to .pb.c file
+ Added CMake file (by Pavel Ilin)
+ Add option to give file extension to generator (by Michael Haberler)
+ Documentation updates
+
+nanopb-0.1.9
+ Fixed error message bugs (issues 52, 56)
+ Sanitize #ifndef filename (issue 50)
+ Performance improvements
+ Add compile-time option PB_BUFFER_ONLY
+ Add Java package name to nanopb.proto
+ Check for sizeof(double) == 8 (issue 54)
+ Added generator option to ignore some fields. (issue 51)
+ Added generator option to make message structs packed. (issue 49)
+ Add more test cases.
+
+nanopb-0.1.8
+ Fix bugs in the enum short names introduced in 0.1.7 (issues 42, 43)
+ Fix STATIC_ASSERT macro when using multiple .proto files. (issue 41)
+ Fix missing initialization of istream.errmsg
+ Make tests/Makefile work for non-gcc compilers (issue 40)
+
+nanopb-0.1.7
+ Remove "skip" mode from pb_istream_t callbacks. Example implementation had a bug. (issue 37)
+ Add option to use shorter names for enum values (issue 38)
+ Improve options support in generator (issues 12, 30)
+ Add nanopb version number to generated files (issue 36)
+ Add extern "C" to generated headers (issue 35)
+ Add names for structs to allow forward declaration (issue 39)
+ Add buffer size check in example (issue 34)
+ Fix build warnings on MS compilers (issue 33)
+
 nanopb-0.1.6
  Reorganize the field decoder interface (issue 2)
  Improve performance in submessage decoding (issue 28)

cmake/FindNanopb.cmake

@@ -0,0 +1,224 @@
+# Locate and configure the nanopb library.
+#
+# The following varialbes have to be set:
+#
+#   NANOPB_SRC_ROOT_FOLDER  - Path to nanopb source folder
+#
+# The following variables can be set and are optional:
+#
+#
+#   PROTOBUF_SRC_ROOT_FOLDER - When compiling with MSVC, if this cache variable is set
+#                              the protobuf-default VS project build locations
+#                              (vsprojects/Debug & vsprojects/Release) will be searched
+#                              for libraries and binaries.
+#
+#   NANOPB_IMPORT_DIRS       - List of additional directories to be searched for
+#                              imported .proto files.
+#
+#   NANOPB_GENERATE_CPP_APPEND_PATH - By default -I will be passed to protoc
+#                                     for each directory where a proto file is referenced.
+#                                     Set to FALSE if you want to disable this behaviour.
+#
+# Defines the following variables:
+#
+#   NANOPB_FOUND - Found the nanopb library (source&header files, generator tool, protoc compiler tool)
+#   NANOPB_INCLUDE_DIRS - Include directories for Google Protocol Buffers
+#
+# The following cache variables are also available to set or use:
+#   NANOPB_GENERATOR_EXECUTABLE - The nanopb generator
+#   PROTOBUF_PROTOC_EXECUTABLE - The protoc compiler
+#
+#  ====================================================================
+#
+# NANOPB_GENERATE_CPP (public function)
+#   SRCS = Variable to define with autogenerated
+#          source files
+#   HDRS = Variable to define with autogenerated
+#          header files
+#   ARGN = proto files
+#
+#  ====================================================================
+#  Example:
+#
+#   set(NANOPB_SRC_ROOT_FOLDER "/path/to/nanopb")
+#   set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${NANOPB_SRC_ROOT_FOLDER}/cmake)
+#   find_package( Nanopb REQUIRED )
+#   include_directories(${NANOPB_INCLUDE_DIRS})
+#
+#   NANOPB_GENERATE_CPP(PROTO_SRCS PROTO_HDRS foo.proto)
+#
+#   include_directories(${CMAKE_CURRENT_BINARY_DIR})
+#   add_executable(bar bar.cc ${PROTO_SRCS} ${PROTO_HDRS})
+#
+#  ====================================================================
+
+#=============================================================================
+# Copyright 2009 Kitware, Inc.
+# Copyright 2009-2011 Philip Lowman <philip@yhbt.com>
+# Copyright 2008 Esben Mose Hansen, Ange Optimization ApS
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+#   notice, this list of conditions and the following disclaimer in the
+#   documentation and/or other materials provided with the distribution.
+#
+# * Neither the names of Kitware, Inc., the Insight Software Consortium,
+#   nor the names of their contributors may be used to endorse or promote
+#   products derived from this software without specific prior written
+#   permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#=============================================================================
+#
+# Changes
+# 2013.01.31 - Pavlo Ilin - used Modules/FindProtobuf.cmake from cmake 2.8.10 to
+#                           write FindNanopb.cmake
+#
+#=============================================================================
+
+
+function(NANOPB_GENERATE_CPP SRCS HDRS)
+  if(NOT ARGN)
+    return()
+  endif()
+
+  if(NANOPB_GENERATE_CPP_APPEND_PATH)
+    # Create an include path for each file specified
+    foreach(FIL ${ARGN})
+      get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+      get_filename_component(ABS_PATH ${ABS_FIL} PATH)
+
+      list(FIND _nanobp_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _nanobp_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  else()
+    set(_nanobp_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+
+  if(DEFINED NANOPB_IMPORT_DIRS)
+    foreach(DIR ${NANOPB_IMPORT_DIRS})
+      get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+      list(FIND _nanobp_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _nanobp_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  endif()
+
+  set(${SRCS})
+  set(${HDRS})
+  get_filename_component(GENERATOR_PATH ${NANOPB_GENERATOR_EXECUTABLE} PATH)
+
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb"
+      COMMAND  ${PROTOBUF_PROTOC_EXECUTABLE}
+      ARGS -I${GENERATOR_PATH} -I${CMAKE_CURRENT_BINARY_DIR} ${_nanobp_include_path} -o${FIL_WE}.pb ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM )
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      COMMAND python
+      ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb
+      DEPENDS ${FIL_WE}.pb
+      COMMENT "Running nanopb generator on ${FIL_WE}.pb"
+      VERBATIM )
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set(${SRCS} ${${SRCS}} ${NANOPB_SRCS} PARENT_SCOPE)
+  set(${HDRS} ${${HDRS}} ${NANOPB_HDRS} PARENT_SCOPE)
+
+endfunction()
+
+
+
+#
+# Main.
+#
+
+# By default have NANOPB_GENERATE_CPP macro pass -I to protoc
+# for each directory where a proto file is referenced.
+if(NOT DEFINED NANOPB_GENERATE_CPP_APPEND_PATH)
+  set(NANOPB_GENERATE_CPP_APPEND_PATH TRUE)
+endif()
+
+# Find the include directory
+find_path(NANOPB_INCLUDE_DIRS
+    pb.h
+    PATHS ${NANOPB_SRC_ROOT_FOLDER}
+)
+mark_as_advanced(NANOPB_INCLUDE_DIRS)
+
+# Find nanopb source files
+set(NANOPB_SRCS)
+set(NANOPB_HDRS)
+list(APPEND _nanopb_srcs pb_decode.c pb_encode.c)
+list(APPEND _nanopb_hdrs pb_decode.h pb_encode.h pb.h)
+
+foreach(FIL ${_nanopb_srcs})
+  find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_SRC_ROOT_FOLDER} ${NANOPB_INCLUDE_DIRS})
+  list(APPEND NANOPB_SRCS "${${FIL}__nano_pb_file}")
+  mark_as_advanced(${FIL}__nano_pb_file)
+endforeach()
+
+foreach(FIL ${_nanopb_hdrs})
+  find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_INCLUDE_DIRS})
+  mark_as_advanced(${FIL}__nano_pb_file)
+  list(APPEND NANOPB_HDRS "${${FIL}__nano_pb_file}")
+endforeach()
+
+# Find the protoc Executable
+find_program(PROTOBUF_PROTOC_EXECUTABLE
+    NAMES protoc
+    DOC "The Google Protocol Buffers Compiler"
+    PATHS
+    ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Release
+    ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Debug
+)
+mark_as_advanced(PROTOBUF_PROTOC_EXECUTABLE)
+
+# Find nanopb generator
+find_file(NANOPB_GENERATOR_EXECUTABLE
+    NAMES nanopb_generator.py
+    DOC "nanopb generator"
+    PATHS
+    ${NANOPB_SRC_ROOT_FOLDER}/generator
+)
+mark_as_advanced(NANOPB_GENERATOR_EXECUTABLE)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(NANOPB DEFAULT_MSG
+  NANOPB_INCLUDE_DIRS
+  NANOPB_SRCS NANOPB_HDRS
+  NANOPB_GENERATOR_EXECUTABLE
+  PROTOBUF_PROTOC_EXECUTABLE
+  )

docs/concepts.rst

@@ -38,6 +38,20 @@ This file, in turn, requires the file *google/protobuf/descriptor.proto*. This i
 
     protoc -I/usr/include -Inanopb/generator -I. -omessage.pb message.proto
 
+The options can be defined in file, message and field scopes::
+
+    option (nanopb_fileopt).max_size = 20; // File scope
+    message Message
+    {
+        option (nanopb_msgopt).max_size = 30; // Message scope
+        required string fieldsize = 1 [(nanopb).max_size = 40]; // Field scope
+    }
+
+It is also possible to give the options on command line, but then they will affect the whole file. For example::
+
+    user@host:~$ python ../generator/nanopb_generator.py -s 'max_size: 20' message.pb
+
+
 Streams
 =======
 
@@ -92,9 +106,8 @@ Writing to stdout::
 
 Input streams
 -------------
-For input streams, there are a few extra rules:
+For input streams, there is one extra rule:
 
-#) If buf is NULL, read from stream but don't store the data. This is used to skip unknown input.
 #) You don't need to know the length of the message in advance. After getting EOF error when reading, set bytes_left to 0 and return false. Pb_decode will detect this and if the EOF was in a proper position, it will return true.
 
 Here is the structure::
@@ -242,16 +255,8 @@ For example this submessage in the Person.proto file::
 generates this field description array for the structure *Person_PhoneNumber*::
 
  const pb_field_t Person_PhoneNumber_fields[3] = {
-    {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING,
+    PB_FIELD(  1, STRING  , REQUIRED, STATIC, Person_PhoneNumber, number, number, 0),
-    offsetof(Person_PhoneNumber, number), 0,
+    PB_FIELD(  2, ENUM    , OPTIONAL, STATIC, Person_PhoneNumber, type, number, &Person_PhoneNumber_type_default),
-    pb_membersize(Person_PhoneNumber, number), 0, 0},
-
-    {2, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT,
-    pb_delta(Person_PhoneNumber, type, number),
-    pb_delta(Person_PhoneNumber, has_type, type),
-    pb_membersize(Person_PhoneNumber, type), 0,
-    &Person_PhoneNumber_type_default},
-
     PB_LAST_FIELD
  };
 
@@ -263,8 +268,8 @@ Most functions in nanopb return bool: *true* means success, *false* means failur
 
 The error messages help in guessing what is the underlying cause of the error. The most common error conditions are:
 
-1) Running out of memory. Because everything is allocated from the stack, nanopb can't detect this itself. Encoding or decoding the same type of a message always takes the same amount of stack space. Therefore, if it works once, it works always.
+1) Running out of memory, i.e. stack overflow.
-2) Invalid field description. These are usually stored as constants, so if it works under the debugger, it always does.
+2) Invalid field descriptors (would usually mean a bug in the generator).
 3) IO errors in your own stream callbacks.
 4) Errors that happen in your callback functions.
 5) Exceeding the max_size or bytes_left of a stream.

docs/index.rst

@@ -36,23 +36,26 @@ Features and limitations
 **Features**
 
 #) Pure C runtime
-#) Small code size (2–10 kB depending on processor)
+#) Small code size (2–10 kB depending on processor, plus any message definitions)
-#) Small ram usage (typically 200 bytes)
+#) Small ram usage (typically ~300 bytes, plus any message structs)
 #) Allows specifying maximum size for strings and arrays, so that they can be allocated statically.
 #) No malloc needed: everything can be allocated statically or on the stack.
 #) You can use either encoder or decoder alone to cut the code size in half.
+#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, packed arrays.
+#) Callback mechanism for handling messages larger than can fit in available RAM.
+#) Extensive set of tests.
 
 **Limitations**
 
 #) User must provide callbacks when decoding arrays or strings without maximum size. Malloc support could be added as a separate module.
-#) Some speed has been sacrificed for code size. For example varint calculations are always done in 64 bits.
+#) Some speed has been sacrificed for code size.
 #) Encoding is focused on writing to streams. For memory buffers only it could be made more efficient.
 #) The deprecated Protocol Buffers feature called "groups" is not supported.
 #) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file.
 #) Unknown fields are not preserved when decoding and re-encoding a message.
 #) Reflection (runtime introspection) is not supported. E.g. you can't request a field by giving its name in a string.
 #) Numeric arrays are always encoded as packed, even if not marked as packed in .proto. This causes incompatibility with decoders that do not support packed format.
-#) Cyclic references between messages are not supported. They could be supported in callback-mode if there was an option in the generator to set the mode.
+#) Cyclic references between messages are supported only in callback mode.
 
 Getting started
 ===============
@@ -104,10 +107,3 @@ Debugging and testing
 =====================
 Extensive unittests are included under the *tests* folder. Just type *make* there to run the tests.
 
-This also generates a file called *breakpoints* which includes all lines returning *false* in nanopb. You can use this in gdb by typing *source breakpoints*, after which gdb will break on first nanopb error.
-
-Wishlist
-========
-#) A specialized encoder for encoding to a memory buffer. Should serialize in reverse order to avoid having to determine submessage size beforehand.
-#) A cleaner rewrite of the Python-based source generator.
-#) Better performance for 16- and 8-bit platforms: use smaller datatypes where possible.

docs/reference.rst

@@ -22,6 +22,8 @@ PB_FIELD_32BIT                 Add support for tag numbers > 65535 and fields la
                                Increases code size 9 bytes per each field. Compiler error will tell if you need this.
 PB_NO_ERRMSG                   Disables the support for error messages; only error information is the true/false return value.
                                Decreases the code size by a few hundred bytes.
+PB_BUFFER_ONLY                 Disables the support for custom streams. Only supports encoding to memory buffers.
+                               Speeds up execution and decreases code size slightly.
 ============================  ================================================================================================
 
 The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow raising some datatype limits to suit larger messages.
@@ -35,22 +37,23 @@ pb_type_t
 ---------
 Defines the encoder/decoder behaviour that should be used for a field. ::
 
-    typedef enum { ... } pb_type_t;
+    typedef uint8_t pb_type_t;
 
-The low-order byte of the enumeration values defines the function that can be used for encoding and decoding the field data:
+The low-order nibble of the enumeration values defines the function that can be used for encoding and decoding the field data:
 
 ==================== ===== ================================================
 LTYPE identifier     Value Storage format
 ==================== ===== ================================================
 PB_LTYPE_VARINT      0x00  Integer.
 PB_LTYPE_SVARINT     0x01  Integer, zigzag encoded.
-PB_LTYPE_FIXED       0x02  Integer or floating point.
+PB_LTYPE_FIXED32     0x02  32-bit integer or floating point.
-PB_LTYPE_BYTES       0x03  Structure with *size_t* field and byte array.
+PB_LTYPE_FIXED64     0x03  64-bit integer or floating point.
-PB_LTYPE_STRING      0x04  Null-terminated string.
+PB_LTYPE_BYTES       0x04  Structure with *size_t* field and byte array.
-PB_LTYPE_SUBMESSAGE  0x05  Submessage structure.
+PB_LTYPE_STRING      0x05  Null-terminated string.
+PB_LTYPE_SUBMESSAGE  0x06  Submessage structure.
 ==================== ===== ================================================
 
-The high-order byte defines whether the field is required, optional, repeated or callback:
+The bits 4-5 define whether the field is required, optional or repeated:
 
 ==================== ===== ================================================
 HTYPE identifier     Value Field handling
@@ -58,13 +61,24 @@ HTYPE identifier     Value Field handling
 PB_HTYPE_REQUIRED    0x00  Verify that field exists in decoded message.
 PB_HTYPE_OPTIONAL    0x10  Use separate *has_<field>* boolean to specify
                            whether the field is present.
-PB_HTYPE_ARRAY       0x20  A repeated field with preallocated array.
+                           (Unless it is a callback)
+PB_HTYPE_REPEATED    0x20  A repeated field with preallocated array.
                            Separate *<field>_count* for number of items.
-PB_HTYPE_CALLBACK    0x30  A field with dynamic storage size, data is
+                           (Unless it is a callback)
-                           actually a pointer to a structure containing a
-                           callback function.
 ==================== ===== ================================================
 
+The bits 6-7 define the how the storage for the field is allocated:
+
+==================== ===== ================================================
+ATYPE identifier     Value Allocation method
+==================== ===== ================================================
+PB_ATYPE_STATIC      0x00  Statically allocated storage in the structure.
+PB_ATYPE_CALLBACK    0x40  A field with dynamic storage size. Struct field
+                           actually contains a pointer to a callback
+                           function.
+==================== ===== ================================================
+
+
 pb_field_t
 ----------
 Describes a single structure field with memory position in relation to others. The descriptions are usually autogenerated. ::
@@ -81,7 +95,7 @@ Describes a single structure field with memory position in relation to others. T
     } pb_packed;
 
 :tag:           Tag number of the field or 0 to terminate a list of fields.
-:type:          LTYPE and HTYPE of the field.
+:type:          LTYPE, HTYPE and ATYPE of the field.
 :data_offset:   Offset of field data, relative to the end of the previous field.
 :size_offset:   Offset of *bool* flag for optional fields or *size_t* count for arrays, relative to field data.
 :data_size:     Size of a single data entry, in bytes. For PB_LTYPE_BYTES, the size of the byte array inside the containing structure. For PB_HTYPE_CALLBACK, size of the C data type if known.
@@ -381,7 +395,7 @@ Remove the data for a field from the stream, without actually decoding it::
 
     For decoding numeric (including enumerated and boolean) values, use `pb_decode_varint`_, `pb_decode_svarint`_, `pb_decode_fixed32`_ and `pb_decode_fixed64`_. They take a pointer to a 32- or 64-bit C variable, which you may then cast to smaller datatype for storage.
 
-    For decoding strings and bytes fields, the length has already been decoded. You can therefore check the total length in *stream->state* and read the data using `pb_read`_.
+    For decoding strings and bytes fields, the length has already been decoded. You can therefore check the total length in *stream->bytes_left* and read the data using `pb_read`_.
 
     Finally, for decoding submessages in a callback, simply use `pb_decode`_ and pass it the *SubMessage_fields* descriptor array.
 

example/common.c

@@ -19,15 +19,6 @@ static bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count)
     int fd = (intptr_t)stream->state;
     int result;
     
-    if (buf == NULL)
-    {
-        /* Well, this is a really inefficient way to skip input. */
-        /* It is only used when there are unknown fields. */
-        char dummy;
-        while (count-- && recv(fd, &dummy, 1, 0) == 1);
-        return count == 0;
-    }
-    
     result = recv(fd, buf, count, MSG_WAITALL);
     
     if (result == 0)

example_avr_double/Makefile

@@ -0,0 +1,22 @@
+CFLAGS=-Wall -Werror -I .. -g -O0
+DEPS=double_conversion.c ../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h
+
+all: run_tests
+
+clean:
+	rm -f test_conversions encode_double decode_double doubleproto.pb.c doubleproto.pb.h
+
+test_conversions: test_conversions.c double_conversion.c
+	$(CC) $(CFLAGS) -o $@ $^
+
+%: %.c $(DEPS) doubleproto.pb.h doubleproto.pb.c
+	$(CC) $(CFLAGS) -o $@ $< double_conversion.c ../pb_decode.c ../pb_encode.c doubleproto.pb.c
+
+doubleproto.pb.c doubleproto.pb.h: doubleproto.proto ../generator/nanopb_generator.py
+	protoc -I. -I../generator -I/usr/include -odoubleproto.pb $<
+	python ../generator/nanopb_generator.py doubleproto.pb
+
+run_tests: test_conversions encode_double decode_double
+	./test_conversions
+	./encode_double | ./decode_double
+    

example_avr_double/README.txt

@@ -0,0 +1,22 @@
+Some processors/compilers, such as AVR-GCC, do not support the double
+datatype. Instead, they have sizeof(double) == 4. Because protocol
+binary format uses the double encoding directly, this causes trouble
+if the protocol in .proto requires double fields.
+
+This directory contains a solution to this problem. It uses uint64_t
+to store the raw wire values, because its size is correct on all
+platforms. The file double_conversion.c provides functions that
+convert these values to/from floats, without relying on compiler
+support.
+
+To use this method, you need to make two modifications to your code:
+
+1) Change all 'double' fields into 'fixed64' in the .proto.
+
+2) Whenever writing to a 'double' field, use float_to_double().
+
+3) Whenever reading a 'double' field, use double_to_float().
+
+The conversion routines should be as accurate as the float datatype can
+be. Furthermore, they should handle all special values (NaN, inf, denormalized
+numbers) correctly. There are testcases in test_conversions.c.

example_avr_double/decode_double.c

@@ -0,0 +1,33 @@
+/* Decodes a double value into a float variable.
+ * Used to read double values with AVR code, which doesn't support double directly.
+ */
+
+#include <stdio.h>
+#include <pb_decode.h>
+#include "double_conversion.h"
+#include "doubleproto.pb.h"
+
+int main()
+{
+    uint8_t buffer[32];
+    size_t count = fread(buffer, 1, sizeof(buffer), stdin);
+    pb_istream_t stream = pb_istream_from_buffer(buffer, count);
+    
+    AVRDoubleMessage message;
+    pb_decode(&stream, AVRDoubleMessage_fields, &message);
+    
+    float v1 = double_to_float(message.field1);
+    float v2 = double_to_float(message.field2);
+
+    printf("Values: %f %f\n", v1, v2);
+    
+    if (v1 == 1234.5678f &&
+        v2 == 0.00001f)
+    {
+        return 0;
+    }
+    else
+    {
+        return 1;
+    }
+}

example_avr_double/double_conversion.c

@@ -0,0 +1,123 @@
+/* Conversion routines for platforms that do not support 'double' directly. */
+
+#include "double_conversion.h"
+#include <math.h>
+
+typedef union {
+    float f;
+    uint32_t i;
+} conversion_t;
+
+/* Note: IEE 754 standard specifies float formats as follows:
+ * Single precision: sign,  8-bit exp, 23-bit frac.
+ * Double precision: sign, 11-bit exp, 52-bit frac.
+ */
+
+uint64_t float_to_double(float value)
+{
+    conversion_t in;
+    in.f = value;
+    uint8_t sign;
+    int16_t exponent;
+    uint64_t mantissa;
+    
+    /* Decompose input value */
+    sign = (in.i >> 31) & 1;
+    exponent = ((in.i >> 23) & 0xFF) - 127;
+    mantissa = in.i & 0x7FFFFF;
+    
+    if (exponent == 128)
+    {
+        /* Special value (NaN etc.) */
+        exponent = 1024;
+    }
+    else if (exponent == -127)
+    {
+        if (!mantissa)
+        {
+            /* Zero */
+            exponent = -1023;
+        }
+        else
+        {
+            /* Denormalized */
+            mantissa <<= 1;
+            while (!(mantissa & 0x800000))
+            {
+                mantissa <<= 1;
+                exponent--;
+            }
+            mantissa &= 0x7FFFFF;
+        }
+    }
+    
+    /* Combine fields */
+    mantissa <<= 29;
+    mantissa |= (uint64_t)(exponent + 1023) << 52;
+    mantissa |= (uint64_t)sign << 63;
+    
+    return mantissa;
+}
+
+float double_to_float(uint64_t value)
+{
+    uint8_t sign;
+    int16_t exponent;
+    uint32_t mantissa;
+    conversion_t out;
+
+    /* Decompose input value */
+    sign = (value >> 63) & 1;
+    exponent = ((value >> 52) & 0x7FF) - 1023;
+    mantissa = (value >> 28) & 0xFFFFFF; /* Highest 24 bits */
+ 
+    /* Figure if value is in range representable by floats. */
+    if (exponent == 1024)
+    {
+        /* Special value */
+        exponent = 128;
+    }
+    else if (exponent > 127)
+    {
+        /* Too large */        
+        if (sign)
+            return -INFINITY;
+        else
+            return INFINITY;
+    }
+    else if (exponent < -150)
+    {
+        /* Too small */
+        if (sign)
+            return -0.0f;
+        else
+            return 0.0f;
+    }
+    else if (exponent < -126)
+    {
+        /* Denormalized */
+        mantissa |= 0x1000000;
+        mantissa >>= (-126 - exponent);
+        exponent = -127;
+    }
+ 
+    /* Round off mantissa */
+    mantissa = (mantissa + 1) >> 1;
+    
+    /* Check if mantissa went over 2.0 */
+    if (mantissa & 0x800000)
+    {
+        exponent += 1;
+        mantissa &= 0x7FFFFF;
+        mantissa >>= 1;
+    }
+    
+    /* Combine fields */
+    out.i = mantissa;
+    out.i |= (uint32_t)(exponent + 127) << 23;
+    out.i |= (uint32_t)sign << 31;
+    
+    return out.f;
+}
+
+

example_avr_double/double_conversion.h

@@ -0,0 +1,26 @@
+/* AVR-GCC does not have real double datatype. Instead its double
+ * is equal to float, i.e. 32 bit value. If you need to communicate
+ * with other systems that use double in their .proto files, you
+ * need to do some conversion.
+ *
+ * These functions use bitwise operations to mangle floats into doubles
+ * and then store them in uint64_t datatype.
+ */
+
+#ifndef DOUBLE_CONVERSION
+#define DOUBLE_CONVERSION
+
+#include <stdint.h>
+
+/* Convert native 4-byte float into a 8-byte double. */
+extern uint64_t float_to_double(float value);
+
+/* Convert 8-byte double into native 4-byte float.
+ * Values are rounded to nearest, 0.5 away from zero.
+ * Overflowing values are converted to Inf or -Inf.
+ */
+extern float double_to_float(uint64_t value);
+
+
+#endif
+

example_avr_double/doubleproto.proto

@@ -0,0 +1,13 @@
+// A message containing doubles, as used by other applications.
+message DoubleMessage {
+    required double field1 = 1;
+    required double field2 = 2;
+}
+
+// A message containing doubles, but redefined using uint64_t.
+// For use in AVR code.
+message AVRDoubleMessage {
+    required fixed64 field1 = 1;
+    required fixed64 field2 = 2;
+}
+

example_avr_double/encode_double.c

@@ -0,0 +1,25 @@
+/* Encodes a float value into a double on the wire.
+ * Used to emit doubles from AVR code, which doesn't support double directly.
+ */
+
+#include <stdio.h>
+#include <pb_encode.h>
+#include "double_conversion.h"
+#include "doubleproto.pb.h"
+
+int main()
+{
+    AVRDoubleMessage message = {
+        float_to_double(1234.5678f),
+        float_to_double(0.00001f)
+    };
+    
+    uint8_t buffer[32];
+    pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+    
+    pb_encode(&stream, AVRDoubleMessage_fields, &message);
+    fwrite(buffer, 1, stream.bytes_written, stdout);
+
+    return 0;
+}
+

example_avr_double/test_conversions.c

@@ -0,0 +1,56 @@
+#include "double_conversion.h"
+#include <math.h>
+#include <stdio.h>
+
+static const double testvalues[] = {
+           0.0,        -0.0,         0.1,         -0.1,
+          M_PI,       -M_PI,  123456.789,  -123456.789,
+      INFINITY,   -INFINITY,         NAN, INFINITY - INFINITY,
+          1e38,       -1e38,        1e39,        -1e39,
+         1e-38,      -1e-38,       1e-39,       -1e-39,
+   3.14159e-37,-3.14159e-37, 3.14159e-43, -3.14159e-43,
+         1e-60,      -1e-60,       1e-45,       -1e-45,
+    0.99999999999999, -0.99999999999999, 127.999999999999, -127.999999999999
+};
+
+#define TESTVALUES_COUNT (sizeof(testvalues)/sizeof(testvalues[0]))
+
+int main()
+{
+    int status = 0;
+    int i;
+    for (i = 0; i < TESTVALUES_COUNT; i++)
+    {
+        double orig = testvalues[i];
+        float expected_float = (float)orig;
+        double expected_double = (double)expected_float;
+        
+        float got_float = double_to_float(*(uint64_t*)&orig);
+        uint64_t got_double = float_to_double(got_float);
+        
+        uint32_t e1 = *(uint32_t*)&expected_float;
+        uint32_t g1 = *(uint32_t*)&got_float;
+        uint64_t e2 = *(uint64_t*)&expected_double;
+        uint64_t g2 = got_double;
+        
+        if (g1 != e1)
+        {
+            printf("%3d double_to_float fail: %08x != %08x\n", i, g1, e1);
+            status = 1;
+        }
+        
+        if (g2 != e2)
+        {
+            printf("%3d float_to_double fail: %016llx != %016llx\n", i,
+                (unsigned long long)g2,
+                (unsigned long long)e2);
+            status = 1;
+        }
+    }
+
+    return status;
+}
+
+    
+    
+

generator/nanopb.proto

@@ -7,9 +7,30 @@
 
 import "google/protobuf/descriptor.proto";
 
+option java_package = "fi.kapsi.koti.jpa.nanopb";
+
+enum FieldType {
+    FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible.
+    FT_CALLBACK = 1; // Always generate a callback field.
+    FT_STATIC = 2; // Generate a static field or raise an exception if not possible.
+    FT_IGNORE = 3; // Ignore the field completely.
+}
+
 message NanoPBOptions {
+  // Allocated size for 'bytes' and 'string' fields.
   optional int32 max_size = 1;
+  
+  // Allocated number of entries in arrays ('repeated' fields)
   optional int32 max_count = 2;
+  
+  // Force type of field (callback or static allocation)
+  optional FieldType type = 3 [default = FT_DEFAULT];
+  
+  // Use long names for enums, i.e. EnumName_EnumValue.
+  optional bool long_names = 4 [default = true];
+  
+  // Add 'packed' attribute to generated structs.
+  optional bool packed_struct = 5 [default = false];
 }
 
 // Protocol Buffers extension number registry
@@ -20,7 +41,20 @@ message NanoPBOptions {
 // Extensions: 1010 (all types)
 // --------------------------------
 
+extend google.protobuf.FileOptions {
+    optional NanoPBOptions nanopb_fileopt = 1010;
+}
+
+extend google.protobuf.MessageOptions {
+    optional NanoPBOptions nanopb_msgopt = 1010;
+}
+
+extend google.protobuf.EnumOptions {
+    optional NanoPBOptions nanopb_enumopt = 1010;
+}
+
 extend google.protobuf.FieldOptions {
     optional NanoPBOptions nanopb = 1010;
 }
 
+

generator/nanopb_generator.py

@@ -1,4 +1,5 @@
 '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.'''
+nanopb_version = "nanopb-0.2.0"
 
 try:
     import google.protobuf.descriptor_pb2 as descriptor
@@ -22,24 +23,34 @@ except:
     print
     raise
 
+
+
+
+
+
+# ---------------------------------------------------------------------------
+#                     Generation of single fields
+# ---------------------------------------------------------------------------
+
+import time
 import os.path
 
-# Values are tuple (c type, pb ltype)
+# Values are tuple (c type, pb type)
 FieldD = descriptor.FieldDescriptorProto
 datatypes = {
-    FieldD.TYPE_BOOL: ('bool', 'PB_LTYPE_VARINT'),
+    FieldD.TYPE_BOOL:       ('bool',     'BOOL'),
-    FieldD.TYPE_DOUBLE: ('double', 'PB_LTYPE_FIXED64'),
+    FieldD.TYPE_DOUBLE:     ('double',   'DOUBLE'),
-    FieldD.TYPE_FIXED32: ('uint32_t', 'PB_LTYPE_FIXED32'),
+    FieldD.TYPE_FIXED32:    ('uint32_t', 'FIXED32'),
-    FieldD.TYPE_FIXED64: ('uint64_t', 'PB_LTYPE_FIXED64'),
+    FieldD.TYPE_FIXED64:    ('uint64_t', 'FIXED64'),
-    FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED32'),
+    FieldD.TYPE_FLOAT:      ('float',    'FLOAT'),
-    FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT'),
+    FieldD.TYPE_INT32:      ('int32_t',  'INT32'),
-    FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT'),
+    FieldD.TYPE_INT64:      ('int64_t',  'INT64'),
-    FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED32'),
+    FieldD.TYPE_SFIXED32:   ('int32_t',  'SFIXED32'),
-    FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED64'),
+    FieldD.TYPE_SFIXED64:   ('int64_t',  'SFIXED64'),
-    FieldD.TYPE_SINT32: ('int32_t', 'PB_LTYPE_SVARINT'),
+    FieldD.TYPE_SINT32:     ('int32_t',  'SINT32'),
-    FieldD.TYPE_SINT64: ('int64_t', 'PB_LTYPE_SVARINT'),
+    FieldD.TYPE_SINT64:     ('int64_t',  'SINT64'),
-    FieldD.TYPE_UINT32: ('uint32_t', 'PB_LTYPE_VARINT'),
+    FieldD.TYPE_UINT32:     ('uint32_t', 'UINT32'),
-    FieldD.TYPE_UINT64: ('uint64_t', 'PB_LTYPE_VARINT')
+    FieldD.TYPE_UINT64:     ('uint64_t', 'UINT64')
 }
 
 class Names:
@@ -62,6 +73,9 @@ class Names:
         else:
             raise ValueError("Name parts should be of type str")
     
+    def __eq__(self, other):
+        return isinstance(other, Names) and self.parts == other.parts
+    
 def names_from_type_name(type_name):
     '''Parse Names() from FieldDescriptorProto type_name'''
     if type_name[0] != '.':
@@ -69,19 +83,27 @@ def names_from_type_name(type_name):
     return Names(type_name[1:].split('.'))
 
 class Enum:
-    def __init__(self, names, desc):
+    def __init__(self, names, desc, enum_options):
         '''desc is EnumDescriptorProto'''
+        
+        self.options = enum_options
         self.names = names + desc.name
+        
+        if enum_options.long_names:
             self.values = [(self.names + x.name, x.number) for x in desc.value]            
+        else:
+            self.values = [(names + x.name, x.number) for x in desc.value] 
+        
+        self.value_longnames = [self.names + x.name for x in desc.value]
     
     def __str__(self):
-        result = 'typedef enum {\n'
+        result = 'typedef enum _%s {\n' % self.names
         result += ',\n'.join(["    %s = %d" % x for x in self.values])
         result += '\n} %s;' % self.names
         return result
 
 class Field:
-    def __init__(self, struct_name, desc):
+    def __init__(self, struct_name, desc, field_options):
         '''desc is FieldDescriptorProto'''
         self.tag = desc.number
         self.struct_name = struct_name
@@ -91,76 +113,83 @@ class Field:
         self.max_count = None
         self.array_decl = ""
         
-        # Parse nanopb-specific field options
+        # Parse field options
-        if desc.options.HasExtension(nanopb_pb2.nanopb):
+        if field_options.HasField("max_size"):
-            ext = desc.options.Extensions[nanopb_pb2.nanopb]
+            self.max_size = field_options.max_size
-            if ext.HasField("max_size"):
+        
-                self.max_size = ext.max_size
+        if field_options.HasField("max_count"):
-            if ext.HasField("max_count"):
+            self.max_count = field_options.max_count
-                self.max_count = ext.max_count
         
         if desc.HasField('default_value'):
             self.default = desc.default_value
            
-        # Decide HTYPE
+        # Check field rules, i.e. required/optional/repeated.
-        # HTYPE is the high-order nibble of nanopb field description,
+        can_be_static = True
-        # defining whether value is required/optional/repeated.
-        is_callback = False
         if desc.label == FieldD.LABEL_REQUIRED:
-            self.htype = 'PB_HTYPE_REQUIRED'
+            self.rules = 'REQUIRED'
         elif desc.label == FieldD.LABEL_OPTIONAL:
-            self.htype = 'PB_HTYPE_OPTIONAL'
+            self.rules = 'OPTIONAL'
         elif desc.label == FieldD.LABEL_REPEATED:
+            self.rules = 'REPEATED'
             if self.max_count is None:
-                is_callback = True
+                can_be_static = False
             else:
-                self.htype = 'PB_HTYPE_ARRAY'
                 self.array_decl = '[%d]' % self.max_count
         else:
             raise NotImplementedError(desc.label)
         
-        # Decide LTYPE and CTYPE
+        # Decide the C data type to use in the struct.
-        # LTYPE is the low-order nibble of nanopb field description,
-        # defining how to decode an individual value.
-        # CTYPE is the name of the c type to use in the struct.
         if datatypes.has_key(desc.type):
-            self.ctype, self.ltype = datatypes[desc.type]
+            self.ctype, self.pbtype = datatypes[desc.type]
         elif desc.type == FieldD.TYPE_ENUM:
-            self.ltype = 'PB_LTYPE_VARINT'
+            self.pbtype = 'ENUM'
             self.ctype = names_from_type_name(desc.type_name)
             if self.default is not None:
                 self.default = self.ctype + self.default
         elif desc.type == FieldD.TYPE_STRING:
-            self.ltype = 'PB_LTYPE_STRING'
+            self.pbtype = 'STRING'
             if self.max_size is None:
-                is_callback = True
+                can_be_static = False
             else:
                 self.ctype = 'char'
                 self.array_decl += '[%d]' % self.max_size
         elif desc.type == FieldD.TYPE_BYTES:
-            self.ltype = 'PB_LTYPE_BYTES'
+            self.pbtype = 'BYTES'
             if self.max_size is None:
-                is_callback = True
+                can_be_static = False
             else:
                 self.ctype = self.struct_name + self.name + 't'
         elif desc.type == FieldD.TYPE_MESSAGE:
-            self.ltype = 'PB_LTYPE_SUBMESSAGE'
+            self.pbtype = 'MESSAGE'
             self.ctype = self.submsgname = names_from_type_name(desc.type_name)
         else:
             raise NotImplementedError(desc.type)
         
-        if is_callback:
+        if field_options.type == nanopb_pb2.FT_DEFAULT:
-            self.htype = 'PB_HTYPE_CALLBACK'
+            if can_be_static:
+                field_options.type = nanopb_pb2.FT_STATIC
+            else:
+                field_options.type = nanopb_pb2.FT_CALLBACK
+        
+        if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static:
+            raise Exception("Field %s is defined as static, but max_size or max_count is not given." % self.name)
+        
+        if field_options.type == nanopb_pb2.FT_STATIC:
+            self.allocation = 'STATIC'
+        elif field_options.type == nanopb_pb2.FT_CALLBACK:
+            self.allocation = 'CALLBACK'
             self.ctype = 'pb_callback_t'
             self.array_decl = ''
+        else:
+            raise NotImplementedError(field_options.type)
     
     def __cmp__(self, other):
         return cmp(self.tag, other.tag)
     
     def __str__(self):
-        if self.htype == 'PB_HTYPE_OPTIONAL':
+        if self.rules == 'OPTIONAL':
             result = '    bool has_' + self.name + ';\n'
-        elif self.htype == 'PB_HTYPE_ARRAY':
+        elif self.rules == 'REPEATED' and self.allocation == 'STATIC':
             result = '    size_t ' + self.name + '_count;\n'
         else:
             result = ''
@@ -169,7 +198,7 @@ class Field:
     
     def types(self):
         '''Return definitions for any special types this field might need.'''
-        if self.ltype == 'PB_LTYPE_BYTES' and self.max_size is not None:
+        if self.pbtype == 'BYTES' and self.allocation == 'STATIC':
             result = 'typedef struct {\n'
             result += '    size_t size;\n'
             result += '    uint8_t bytes[%d];\n' % self.max_size
@@ -183,29 +212,24 @@ class Field:
         if self.default is None:
             return None
 
-        if self.ltype == 'PB_LTYPE_STRING':
+        ctype, default = self.ctype, self.default
-            ctype = 'char'
+        array_decl = ''
-            if self.max_size is None:
+        
+        if self.pbtype == 'STRING':
+            if self.allocation != 'STATIC':
                 return None # Not implemented
-            else:
+        
-                array_decl = '[%d]' % (self.max_size + 1)
+            array_decl = '[%d]' % self.max_size
             default = str(self.default).encode('string_escape')
             default = default.replace('"', '\\"')
             default = '"' + default + '"'
-        elif self.ltype == 'PB_LTYPE_BYTES':
+        elif self.pbtype == 'BYTES':
+            if self.allocation != 'STATIC':
+                return None # Not implemented
+
             data = self.default.decode('string_escape')
             data = ['0x%02x' % ord(c) for c in data]
-            
-            if self.max_size is None:
-                return None # Not implemented
-            else:
-                ctype = self.ctype
-            
             default = '{%d, {%s}}' % (len(data), ','.join(data))
-            array_decl = ''
-        else:
-            ctype, default = self.ctype, self.default
-            array_decl = ''
         
         if declaration_only:
             return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl)
@@ -216,47 +240,30 @@ class Field:
         '''Return the pb_field_t initializer to use in the constant array.
         prev_field_name is the name of the previous field or None.
         '''
-        result = '    {%d, ' % self.tag
+        result  = '    PB_FIELD(%3d, ' % self.tag
-        result += '(pb_type_t) ((int) ' + self.htype
+        result += '%-8s, ' % self.pbtype
-        if self.ltype is not None:
+        result += '%s, ' % self.rules
-            result += ' | (int) ' + self.ltype
+        result += '%s, ' % self.allocation
-        result += '),\n'
+        result += '%s, ' % self.struct_name
+        result += '%s, ' % self.name
+        result += '%s, ' % (prev_field_name or self.name)
         
-        if prev_field_name is None:
+        if self.pbtype == 'MESSAGE':
-            result += '    offsetof(%s, %s),' % (self.struct_name, self.name)
+            result += '&%s_fields)' % self.submsgname
+        elif self.default is None:
+            result += '0)'
+        elif self.pbtype in ['BYTES', 'STRING'] and self.allocation != 'STATIC':
+            result += '0)' # Arbitrary size default values not implemented
         else:
-            result += '    pb_delta_end(%s, %s, %s),' % (self.struct_name, self.name, prev_field_name)
+            result += '&%s_default)' % (self.struct_name + self.name)
-        
-        if self.htype == 'PB_HTYPE_OPTIONAL':
-            result += '\n    pb_delta(%s, has_%s, %s),' % (self.struct_name, self.name, self.name)
-        elif self.htype == 'PB_HTYPE_ARRAY':
-            result += '\n    pb_delta(%s, %s_count, %s),' % (self.struct_name, self.name, self.name)
-        else:
-            result += ' 0,'
-        
-        
-        if self.htype == 'PB_HTYPE_ARRAY':
-            result += '\n    pb_membersize(%s, %s[0]),' % (self.struct_name, self.name)
-            result += ('\n    pb_membersize(%s, %s) / pb_membersize(%s, %s[0]),'
-                       % (self.struct_name, self.name, self.struct_name, self.name))
-        else:
-            result += '\n    pb_membersize(%s, %s),' % (self.struct_name, self.name)
-            result += ' 0,'
-        
-        if self.ltype == 'PB_LTYPE_SUBMESSAGE':
-            result += '\n    &%s_fields}' % self.submsgname
-        elif self.default is None or self.htype == 'PB_HTYPE_CALLBACK':
-            result += ' 0}'
-        else:
-            result += '\n    &%s_default}' % (self.struct_name + self.name)
         
         return result
     
     def largest_field_value(self):
         '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly.
         Returns numeric value or a C-expression for assert.'''
-        if self.ltype == 'PB_LTYPE_SUBMESSAGE':
+        if self.pbtype == 'MESSAGE':
-            if self.htype == 'PB_HTYPE_ARRAY':
+            if self.rules == 'REPEATED' and self.allocation == 'STATIC':
                 return 'pb_membersize(%s, %s[0])' % (self.struct_name, self.name)
             else:
                 return 'pb_membersize(%s, %s)' % (self.struct_name, self.name)
@@ -264,10 +271,26 @@ class Field:
         return max(self.tag, self.max_size, self.max_count)        
 
 
+
+
+
+
+# ---------------------------------------------------------------------------
+#                   Generation of messages (structures)
+# ---------------------------------------------------------------------------
+
+
 class Message:
-    def __init__(self, names, desc):
+    def __init__(self, names, desc, message_options):
         self.name = names
-        self.fields = [Field(self.name, f) for f in desc.field]
+        self.fields = []
+        
+        for f in desc.field:
+            field_options = get_nanopb_suboptions(f, message_options)
+            if field_options.type != nanopb_pb2.FT_IGNORE:
+                self.fields.append(Field(self.name, f, field_options))
+        
+        self.packed = message_options.packed_struct
         self.ordered_fields = self.fields[:]
         self.ordered_fields.sort()
 
@@ -276,9 +299,14 @@ class Message:
         return [str(field.ctype) for field in self.fields]
     
     def __str__(self):
-        result = 'typedef struct {\n'
+        result = 'typedef struct _%s {\n' % self.name
         result += '\n'.join([str(f) for f in self.ordered_fields])
-        result += '\n} %s;' % self.name
+        result += '\n}'
+        
+        if self.packed:
+            result += ' pb_packed'
+        
+        result += ' %s;' % self.name
         return result
     
     def types(self):
@@ -307,12 +335,22 @@ class Message:
         prev = None
         for field in self.ordered_fields:
             result += field.pb_field_t(prev)
-            result += ',\n\n'
+            result += ',\n'
             prev = field.name
         
         result += '    PB_LAST_FIELD\n};'
         return result
 
+
+
+
+
+
+# ---------------------------------------------------------------------------
+#                    Processing of entire .proto files
+# ---------------------------------------------------------------------------
+
+
 def iterate_messages(desc, names = Names()):
     '''Recursively find all messages. For each, yield name, DescriptorProto.'''
     if hasattr(desc, 'message_type'):
@@ -327,7 +365,7 @@ def iterate_messages(desc, names = Names()):
         for x in iterate_messages(submsg, sub_names):
             yield x
 
-def parse_file(fdesc):
+def parse_file(fdesc, file_options):
     '''Takes a FileDescriptorProto and returns tuple (enum, messages).'''
     
     enums = []
@@ -339,12 +377,24 @@ def parse_file(fdesc):
         base_name = Names()
     
     for enum in fdesc.enum_type:
-        enums.append(Enum(base_name, enum))
+        enum_options = get_nanopb_suboptions(enum, file_options)
+        enums.append(Enum(base_name, enum, enum_options))
     
     for names, message in iterate_messages(fdesc, base_name):
-        messages.append(Message(names, message))
+        message_options = get_nanopb_suboptions(message, file_options)
+        messages.append(Message(names, message, message_options))
         for enum in message.enum_type:
-            enums.append(Enum(names, enum))
+            enum_options = get_nanopb_suboptions(enum, message_options)
+            enums.append(Enum(names, enum, enum_options))
+    
+    # Fix field default values where enum short names are used.
+    for enum in enums:
+        if not enum.options.long_names:
+            for message in messages:
+                for field in message.fields:
+                    if field.default in enum.value_longnames:
+                        idx = enum.value_longnames.index(field.default)
+                        field.default = enum.values[idx][0]
     
     return enums, messages
 
@@ -379,22 +429,36 @@ def sort_dependencies(messages):
         if msgname in message_by_name:
             yield message_by_name[msgname]
 
-def generate_header(dependencies, headername, enums, messages):
+def make_identifier(headername):
+    '''Make #ifndef identifier that contains uppercase A-Z and digits 0-9'''
+    result = ""
+    for c in headername.upper():
+        if c.isalnum():
+            result += c
+        else:
+            result += '_'
+    return result
+
+def generate_header(dependencies, headername, enums, messages, options):
     '''Generate content for a header file.
     Generates strings, which should be concatenated and stored to file.
     '''
     
     yield '/* Automatically generated nanopb header */\n'
+    yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
     
-    symbol = headername.replace('.', '_').upper()
+    symbol = make_identifier(headername)
     yield '#ifndef _PB_%s_\n' % symbol
     yield '#define _PB_%s_\n' % symbol
     yield '#include <pb.h>\n\n'
     
     for dependency in dependencies:
         noext = os.path.splitext(dependency)[0]
-        yield '#include "%s.pb.h"\n' % noext
+        yield '#include "%s.%s.h"\n' % (noext,options.extension)
-    yield '\n'
+    
+    yield '#ifdef __cplusplus\n'
+    yield 'extern "C" {\n'
+    yield '#endif\n\n'
     
     yield '/* Enum definitions */\n'
     for enum in enums:
@@ -414,8 +478,30 @@ def generate_header(dependencies, headername, enums, messages):
     for msg in messages:
         yield msg.fields_declaration() + '\n'
     
+    yield '\n#ifdef __cplusplus\n'
+    yield '} /* extern "C" */\n'
+    yield '#endif\n'
+    
+    # End of header
+    yield '\n#endif\n'
+
+def generate_source(headername, enums, messages):
+    '''Generate content for a source file.'''
+    
+    yield '/* Automatically generated nanopb constant definitions */\n'
+    yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
+    yield '#include "%s"\n\n' % headername
+    
+    for msg in messages:
+        yield msg.default_decl(False)
+    
+    yield '\n\n'
+    
+    for msg in messages:
+        yield msg.fields_definition() + '\n\n'
+        
     if messages:
-        count_required_fields = lambda m: len([f for f in msg.fields if f.htype == 'PB_HTYPE_REQUIRED'])
+        count_required_fields = lambda m: len([f for f in msg.fields if f.rules == 'REQUIRED'])
         largest_msg = max(messages, key = count_required_fields)
         largest_count = count_required_fields(largest_msg)
         if largest_count > 64:
@@ -425,10 +511,13 @@ def generate_header(dependencies, headername, enums, messages):
             yield '       setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count
             yield '#endif\n'
     
+    # Add checks for numeric limits
     worst = 0
     worst_field = ''
     checks = []
+    checks_msgnames = []
     for msg in messages:
+        checks_msgnames.append(msg.name)
         for field in msg.fields:
             status = field.largest_field_value()
             if isinstance(status, (str, unicode)):
@@ -446,7 +535,8 @@ def generate_header(dependencies, headername, enums, messages):
                 yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field
             else:
                 assertion = ' && '.join(str(c) + ' < 256' for c in checks)
-                yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT)\n' % assertion
+                msgs = '_'.join(str(n) for n in checks_msgnames)
+                yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
             yield '#endif\n\n'
         
         if worst > 65535 or checks:
@@ -455,59 +545,127 @@ def generate_header(dependencies, headername, enums, messages):
                 yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field
             else:
                 assertion = ' && '.join(str(c) + ' < 65536' for c in checks)
-                yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT)\n' % assertion
+                msgs = '_'.join(str(n) for n in checks_msgnames)
+                yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
             yield '#endif\n'
     
-    # End of header
+    # Add check for sizeof(double)
-    yield '\n#endif\n'
+    has_double = False
-
-def generate_source(headername, enums, messages):
-    '''Generate content for a source file.'''
-    
-    yield '/* Automatically generated nanopb constant definitions */\n'
-    yield '#include "%s"\n\n' % headername
-    
     for msg in messages:
-        yield msg.default_decl(False)
+        for field in msg.fields:
+            if field.ctype == 'double':
+                has_double = True
     
-    yield '\n\n'
+    if has_double:
+        yield '\n'
+        yield '/* On some platforms (such as AVR), double is really float.\n'
+        yield ' * These are not directly supported by nanopb, but see example_avr_double.\n'
+        yield ' * To get rid of this error, remove any double fields from your .proto.\n'
+        yield ' */\n'
+        yield 'STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n'
     
-    for msg in messages:
+    yield '\n'
-        yield msg.fields_definition() + '\n\n'
+
+
+# ---------------------------------------------------------------------------
+#                         Command line interface
+# ---------------------------------------------------------------------------
 
-if __name__ == '__main__':
 import sys
 import os.path    
+from optparse import OptionParser
+import google.protobuf.text_format as text_format
 
-    if len(sys.argv) != 2:
+optparser = OptionParser(
-        print "Usage: " + sys.argv[0] + " file.pb"
+    usage = "Usage: nanopb_generator.py [options] file.pb ...",
-        print "where file.pb has been compiled from .proto by:"
+    epilog = "Compile file.pb from file.proto by: 'protoc -ofile.pb file.proto'. " +
-        print "protoc -ofile.pb file.proto"
+             "Output will be written to file.pb.h and file.pb.c.")
-        print "Output fill be written to file.pb.h and file.pb.c"
+optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", default=[],
-        sys.exit(1)
+    help="Exclude file from generated #include list.")
+optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default="pb",
+    help="use extension instead of 'pb' for generated files.")
+optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False,
+    help="Don't print anything except errors.")
+optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
+    help="Print more information.")
+optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[],
+    help="Set generator option (max_size, max_count etc.).")
 
-    data = open(sys.argv[1], 'rb').read()
+def get_nanopb_suboptions(subdesc, options):
+    '''Get copy of options, and merge information from subdesc.'''
+    new_options = nanopb_pb2.NanoPBOptions()
+    new_options.CopyFrom(options)
+    
+    if isinstance(subdesc.options, descriptor.FieldOptions):
+        ext_type = nanopb_pb2.nanopb
+    elif isinstance(subdesc.options, descriptor.FileOptions):
+        ext_type = nanopb_pb2.nanopb_fileopt
+    elif isinstance(subdesc.options, descriptor.MessageOptions):
+        ext_type = nanopb_pb2.nanopb_msgopt
+    elif isinstance(subdesc.options, descriptor.EnumOptions):
+        ext_type = nanopb_pb2.nanopb_enumopt
+    else:
+        raise Exception("Unknown options type")
+    
+    if subdesc.options.HasExtension(ext_type):
+        ext = subdesc.options.Extensions[ext_type]
+        new_options.MergeFrom(ext)
+    
+    return new_options
+
+def process(filenames, options):
+    '''Process the files given on the command line.'''
+    
+    if not filenames:
+        optparser.print_help()
+        return False
+    
+    if options.quiet:
+        options.verbose = False
+    
+    toplevel_options = nanopb_pb2.NanoPBOptions()
+    for s in options.settings:
+        text_format.Merge(s, toplevel_options)
+    
+    for filename in filenames:
+        data = open(filename, 'rb').read()
         fdesc = descriptor.FileDescriptorSet.FromString(data)
-    enums, messages = parse_file(fdesc.file[0])
         
-    noext = os.path.splitext(sys.argv[1])[0]
+        file_options = get_nanopb_suboptions(fdesc.file[0], toplevel_options)
-    headername = noext + '.pb.h'
+        
-    sourcename = noext + '.pb.c'
+        if options.verbose:
+            print "Options for " + filename + ":"
+            print text_format.MessageToString(file_options)
+        
+        enums, messages = parse_file(fdesc.file[0], file_options)
+        
+        noext = os.path.splitext(filename)[0]
+        headername = noext + '.' + options.extension + '.h'
+        sourcename = noext + '.' + options.extension + '.c'
         headerbasename = os.path.basename(headername)
         
+        if not options.quiet:
             print "Writing to " + headername + " and " + sourcename
         
         # List of .proto files that should not be included in the C header file
         # even if they are mentioned in the source .proto.
-    excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto']
+        excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude
         dependencies = [d for d in fdesc.file[0].dependency if d not in excludes]
         
         header = open(headername, 'w')
-    for part in generate_header(dependencies, headerbasename, enums, messages):
+        for part in generate_header(dependencies, headerbasename, enums, messages, options):
             header.write(part)
 
         source = open(sourcename, 'w')
         for part in generate_source(headerbasename, enums, messages):
             source.write(part)
 
+    return True
+
+if __name__ == '__main__':
+    options, filenames = optparser.parse_args()
+    status = process(filenames, options)
+    
+    if not status:
+        sys.exit(1)
     

generator/nanopb_pb2.py

@@ -7,15 +7,75 @@ from google.protobuf import descriptor_pb2
 # @@protoc_insertion_point(imports)
 
 
+import google.protobuf.descriptor_pb2
+
 DESCRIPTOR = descriptor.FileDescriptor(
   name='nanopb.proto',
   package='',
-  serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"4\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions')
+  serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*J\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions')
+
+_FIELDTYPE = descriptor.EnumDescriptor(
+  name='FieldType',
+  full_name='FieldType',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    descriptor.EnumValueDescriptor(
+      name='FT_DEFAULT', index=0, number=0,
+      options=None,
+      type=None),
+    descriptor.EnumValueDescriptor(
+      name='FT_CALLBACK', index=1, number=1,
+      options=None,
+      type=None),
+    descriptor.EnumValueDescriptor(
+      name='FT_STATIC', index=2, number=2,
+      options=None,
+      type=None),
+    descriptor.EnumValueDescriptor(
+      name='FT_IGNORE', index=3, number=3,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=199,
+  serialized_end=273,
+)
 
 
+FT_DEFAULT = 0
+FT_CALLBACK = 1
+FT_STATIC = 2
+FT_IGNORE = 3
+
+NANOPB_FILEOPT_FIELD_NUMBER = 1010
+nanopb_fileopt = descriptor.FieldDescriptor(
+  name='nanopb_fileopt', full_name='nanopb_fileopt', index=0,
+  number=1010, type=11, cpp_type=10, label=1,
+  has_default_value=False, default_value=None,
+  message_type=None, enum_type=None, containing_type=None,
+  is_extension=True, extension_scope=None,
+  options=None)
+NANOPB_MSGOPT_FIELD_NUMBER = 1010
+nanopb_msgopt = descriptor.FieldDescriptor(
+  name='nanopb_msgopt', full_name='nanopb_msgopt', index=1,
+  number=1010, type=11, cpp_type=10, label=1,
+  has_default_value=False, default_value=None,
+  message_type=None, enum_type=None, containing_type=None,
+  is_extension=True, extension_scope=None,
+  options=None)
+NANOPB_ENUMOPT_FIELD_NUMBER = 1010
+nanopb_enumopt = descriptor.FieldDescriptor(
+  name='nanopb_enumopt', full_name='nanopb_enumopt', index=2,
+  number=1010, type=11, cpp_type=10, label=1,
+  has_default_value=False, default_value=None,
+  message_type=None, enum_type=None, containing_type=None,
+  is_extension=True, extension_scope=None,
+  options=None)
 NANOPB_FIELD_NUMBER = 1010
 nanopb = descriptor.FieldDescriptor(
-  name='nanopb', full_name='nanopb', index=0,
+  name='nanopb', full_name='nanopb', index=3,
   number=1010, type=11, cpp_type=10, label=1,
   has_default_value=False, default_value=None,
   message_type=None, enum_type=None, containing_type=None,
@@ -44,6 +104,27 @@ _NANOPBOPTIONS = descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
+    descriptor.FieldDescriptor(
+      name='type', full_name='NanoPBOptions.type', index=2,
+      number=3, type=14, cpp_type=8, label=1,
+      has_default_value=True, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    descriptor.FieldDescriptor(
+      name='long_names', full_name='NanoPBOptions.long_names', index=3,
+      number=4, type=8, cpp_type=7, label=1,
+      has_default_value=True, default_value=True,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    descriptor.FieldDescriptor(
+      name='packed_struct', full_name='NanoPBOptions.packed_struct', index=4,
+      number=5, type=8, cpp_type=7, label=1,
+      has_default_value=True, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
   ],
   extensions=[
   ],
@@ -53,12 +134,12 @@ _NANOPBOPTIONS = descriptor.Descriptor(
   options=None,
   is_extendable=False,
   extension_ranges=[],
-  serialized_start=50,
+  serialized_start=51,
-  serialized_end=102,
+  serialized_end=197,
 )
 
-import google.protobuf.descriptor_pb2
+_NANOPBOPTIONS.fields_by_name['type'].enum_type = _FIELDTYPE
-
+DESCRIPTOR.message_types_by_name['NanoPBOptions'] = _NANOPBOPTIONS
 
 class NanoPBOptions(message.Message):
   __metaclass__ = reflection.GeneratedProtocolMessageType
@@ -66,6 +147,12 @@ class NanoPBOptions(message.Message):
   
   # @@protoc_insertion_point(class_scope:NanoPBOptions)
 
+nanopb_fileopt.message_type = _NANOPBOPTIONS
+google.protobuf.descriptor_pb2.FileOptions.RegisterExtension(nanopb_fileopt)
+nanopb_msgopt.message_type = _NANOPBOPTIONS
+google.protobuf.descriptor_pb2.MessageOptions.RegisterExtension(nanopb_msgopt)
+nanopb_enumopt.message_type = _NANOPBOPTIONS
+google.protobuf.descriptor_pb2.EnumOptions.RegisterExtension(nanopb_enumopt)
 nanopb.message_type = _NANOPBOPTIONS
 google.protobuf.descriptor_pb2.FieldOptions.RegisterExtension(nanopb)
 # @@protoc_insertion_point(module_scope)

pb.h

@@ -6,6 +6,8 @@
  * see pb_encode.h or pb_decode.h
  */
 
+#define NANOPB_VERSION nanopb-0.2.0
+
 #include <stdint.h>
 #include <stddef.h>
 #include <stdbool.h>
@@ -22,9 +24,13 @@
 #define UNUSED(x) (void)(x)
 #endif
 
-/* Compile-time assertion, used for checking compatible compilation options. */
+/* Compile-time assertion, used for checking compatible compilation options.
+ * If this fails on your compiler for some reason, use #define STATIC_ASSERT
+ * to disable it. */
 #ifndef STATIC_ASSERT
-#define STATIC_ASSERT(COND,MSG) typedef char static_assertion_##MSG[(COND)?1:-1];
+#define STATIC_ASSERT(COND,MSG) typedef char STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND)?1:-1];
+#define STATIC_ASSERT_MSG(MSG, LINE, COUNTER) STATIC_ASSERT_MSG_(MSG, LINE, COUNTER)
+#define STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) static_assertion_##MSG##LINE##COUNTER
 #endif
 
 /* Number of required fields to keep track of
@@ -47,59 +53,57 @@
  * SINT* is different, though, because it is zig-zag coded.
  */
 
-typedef enum {
+typedef uint8_t pb_type_t;
+
 /************************
  * Field contents types *
  ************************/
 
 /* Numeric types */
-    PB_LTYPE_VARINT = 0x00, /* int32, uint32, int64, uint64, bool, enum */
+#define PB_LTYPE_VARINT 0x00 /* int32, uint32, int64, uint64, bool, enum */
-    PB_LTYPE_SVARINT = 0x01, /* sint32, sint64 */
+#define PB_LTYPE_SVARINT 0x01 /* sint32, sint64 */
-    PB_LTYPE_FIXED32 = 0x02, /* fixed32, sfixed32, float */
+#define PB_LTYPE_FIXED32 0x02 /* fixed32, sfixed32, float */
-    PB_LTYPE_FIXED64 = 0x03, /* fixed64, sfixed64, double */
+#define PB_LTYPE_FIXED64 0x03 /* fixed64, sfixed64, double */
 
 /* Marker for last packable field type. */
-    PB_LTYPE_LAST_PACKABLE = 0x03,
+#define PB_LTYPE_LAST_PACKABLE 0x03
 
 /* Byte array with pre-allocated buffer.
  * data_size is the length of the allocated PB_BYTES_ARRAY structure. */
-    PB_LTYPE_BYTES = 0x04,
+#define PB_LTYPE_BYTES 0x04
 
 /* String with pre-allocated buffer.
  * data_size is the maximum length. */
-    PB_LTYPE_STRING = 0x05,
+#define PB_LTYPE_STRING 0x05
 
 /* Submessage
  * submsg_fields is pointer to field descriptions */
-    PB_LTYPE_SUBMESSAGE = 0x06,
+#define PB_LTYPE_SUBMESSAGE 0x06
 
 /* Number of declared LTYPES */
-    PB_LTYPES_COUNT = 7,
+#define PB_LTYPES_COUNT 7
+#define PB_LTYPE_MASK 0x0F
 
-    /******************
+/**************************
-     * Modifier flags *
+ * Field repetition rules *
-     ******************/
+ **************************/
 
-    /* Just the basic, write data at data_offset */
+#define PB_HTYPE_REQUIRED 0x00
-    PB_HTYPE_REQUIRED = 0x00,
+#define PB_HTYPE_OPTIONAL 0x10
+#define PB_HTYPE_REPEATED 0x20
+#define PB_HTYPE_MASK     0x30
 
-    /* Write true at size_offset */
+/********************
-    PB_HTYPE_OPTIONAL = 0x10,
+ * Allocation types *
+ ********************/
  
-    /* Read to pre-allocated array
+#define PB_ATYPE_STATIC   0x00
-     * Maximum number of entries is array_size,
+#define PB_ATYPE_CALLBACK 0x40
-     * actual number is stored at size_offset */
+#define PB_ATYPE_MASK     0xC0
-    PB_HTYPE_ARRAY = 0x20,
 
-    /* Works for all required/optional/repeated fields.
+#define PB_ATYPE(x) ((x) & PB_ATYPE_MASK)
-     * data_offset points to pb_callback_t structure.
+#define PB_HTYPE(x) ((x) & PB_HTYPE_MASK)
-     * LTYPE should be 0 (it is ignored, but sometimes
+#define PB_LTYPE(x) ((x) & PB_LTYPE_MASK)
-     * used to speculatively index an array). */
-    PB_HTYPE_CALLBACK = 0x30
-} pb_packed pb_type_t;
-
-#define PB_HTYPE(x) ((x) & 0xF0)
-#define PB_LTYPE(x) ((x) & 0x0F)
 
 /* This structure is used in auto-generated constants
  * to specify struct fields.
@@ -145,10 +149,12 @@ struct _pb_field_t {
  * It has the number of bytes in the beginning, and after that an array.
  * Note that actual structs used will have a different length of bytes array.
  */
-typedef struct {
+struct _pb_bytes_array_t {
     size_t size;
     uint8_t bytes[1];
-} pb_bytes_array_t;
+};
+
+typedef struct _pb_bytes_array_t pb_bytes_array_t;
 
 /* This structure is used for giving the callback function.
  * It is stored in the message structure and filled in by the method that
@@ -193,9 +199,84 @@ typedef enum {
 #define pb_membersize(st, m) (sizeof ((st*)0)->m)
 #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0]))
 #define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2))
-#define pb_delta_end(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2))
+#define pb_delta_end(st, m1, m2) (offsetof(st, m1) == offsetof(st, m2) \
+                                  ? offsetof(st, m1) \
+                                  : offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2))
 #define PB_LAST_FIELD {0,(pb_type_t) 0,0,0,0,0,0}
 
+/* Required fields are the simplest. They just have delta (padding) from
+ * previous field end, and the size of the field. Pointer is used for
+ * submessages and default values.
+ */
+#define PB_REQUIRED_STATIC(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | ltype, \
+    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+
+/* Optional fields add the delta to the has_ variable. */
+#define PB_OPTIONAL_STATIC(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \
+    pb_delta_end(st, m, pm), \
+    pb_delta(st, has_ ## m, m), \
+    pb_membersize(st, m), 0, ptr}
+
+/* Repeated fields have a _count field and also the maximum number of entries. */
+#define PB_REPEATED_STATIC(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | ltype, \
+    pb_delta_end(st, m, pm), \
+    pb_delta(st, m ## _count, m), \
+    pb_membersize(st, m[0]), \
+    pb_arraysize(st, m), ptr}
+
+/* Callbacks are much like required fields except with special datatype. */
+#define PB_REQUIRED_CALLBACK(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REQUIRED | ltype, \
+    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+
+#define PB_OPTIONAL_CALLBACK(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \
+    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+    
+#define PB_REPEATED_CALLBACK(tag, st, m, pm, ltype, ptr) \
+    {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REPEATED | ltype, \
+    pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr}
+
+/* The mapping from protobuf types to LTYPEs is done using these macros. */
+#define PB_LTYPE_MAP_BOOL       PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_BYTES      PB_LTYPE_BYTES
+#define PB_LTYPE_MAP_DOUBLE     PB_LTYPE_FIXED64
+#define PB_LTYPE_MAP_ENUM       PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_FIXED32    PB_LTYPE_FIXED32
+#define PB_LTYPE_MAP_FIXED64    PB_LTYPE_FIXED64
+#define PB_LTYPE_MAP_FLOAT      PB_LTYPE_FIXED32
+#define PB_LTYPE_MAP_INT32      PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_INT64      PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_MESSAGE    PB_LTYPE_SUBMESSAGE
+#define PB_LTYPE_MAP_SFIXED32   PB_LTYPE_FIXED32
+#define PB_LTYPE_MAP_SFIXED64   PB_LTYPE_FIXED64
+#define PB_LTYPE_MAP_SINT32     PB_LTYPE_SVARINT
+#define PB_LTYPE_MAP_SINT64     PB_LTYPE_SVARINT
+#define PB_LTYPE_MAP_STRING     PB_LTYPE_STRING
+#define PB_LTYPE_MAP_UINT32     PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_UINT64     PB_LTYPE_VARINT
+
+/* This is the actual macro used in field descriptions.
+ * It takes these arguments:
+ * - Field tag number
+ * - Field type:   BOOL, BYTES, DOUBLE, ENUM, FIXED32, FIXED64,
+ *                 FLOAT, INT32, INT64, MESSAGE, SFIXED32, SFIXED64
+ *                 SINT32, SINT64, STRING, UINT32 or UINT64
+ * - Field rules:  REQUIRED, OPTIONAL or REPEATED
+ * - Allocation:   STATIC or CALLBACK
+ * - Message name
+ * - Field name
+ * - Previous field name (or field name again for first field)
+ * - Pointer to default value or submsg fields.
+ */
+
+#define PB_FIELD(tag, type, rules, allocation, message, field, prevfield, ptr) \
+    PB_ ## rules ## _ ## allocation(tag, message, field, prevfield, \
+                                    PB_LTYPE_MAP_ ## type, ptr)
+
 /* These macros are used for giving out error messages.
  * They are mostly a debugging aid; the main error information
  * is the true/false return value from functions.

pb_decode.c

@@ -36,35 +36,67 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = {
  * pb_istream *
  **************/
 
-bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count)
-{
-    if (stream->bytes_left < count)
-        PB_RETURN_ERROR(stream, "end-of-stream");
-    
-    if (!stream->callback(stream, buf, count))
-        PB_RETURN_ERROR(stream, "io error");
-    
-    stream->bytes_left -= count;
-    return true;
-}
-
 static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count)
 {
     uint8_t *source = (uint8_t*)stream->state;
+    stream->state = source + count;
     
     if (buf != NULL)
-        memcpy(buf, source, count);
+    {
+        while (count--)
+            *buf++ = *source++;
+    }
     
-    stream->state = source + count;
+    return true;
+}
+
+bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count)
+{
+#ifndef PB_BUFFER_ONLY
+	if (buf == NULL && stream->callback != buf_read)
+	{
+		/* Skip input bytes */
+		uint8_t tmp[16];
+		while (count > 16)
+		{
+			if (!pb_read(stream, tmp, 16))
+				return false;
+			
+			count -= 16;
+		}
+		
+		return pb_read(stream, tmp, count);
+	}
+#endif
+
+    if (stream->bytes_left < count)
+        PB_RETURN_ERROR(stream, "end-of-stream");
+    
+#ifndef PB_BUFFER_ONLY
+    if (!stream->callback(stream, buf, count))
+        PB_RETURN_ERROR(stream, "io error");
+#else
+    if (!buf_read(stream, buf, count))
+        return false;
+#endif
+    
+    stream->bytes_left -= count;
     return true;
 }
 
 pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize)
 {
     pb_istream_t stream;
+#ifdef PB_BUFFER_ONLY
+    stream.callback = NULL;
+#else
     stream.callback = &buf_read;
+#endif
     stream.state = buf;
     stream.bytes_left = bufsize;
+#ifndef PB_NO_ERRMSG
+    stream.errmsg = NULL;
+#endif
     return stream;
 }
 
@@ -74,28 +106,60 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize)
 
 static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest)
 {
-    uint64_t temp;
+    uint8_t byte;
-    bool status = pb_decode_varint(stream, &temp);
+    uint32_t result;
-    *dest = (uint32_t)temp;
+    
-    return status;
+    if (!pb_read(stream, &byte, 1))
+        return false;
+    
+    if (!(byte & 0x80))
+    {
+        /* Quick case, 1 byte value */
+        result = byte;
+    }
+    else
+    {
+        /* Multibyte case */
+        uint8_t bitpos = 7;
+        result = byte & 0x7F;
+        
+        do
+        {
+            if (bitpos >= 32)
+                PB_RETURN_ERROR(stream, "varint overflow");
+            
+            if (!pb_read(stream, &byte, 1))
+                return false;
+            
+            result |= (uint32_t)(byte & 0x7F) << bitpos;
+            bitpos = (uint8_t)(bitpos + 7);
+        } while (byte & 0x80);
+   }
+   
+   *dest = result;
+   return true;
 }
 
 bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest)
 {
     uint8_t byte;
     uint8_t bitpos = 0;
-    *dest = 0;
+    uint64_t result = 0;
     
-    while (bitpos < 64 && pb_read(stream, &byte, 1))
+    do
     {
-        *dest |= (uint64_t)(byte & 0x7F) << bitpos;
+        if (bitpos >= 64)
-        bitpos += 7;
-        
-        if (!(byte & 0x80))
-            return true;
-    }
-    
             PB_RETURN_ERROR(stream, "varint overflow");
+        
+        if (!pb_read(stream, &byte, 1))
+            return false;
+
+        result |= (uint64_t)(byte & 0x7F) << bitpos;
+        bitpos = (uint8_t)(bitpos + 7);
+    } while (byte & 0x80);
+    
+    *dest = result;
+    return true;
 }
 
 bool checkreturn pb_skip_varint(pb_istream_t *stream)
@@ -207,14 +271,18 @@ bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *su
 void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream)
 {
     stream->state = substream->state;
+
+#ifndef PB_NO_ERRMSG
+    stream->errmsg = substream->errmsg;
+#endif
 }
 
 /* Iterator for pb_field_t list */
 typedef struct {
     const pb_field_t *start; /* Start of the pb_field_t array */
     const pb_field_t *current; /* Current position of the iterator */
-    int field_index; /* Zero-based index of the field. */
+    unsigned field_index; /* Zero-based index of the field. */
-    int required_field_index; /* Zero-based index that counts only the required fields */
+    unsigned required_field_index; /* Zero-based index that counts only the required fields */
     void *dest_struct; /* Pointer to the destination structure to decode to */
     void *pData; /* Pointer where to store current field value */
     void *pSize; /* Pointer where to store the size of current array field */
@@ -235,8 +303,11 @@ static bool pb_field_next(pb_field_iterator_t *iter)
     bool notwrapped = true;
     size_t prev_size = iter->current->data_size;
     
-    if (PB_HTYPE(iter->current->type) == PB_HTYPE_ARRAY)
+    if (PB_ATYPE(iter->current->type) == PB_ATYPE_STATIC &&
+        PB_HTYPE(iter->current->type) == PB_HTYPE_REPEATED)
+    {
         prev_size *= iter->current->array_size;
+    }
     
     if (PB_HTYPE(iter->current->type) == PB_HTYPE_REQUIRED)
         iter->required_field_index++;
@@ -260,7 +331,7 @@ static bool pb_field_next(pb_field_iterator_t *iter)
 
 static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag)
 {
-    int start = iter->field_index;
+    unsigned start = iter->field_index;
     
     do {
         if (iter->current->tag == tag)
@@ -275,11 +346,15 @@ static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag)
  * Decode a single field *
  *************************/
 
-static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
+static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
 {
-    pb_decoder_t func = PB_DECODERS[PB_LTYPE(iter->current->type)];
+    pb_type_t type;
+    pb_decoder_t func;
     
-    switch (PB_HTYPE(iter->current->type))
+    type = iter->current->type;
+    func = PB_DECODERS[PB_LTYPE(type)];
+
+    switch (PB_HTYPE(type))
     {
         case PB_HTYPE_REQUIRED:
             return func(stream, iter->current, iter->pData);
@@ -288,12 +363,12 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
             *(bool*)iter->pSize = true;
             return func(stream, iter->current, iter->pData);
     
-        case PB_HTYPE_ARRAY:
+        case PB_HTYPE_REPEATED:
             if (wire_type == PB_WT_STRING
-                && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE)
+                && PB_LTYPE(type) <= PB_LTYPE_LAST_PACKABLE)
             {
                 /* Packed array */
-                bool status;
+                bool status = true;
                 size_t *size = (size_t*)iter->pSize;
                 pb_istream_t substream;
                 if (!pb_make_string_substream(stream, &substream))
@@ -303,11 +378,17 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
                 {
                     void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size);
                     if (!func(&substream, iter->current, pItem))
-                        return false;
+                    {
+                        status = false;
+                        break;
+                    }
                     (*size)++;
                 }
-                status = (substream.bytes_left == 0);
                 pb_close_string_substream(stream, &substream);
+                
+                if (substream.bytes_left != 0)
+                    PB_RETURN_ERROR(stream, "array overflow");
+                
                 return status;
             }
             else
@@ -322,7 +403,12 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
                 return func(stream, iter->current, pItem);
             }
 
-        case PB_HTYPE_CALLBACK:
+        default:
+            PB_RETURN_ERROR(stream, "invalid field type");
+    }
+}
+
+static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
 {
     pb_callback_t *pCallback = (pb_callback_t*)iter->pData;
     
@@ -363,6 +449,16 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
     }
 }
 
+static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
+{
+    switch (PB_ATYPE(iter->current->type))
+    {
+        case PB_ATYPE_STATIC:
+            return decode_static_field(stream, wire_type, iter);
+        
+        case PB_ATYPE_CALLBACK:
+            return decode_callback_field(stream, wire_type, iter);
+        
         default:
             PB_RETURN_ERROR(stream, "invalid field type");
     }
@@ -377,26 +473,27 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str
     /* Initialize size/has fields and apply default values */
     do
     {
+        pb_type_t type;
+        type = iter.current->type;
+    
         if (iter.current->tag == 0)
             continue;
         
+        if (PB_ATYPE(type) == PB_ATYPE_STATIC)
+        {
             /* Initialize the size field for optional/repeated fields to 0. */
-        if (PB_HTYPE(iter.current->type) == PB_HTYPE_OPTIONAL)
+            if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL)
             {
                 *(bool*)iter.pSize = false;
             }
-        else if (PB_HTYPE(iter.current->type) == PB_HTYPE_ARRAY)
+            else if (PB_HTYPE(type) == PB_HTYPE_REPEATED)
             {
                 *(size_t*)iter.pSize = 0;
                 continue; /* Array is empty, no need to initialize contents */
             }
             
             /* Initialize field contents to default value */
-        if (PB_HTYPE(iter.current->type) == PB_HTYPE_CALLBACK)
+            if (PB_LTYPE(iter.current->type) == PB_LTYPE_SUBMESSAGE)
-        {
-            continue; /* Don't overwrite callback */
-        }
-        else if (PB_LTYPE(iter.current->type) == PB_LTYPE_SUBMESSAGE)
             {
                 pb_message_set_to_defaults((const pb_field_t *) iter.current->ptr, iter.pData);
             }
@@ -408,6 +505,11 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str
             {
                 memset(iter.pData, 0, iter.current->data_size);
             }
+        }
+        else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK)
+        {
+            continue; /* Don't overwrite callback */
+        }
     } while (pb_field_next(&iter));
 }
 
@@ -447,7 +549,7 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[
         if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED
             && iter.required_field_index < PB_MAX_REQUIRED_FIELDS)
         {
-            fields_seen[iter.required_field_index >> 3] |= 1 << (iter.required_field_index & 7);
+            fields_seen[iter.required_field_index >> 3] |= (uint8_t)(1 << (iter.required_field_index & 7));
         }
             
         if (!decode_field(stream, wire_type, &iter))
@@ -455,15 +557,34 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[
     }
     
     /* Check that all required fields were present. */
-    pb_field_init(&iter, fields, dest_struct);
-    do {
-        if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED &&
-            iter.required_field_index < PB_MAX_REQUIRED_FIELDS &&
-            !(fields_seen[iter.required_field_index >> 3] & (1 << (iter.required_field_index & 7))))
     {
+        /* First figure out the number of required fields by
+         * seeking to the end of the field array. Usually we
+         * are already close to end after decoding.
+         */
+        unsigned req_field_count;
+        pb_type_t last_type;
+        unsigned i;
+        do {
+            req_field_count = iter.required_field_index;
+            last_type = iter.current->type;
+        } while (pb_field_next(&iter));
+        
+        /* Fixup if last field was also required. */
+        if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED)
+            req_field_count++;
+        
+        /* Check the whole bytes */
+        for (i = 0; i < (req_field_count >> 3); i++)
+        {
+            if (fields_seen[i] != 0xFF)
+                PB_RETURN_ERROR(stream, "missing required field");
+        }
+        
+        /* Check the remaining bits */
+        if (fields_seen[req_field_count >> 3] != (0xFF >> (8 - (req_field_count & 7))))
             PB_RETURN_ERROR(stream, "missing required field");
     }
-    } while (pb_field_next(&iter));
     
     return true;
 }
@@ -483,9 +604,9 @@ bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest)
         return false;
     
     if (value & 1)
-        *dest = ~(value >> 1);
+        *dest = (int64_t)(~(value >> 1));
     else
-        *dest = value >> 1;
+        *dest = (int64_t)(value >> 1);
     
     return true;
 }
@@ -539,9 +660,9 @@ bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, vo
     
     switch (field->data_size)
     {
-        case 1: *(uint8_t*)dest = value; break;
+        case 1: *(uint8_t*)dest = (uint8_t)value; break;
-        case 2: *(uint16_t*)dest = value; break;
+        case 2: *(uint16_t*)dest = (uint16_t)value; break;
-        case 4: *(uint32_t*)dest = value; break;
+        case 4: *(uint32_t*)dest = (uint32_t)value; break;
         case 8: *(uint64_t*)dest = value; break;
         default: PB_RETURN_ERROR(stream, "invalid data_size");
     }
@@ -556,7 +677,7 @@ bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, v
     
     switch (field->data_size)
     {
-        case 4: *(int32_t*)dest = value; break;
+        case 4: *(int32_t*)dest = (int32_t)value; break;
         case 8: *(int64_t*)dest = value; break;
         default: PB_RETURN_ERROR(stream, "invalid data_size");
     }
@@ -622,7 +743,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field
     
     /* New array entries need to be initialized, while required and optional
      * submessages have already been initialized in the top-level pb_decode. */
-    if (PB_HTYPE(field->type) == PB_HTYPE_ARRAY)
+    if (PB_HTYPE(field->type) == PB_HTYPE_REPEATED)
         status = pb_decode(&substream, submsg_fields, dest);
     else
         status = pb_decode_noinit(&substream, submsg_fields, dest);

pb_decode.h

@@ -12,6 +12,10 @@
 #include <stdbool.h>
 #include "pb.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /* Lightweight input stream.
  * You can provide a callback function for reading or use
  * pb_istream_from_buffer.
@@ -19,18 +23,25 @@
  * Rules for callback:
  * 1) Return false on IO errors. This will cause decoding to abort.
  * 
- * 2) If buf is NULL, read but don't store bytes ("skip input").
+ * 2) You can use state to store your own data (e.g. buffer pointer),
- * 
- * 3) You can use state to store your own data (e.g. buffer pointer),
  * and rely on pb_read to verify that no-body reads past bytes_left.
  * 
- * 4) Your callback may be used with substreams, in which case bytes_left
+ * 3) Your callback may be used with substreams, in which case bytes_left
  * is different than from the main stream. Don't use bytes_left to compute
  * any pointers.
  */
 struct _pb_istream_t
 {
+#ifdef PB_BUFFER_ONLY
+    /* Callback pointer is not used in buffer-only configuration.
+     * Having an int pointer here allows binary compatibility but
+     * gives an error if someone tries to assign callback function.
+     */
+    int *callback;
+#else
     bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count);
+#endif
+
     void *state; /* Free field for use by callback implementation */
     size_t bytes_left;
     
@@ -104,4 +115,8 @@ bool pb_skip_varint(pb_istream_t *stream);
 bool pb_skip_string(pb_istream_t *stream);
 #endif
 
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
 #endif

pb_encode.c

@@ -37,15 +37,22 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = {
 static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count)
 {
     uint8_t *dest = (uint8_t*)stream->state;
-    memcpy(dest, buf, count);
     stream->state = dest + count;
+    
+    while (count--)
+        *dest++ = *buf++;
+    
     return true;
 }
 
 pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize)
 {
     pb_ostream_t stream;
+#ifdef PB_BUFFER_ONLY
+    stream.callback = (void*)1; /* Just some marker value */
+#else
     stream.callback = &buf_write;
+#endif
     stream.state = buf;
     stream.max_size = bufsize;
     stream.bytes_written = 0;
@@ -59,8 +66,13 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count
         if (stream->bytes_written + count > stream->max_size)
             return false;
 
+#ifdef PB_BUFFER_ONLY
+        if (!buf_write(stream, buf, count))
+            return false;
+#else        
         if (!stream->callback(stream, buf, count))
             return false;
+#endif
     }
     
     stream->bytes_written += count;
@@ -141,23 +153,14 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
     return true;
 }
 
-bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct)
+bool checkreturn encode_static_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData)
 {
-    const pb_field_t *field = fields;
+    pb_encoder_t func;
-    const void *pData = src_struct;
     const void *pSize;
-    size_t prev_size = 0;
     
-    while (field->tag != 0)
+    func = PB_ENCODERS[PB_LTYPE(field->type)];
-    {
-        pb_encoder_t func = PB_ENCODERS[PB_LTYPE(field->type)];
-        pData = (const char*)pData + prev_size + field->data_offset;
     pSize = (const char*)pData + field->size_offset;
     
-        prev_size = field->data_size;
-        if (PB_HTYPE(field->type) == PB_HTYPE_ARRAY)
-            prev_size *= field->array_size;
-                
     switch (PB_HTYPE(field->type))
     {
         case PB_HTYPE_REQUIRED:
@@ -178,12 +181,19 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons
             }
             break;
         
-            case PB_HTYPE_ARRAY:
+        case PB_HTYPE_REPEATED:
             if (!encode_array(stream, field, pData, *(const size_t*)pSize, func))
                 return false;
             break;
         
-            case PB_HTYPE_CALLBACK:
+        default:
+            return false;
+    }
+    
+    return true;
+}
+
+bool checkreturn encode_callback_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData)
 {
     const pb_callback_t *callback = (const pb_callback_t*)pData;
     if (callback->funcs.encode != NULL)
@@ -191,8 +201,41 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons
         if (!callback->funcs.encode(stream, field, callback->arg))
             return false;
     }
-                break;
+    return true;
 }
+
+bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct)
+{
+    const pb_field_t *field = fields;
+    const void *pData = src_struct;
+    size_t prev_size = 0;
+    
+    while (field->tag != 0)
+    {
+        pData = (const char*)pData + prev_size + field->data_offset;
+        prev_size = field->data_size;
+        
+        /* Special case for static arrays */
+        if (PB_ATYPE(field->type) == PB_ATYPE_STATIC &&
+            PB_HTYPE(field->type) == PB_HTYPE_REPEATED)
+        {
+            prev_size *= field->array_size;
+        }
+                
+        switch (PB_ATYPE(field->type))
+        {
+            case PB_ATYPE_STATIC:
+                if (!encode_static_field(stream, field, pData))
+                    return false;
+                break;
+            
+            case PB_ATYPE_CALLBACK:
+                if (!encode_callback_field(stream, field, pData))
+                    return false;
+                break;
+            
+            default:
+                return false;
         }
     
         field++;
@@ -205,7 +248,7 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons
 bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value)
 {
     uint8_t buffer[10];
-    int i = 0;
+    size_t i = 0;
     
     if (value == 0)
         return pb_write(stream, (uint8_t*)&value, 1);
@@ -225,9 +268,9 @@ bool checkreturn pb_encode_svarint(pb_ostream_t *stream, int64_t value)
 {
     uint64_t zigzagged;
     if (value < 0)
-        zigzagged = ~(value << 1);
+        zigzagged = (uint64_t)(~(value << 1));
     else
-        zigzagged = value << 1;
+        zigzagged = (uint64_t)(value << 1);
     
     return pb_encode_varint(stream, zigzagged);
 }
@@ -266,7 +309,7 @@ bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value)
     #endif
 }
 
-bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number)
+bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number)
 {
     uint64_t tag = wiretype | (field_number << 3);
     return pb_encode_varint(stream, tag);
@@ -370,7 +413,7 @@ bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, co
 
 bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
-    uint64_t value = 0;
+    int64_t value = 0;
     
     switch (field->data_size)
     {

pb_encode.h

@@ -9,6 +9,10 @@
 #include <stdbool.h>
 #include "pb.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /* Lightweight output stream.
  * You can provide callback for writing or use pb_ostream_from_buffer.
  * 
@@ -28,7 +32,17 @@
  */
 struct _pb_ostream_t
 {
+#ifdef PB_BUFFER_ONLY
+    /* Callback pointer is not used in buffer-only configuration.
+     * Having an int pointer here allows binary compatibility but
+     * gives an error if someone tries to assign callback function.
+     * Also, NULL pointer marks a 'sizing stream' that does not
+     * write anything.
+     */
+    int *callback;
+#else
     bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count);
+#endif
     void *state; /* Free field for use by callback implementation */
     size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */
     size_t bytes_written;
@@ -54,7 +68,7 @@ bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field);
 
 /* Encode field header by manually specifing wire type. You need to use this if
  * you want to write out packed arrays from a callback field. */
-bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number);
+bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number);
 
 /* Encode an integer in the varint format.
  * This works for bool, enum, int32, int64, uint32 and uint64 field types. */
@@ -99,5 +113,8 @@ bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *sr
  * instead, it has the same functionality with a less confusing interface. */
 bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src);
 
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
 
 #endif

tests/Makefile

@@ -1,15 +1,31 @@
-CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage
+CFLAGS=-ansi -Wall -Werror -I .. -g -O0
-LDFLAGS=--coverage
+DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h \
-DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h missing_fields.pb.h
+	callbacks2.pb.h callbacks.pb.h unittests.h unittestproto.pb.h \
-TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages
+	alltypes.pb.h missing_fields.pb.h
+TESTS=  decode_unittests encode_unittests \
+	test_decode1 test_decode2 test_decode3 test_decode3_buf \
+	test_encode1 test_encode2 test_encode3 test_encode3_buf \
+	test_decode_callbacks test_encode_callbacks \
+	test_missing_fields test_no_messages test_funny_name \
+	test_multiple_files test_cxxcompile test_options \
+	bc_encode bc_decode
 
 # More strict checks for the core part of nanopb
-CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op
+CC_VERSION=$(shell $(CC) -v 2>&1)
+CFLAGS_CORE=
+ifneq (,$(findstring gcc,$(CC_VERSION)))
+	CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op -Wconversion
+        CFLAGS+=--coverage
+        LDFLAGS+=--coverage
+endif
+ifneq (,$(findstring clang,$(CC_VERSION)))
+	CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wconversion
+endif
 
 all: breakpoints $(TESTS) run_unittests
 
 clean:
-	rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno
+	rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno *.pb.h *.pb.c
 
 %.pb.o: %.pb.c %.pb.h
 	$(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
@@ -23,11 +39,26 @@ pb_encode.o: ../pb_encode.c $(DEPS)
 pb_decode.o: ../pb_decode.c $(DEPS)
 	$(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
 
+# Test for compilability with c++ compiler
+
 pb_encode.cxx.o: ../pb_encode.c $(DEPS)
 	$(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
 pb_decode.cxx.o: ../pb_decode.c $(DEPS)
 	$(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
 
+# Test for PB_BUF_ONLY compilation option
+
+pb_encode.buf.o: ../pb_encode.c $(DEPS)
+	$(CC) -DPB_BUFFER_ONLY $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
+pb_decode.buf.o: ../pb_decode.c $(DEPS)
+	$(CC) -DPB_BUFFER_ONLY $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $<
+%.buf.o: %.c $(DEPS)
+	$(CC) -DPB_BUFFER_ONLY $(CFLAGS) -c -o $@ $<	
+test_encode3_buf: test_encode3.buf.o pb_encode.buf.o alltypes.pb.o
+	$(CC) $(LDFLAGS) $^ -o $@
+test_decode3_buf: test_decode3.buf.o pb_decode.buf.o alltypes.pb.o
+	$(CC) $(LDFLAGS) $^ -o $@
+
 test_cxxcompile: pb_encode.cxx.o pb_decode.cxx.o
 test_decode1: test_decode1.o pb_decode.o person.pb.o
 test_decode2: test_decode2.o pb_decode.o person.pb.o
@@ -35,12 +66,16 @@ test_decode3: test_decode3.o pb_decode.o alltypes.pb.o
 test_encode1: test_encode1.o pb_encode.o person.pb.o
 test_encode2: test_encode2.o pb_encode.o person.pb.o
 test_encode3: test_encode3.o pb_encode.o alltypes.pb.o
+test_multiple_files: test_multiple_files.o pb_encode.o callbacks2.pb.o callbacks.pb.o
 test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o
 test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o
 test_missing_fields: test_missing_fields.o pb_encode.o pb_decode.o missing_fields.pb.o
 decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o
 encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o
 test_no_messages: no_messages.pb.h no_messages.pb.c no_messages.pb.o
+test_funny_name: funny-proto+name.pb.h funny-proto+name.pb.o
+bc_encode: bc_alltypes.pb.o pb_encode.o bc_encode.o
+bc_decode: bc_alltypes.pb.o pb_decode.o bc_decode.o
 
 %.pb: %.proto
 	protoc -I. -I../generator -I/usr/include -o$@ $<
@@ -55,7 +90,7 @@ coverage: run_unittests
 	gcov pb_encode.gcda
 	gcov pb_decode.gcda
 
-run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields
+run_unittests: $(TESTS)
 	rm -f *.gcda
 	
 	./decode_unittests > /dev/null
@@ -70,14 +105,27 @@ run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 te
 	[ "`./test_encode2 | ./test_decode2`" = \
 	"`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ]
 	
+	[ "`./test_decode2 < person_with_extra_field.pb`" = \
+	"`./test_encode2 | ./test_decode2`" ]
+	
 	[ "`./test_encode_callbacks | ./test_decode_callbacks`" = \
 	"`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ]
 
 	./test_encode3 | ./test_decode3
 	./test_encode3 1 | ./test_decode3 1
 	./test_encode3 1 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null
+	./test_encode3_buf 1 | ./test_decode3_buf 1
+	./bc_encode | ./bc_decode
 	
 	./test_missing_fields
 
+test_options: options.pb.h options.expected options.pb.o
+	cat options.expected | while read -r p; do \
+	    if ! grep -q "$$p" $<; then \
+	        echo Expected: "$$p"; \
+	        exit 1; \
+	    fi \
+	done
+
 run_fuzztest: test_decode2
 	bash -c 'I=1; while true; do cat /dev/urandom | ./test_decode2 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done'

tests/bc_alltypes.pb.c

@@ -0,0 +1,93 @@
+/* Automatically generated nanopb constant definitions */
+/* Generated by 0.2.0-dev at Sun Feb 17 00:09:53 2013. */
+/* This is a file generated using nanopb-0.2.0-dev.
+ * It is used as a part of test suite in order to detect any
+ * incompatible changes made to the generator in future versions.
+ */
+
+#include "alltypes.pb.h"
+
+const char SubMessage_substuff1_default[16] = "1";
+const int32_t SubMessage_substuff2_default = 2;
+const uint32_t SubMessage_substuff3_default = 3;
+const int32_t AllTypes_opt_int32_default = 4041;
+const int64_t AllTypes_opt_int64_default = 4042;
+const uint32_t AllTypes_opt_uint32_default = 4043;
+const uint64_t AllTypes_opt_uint64_default = 4044;
+const int32_t AllTypes_opt_sint32_default = 4045;
+const int64_t AllTypes_opt_sint64_default = 4046;
+const bool AllTypes_opt_bool_default = false;
+const uint32_t AllTypes_opt_fixed32_default = 4048;
+const int32_t AllTypes_opt_sfixed32_default = 4049;
+const float AllTypes_opt_float_default = 4050;
+const uint64_t AllTypes_opt_fixed64_default = 4051;
+const int64_t AllTypes_opt_sfixed64_default = 4052;
+const double AllTypes_opt_double_default = 4053;
+const char AllTypes_opt_string_default[16] = "4054";
+const AllTypes_opt_bytes_t AllTypes_opt_bytes_default = {4, {0x34,0x30,0x35,0x35}};
+const MyEnum AllTypes_opt_enum_default = MyEnum_Second;
+
+
+const pb_field_t SubMessage_fields[4] = {
+    PB_FIELD(  1, STRING  , REQUIRED, STATIC, SubMessage, substuff1, substuff1, &SubMessage_substuff1_default),
+    PB_FIELD(  2, INT32   , REQUIRED, STATIC, SubMessage, substuff2, substuff1, &SubMessage_substuff2_default),
+    PB_FIELD(  3, FIXED32 , OPTIONAL, STATIC, SubMessage, substuff3, substuff2, &SubMessage_substuff3_default),
+    PB_LAST_FIELD
+};
+
+const pb_field_t AllTypes_fields[53] = {
+    PB_FIELD(  1, INT32   , REQUIRED, STATIC, AllTypes, req_int32, req_int32, 0),
+    PB_FIELD(  2, INT64   , REQUIRED, STATIC, AllTypes, req_int64, req_int32, 0),
+    PB_FIELD(  3, UINT32  , REQUIRED, STATIC, AllTypes, req_uint32, req_int64, 0),
+    PB_FIELD(  4, UINT64  , REQUIRED, STATIC, AllTypes, req_uint64, req_uint32, 0),
+    PB_FIELD(  5, SINT32  , REQUIRED, STATIC, AllTypes, req_sint32, req_uint64, 0),
+    PB_FIELD(  6, SINT64  , REQUIRED, STATIC, AllTypes, req_sint64, req_sint32, 0),
+    PB_FIELD(  7, BOOL    , REQUIRED, STATIC, AllTypes, req_bool, req_sint64, 0),
+    PB_FIELD(  8, FIXED32 , REQUIRED, STATIC, AllTypes, req_fixed32, req_bool, 0),
+    PB_FIELD(  9, SFIXED32, REQUIRED, STATIC, AllTypes, req_sfixed32, req_fixed32, 0),
+    PB_FIELD( 10, FLOAT   , REQUIRED, STATIC, AllTypes, req_float, req_sfixed32, 0),
+    PB_FIELD( 11, FIXED64 , REQUIRED, STATIC, AllTypes, req_fixed64, req_float, 0),
+    PB_FIELD( 12, SFIXED64, REQUIRED, STATIC, AllTypes, req_sfixed64, req_fixed64, 0),
+    PB_FIELD( 13, DOUBLE  , REQUIRED, STATIC, AllTypes, req_double, req_sfixed64, 0),
+    PB_FIELD( 14, STRING  , REQUIRED, STATIC, AllTypes, req_string, req_double, 0),
+    PB_FIELD( 15, BYTES   , REQUIRED, STATIC, AllTypes, req_bytes, req_string, 0),
+    PB_FIELD( 16, MESSAGE , REQUIRED, STATIC, AllTypes, req_submsg, req_bytes, &SubMessage_fields),
+    PB_FIELD( 17, ENUM    , REQUIRED, STATIC, AllTypes, req_enum, req_submsg, 0),
+    PB_FIELD( 21, INT32   , REPEATED, STATIC, AllTypes, rep_int32, req_enum, 0),
+    PB_FIELD( 22, INT64   , REPEATED, STATIC, AllTypes, rep_int64, rep_int32, 0),
+    PB_FIELD( 23, UINT32  , REPEATED, STATIC, AllTypes, rep_uint32, rep_int64, 0),
+    PB_FIELD( 24, UINT64  , REPEATED, STATIC, AllTypes, rep_uint64, rep_uint32, 0),
+    PB_FIELD( 25, SINT32  , REPEATED, STATIC, AllTypes, rep_sint32, rep_uint64, 0),
+    PB_FIELD( 26, SINT64  , REPEATED, STATIC, AllTypes, rep_sint64, rep_sint32, 0),
+    PB_FIELD( 27, BOOL    , REPEATED, STATIC, AllTypes, rep_bool, rep_sint64, 0),
+    PB_FIELD( 28, FIXED32 , REPEATED, STATIC, AllTypes, rep_fixed32, rep_bool, 0),
+    PB_FIELD( 29, SFIXED32, REPEATED, STATIC, AllTypes, rep_sfixed32, rep_fixed32, 0),
+    PB_FIELD( 30, FLOAT   , REPEATED, STATIC, AllTypes, rep_float, rep_sfixed32, 0),
+    PB_FIELD( 31, FIXED64 , REPEATED, STATIC, AllTypes, rep_fixed64, rep_float, 0),
+    PB_FIELD( 32, SFIXED64, REPEATED, STATIC, AllTypes, rep_sfixed64, rep_fixed64, 0),
+    PB_FIELD( 33, DOUBLE  , REPEATED, STATIC, AllTypes, rep_double, rep_sfixed64, 0),
+    PB_FIELD( 34, STRING  , REPEATED, STATIC, AllTypes, rep_string, rep_double, 0),
+    PB_FIELD( 35, BYTES   , REPEATED, STATIC, AllTypes, rep_bytes, rep_string, 0),
+    PB_FIELD( 36, MESSAGE , REPEATED, STATIC, AllTypes, rep_submsg, rep_bytes, &SubMessage_fields),
+    PB_FIELD( 37, ENUM    , REPEATED, STATIC, AllTypes, rep_enum, rep_submsg, 0),
+    PB_FIELD( 41, INT32   , OPTIONAL, STATIC, AllTypes, opt_int32, rep_enum, &AllTypes_opt_int32_default),
+    PB_FIELD( 42, INT64   , OPTIONAL, STATIC, AllTypes, opt_int64, opt_int32, &AllTypes_opt_int64_default),
+    PB_FIELD( 43, UINT32  , OPTIONAL, STATIC, AllTypes, opt_uint32, opt_int64, &AllTypes_opt_uint32_default),
+    PB_FIELD( 44, UINT64  , OPTIONAL, STATIC, AllTypes, opt_uint64, opt_uint32, &AllTypes_opt_uint64_default),
+    PB_FIELD( 45, SINT32  , OPTIONAL, STATIC, AllTypes, opt_sint32, opt_uint64, &AllTypes_opt_sint32_default),
+    PB_FIELD( 46, SINT64  , OPTIONAL, STATIC, AllTypes, opt_sint64, opt_sint32, &AllTypes_opt_sint64_default),
+    PB_FIELD( 47, BOOL    , OPTIONAL, STATIC, AllTypes, opt_bool, opt_sint64, &AllTypes_opt_bool_default),
+    PB_FIELD( 48, FIXED32 , OPTIONAL, STATIC, AllTypes, opt_fixed32, opt_bool, &AllTypes_opt_fixed32_default),
+    PB_FIELD( 49, SFIXED32, OPTIONAL, STATIC, AllTypes, opt_sfixed32, opt_fixed32, &AllTypes_opt_sfixed32_default),
+    PB_FIELD( 50, FLOAT   , OPTIONAL, STATIC, AllTypes, opt_float, opt_sfixed32, &AllTypes_opt_float_default),
+    PB_FIELD( 51, FIXED64 , OPTIONAL, STATIC, AllTypes, opt_fixed64, opt_float, &AllTypes_opt_fixed64_default),
+    PB_FIELD( 52, SFIXED64, OPTIONAL, STATIC, AllTypes, opt_sfixed64, opt_fixed64, &AllTypes_opt_sfixed64_default),
+    PB_FIELD( 53, DOUBLE  , OPTIONAL, STATIC, AllTypes, opt_double, opt_sfixed64, &AllTypes_opt_double_default),
+    PB_FIELD( 54, STRING  , OPTIONAL, STATIC, AllTypes, opt_string, opt_double, &AllTypes_opt_string_default),
+    PB_FIELD( 55, BYTES   , OPTIONAL, STATIC, AllTypes, opt_bytes, opt_string, &AllTypes_opt_bytes_default),
+    PB_FIELD( 56, MESSAGE , OPTIONAL, STATIC, AllTypes, opt_submsg, opt_bytes, &SubMessage_fields),
+    PB_FIELD( 57, ENUM    , OPTIONAL, STATIC, AllTypes, opt_enum, opt_submsg, &AllTypes_opt_enum_default),
+    PB_FIELD( 99, INT32   , REQUIRED, STATIC, AllTypes, end, opt_enum, 0),
+    PB_LAST_FIELD
+};
+

tests/bc_alltypes.pb.h

@@ -0,0 +1,178 @@
+/* Automatically generated nanopb header */
+/* This is a file generated using nanopb-0.2.0-dev.
+ * It is used as a part of test suite in order to detect any
+ * incompatible changes made to the generator in future versions.
+ */
+
+#ifndef _PB_ALLTYPES_PB_H_
+#define _PB_ALLTYPES_PB_H_
+#include <pb.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Enum definitions */
+typedef enum _MyEnum {
+    MyEnum_Zero = 0,
+    MyEnum_First = 1,
+    MyEnum_Second = 2,
+    MyEnum_Truth = 42
+} MyEnum;
+
+/* Struct definitions */
+typedef struct _SubMessage {
+    char substuff1[16];
+    int32_t substuff2;
+    bool has_substuff3;
+    uint32_t substuff3;
+} SubMessage;
+
+typedef struct {
+    size_t size;
+    uint8_t bytes[16];
+} AllTypes_req_bytes_t;
+
+typedef struct {
+    size_t size;
+    uint8_t bytes[16];
+} AllTypes_rep_bytes_t;
+
+typedef struct {
+    size_t size;
+    uint8_t bytes[16];
+} AllTypes_opt_bytes_t;
+
+typedef struct _AllTypes {
+    int32_t req_int32;
+    int64_t req_int64;
+    uint32_t req_uint32;
+    uint64_t req_uint64;
+    int32_t req_sint32;
+    int64_t req_sint64;
+    bool req_bool;
+    uint32_t req_fixed32;
+    int32_t req_sfixed32;
+    float req_float;
+    uint64_t req_fixed64;
+    int64_t req_sfixed64;
+    double req_double;
+    char req_string[16];
+    AllTypes_req_bytes_t req_bytes;
+    SubMessage req_submsg;
+    MyEnum req_enum;
+    size_t rep_int32_count;
+    int32_t rep_int32[5];
+    size_t rep_int64_count;
+    int64_t rep_int64[5];
+    size_t rep_uint32_count;
+    uint32_t rep_uint32[5];
+    size_t rep_uint64_count;
+    uint64_t rep_uint64[5];
+    size_t rep_sint32_count;
+    int32_t rep_sint32[5];
+    size_t rep_sint64_count;
+    int64_t rep_sint64[5];
+    size_t rep_bool_count;
+    bool rep_bool[5];
+    size_t rep_fixed32_count;
+    uint32_t rep_fixed32[5];
+    size_t rep_sfixed32_count;
+    int32_t rep_sfixed32[5];
+    size_t rep_float_count;
+    float rep_float[5];
+    size_t rep_fixed64_count;
+    uint64_t rep_fixed64[5];
+    size_t rep_sfixed64_count;
+    int64_t rep_sfixed64[5];
+    size_t rep_double_count;
+    double rep_double[5];
+    size_t rep_string_count;
+    char rep_string[5][16];
+    size_t rep_bytes_count;
+    AllTypes_rep_bytes_t rep_bytes[5];
+    size_t rep_submsg_count;
+    SubMessage rep_submsg[5];
+    size_t rep_enum_count;
+    MyEnum rep_enum[5];
+    bool has_opt_int32;
+    int32_t opt_int32;
+    bool has_opt_int64;
+    int64_t opt_int64;
+    bool has_opt_uint32;
+    uint32_t opt_uint32;
+    bool has_opt_uint64;
+    uint64_t opt_uint64;
+    bool has_opt_sint32;
+    int32_t opt_sint32;
+    bool has_opt_sint64;
+    int64_t opt_sint64;
+    bool has_opt_bool;
+    bool opt_bool;
+    bool has_opt_fixed32;
+    uint32_t opt_fixed32;
+    bool has_opt_sfixed32;
+    int32_t opt_sfixed32;
+    bool has_opt_float;
+    float opt_float;
+    bool has_opt_fixed64;
+    uint64_t opt_fixed64;
+    bool has_opt_sfixed64;
+    int64_t opt_sfixed64;
+    bool has_opt_double;
+    double opt_double;
+    bool has_opt_string;
+    char opt_string[16];
+    bool has_opt_bytes;
+    AllTypes_opt_bytes_t opt_bytes;
+    bool has_opt_submsg;
+    SubMessage opt_submsg;
+    bool has_opt_enum;
+    MyEnum opt_enum;
+    int32_t end;
+} AllTypes;
+
+/* Default values for struct fields */
+extern const char SubMessage_substuff1_default[16];
+extern const int32_t SubMessage_substuff2_default;
+extern const uint32_t SubMessage_substuff3_default;
+extern const int32_t AllTypes_opt_int32_default;
+extern const int64_t AllTypes_opt_int64_default;
+extern const uint32_t AllTypes_opt_uint32_default;
+extern const uint64_t AllTypes_opt_uint64_default;
+extern const int32_t AllTypes_opt_sint32_default;
+extern const int64_t AllTypes_opt_sint64_default;
+extern const bool AllTypes_opt_bool_default;
+extern const uint32_t AllTypes_opt_fixed32_default;
+extern const int32_t AllTypes_opt_sfixed32_default;
+extern const float AllTypes_opt_float_default;
+extern const uint64_t AllTypes_opt_fixed64_default;
+extern const int64_t AllTypes_opt_sfixed64_default;
+extern const double AllTypes_opt_double_default;
+extern const char AllTypes_opt_string_default[16];
+extern const AllTypes_opt_bytes_t AllTypes_opt_bytes_default;
+extern const MyEnum AllTypes_opt_enum_default;
+
+/* Struct field encoding specification for nanopb */
+extern const pb_field_t SubMessage_fields[4];
+extern const pb_field_t AllTypes_fields[53];
+
+/* Check that field information fits in pb_field_t */
+#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
+STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 256 && pb_membersize(AllTypes, rep_submsg[0]) < 256 && pb_membersize(AllTypes, opt_submsg) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_SubMessage_AllTypes)
+#endif
+
+#if !defined(PB_FIELD_32BIT)
+STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 65536 && pb_membersize(AllTypes, rep_submsg[0]) < 65536 && pb_membersize(AllTypes, opt_submsg) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_SubMessage_AllTypes)
+#endif
+
+/* On some platforms (such as AVR), double is really float.
+ * These are not directly supported by nanopb, but see example_avr_double.
+ */
+STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif

tests/bc_decode.c

@@ -0,0 +1,197 @@
+/* Tests the decoding of all types.
+ * This is a backwards-compatibility test, using bc_alltypes.pb.h.
+ * It is similar to test_decode3, but duplicated in order to allow
+ * test_decode3 to test any new features introduced later.
+ *
+ * Run e.g. ./bc_encode | ./bc_decode
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <pb_decode.h>
+#include "bc_alltypes.pb.h"
+
+#define TEST(x) if (!(x)) { \
+    printf("Test " #x " failed.\n"); \
+    return false; \
+    }
+
+/* This function is called once from main(), it handles
+   the decoding and checks the fields. */
+bool check_alltypes(pb_istream_t *stream, int mode)
+{
+    AllTypes alltypes;
+    
+    /* Fill with garbage to better detect initialization errors */
+    memset(&alltypes, 0xAA, sizeof(alltypes));
+    
+    if (!pb_decode(stream, AllTypes_fields, &alltypes))
+        return false;
+    
+    TEST(alltypes.req_int32         == -1001);
+    TEST(alltypes.req_int64         == -1002);
+    TEST(alltypes.req_uint32        == 1003);
+    TEST(alltypes.req_uint64        == 1004);
+    TEST(alltypes.req_sint32        == -1005);
+    TEST(alltypes.req_sint64        == -1006);
+    TEST(alltypes.req_bool          == true);
+    
+    TEST(alltypes.req_fixed32       == 1008);
+    TEST(alltypes.req_sfixed32      == -1009);
+    TEST(alltypes.req_float         == 1010.0f);
+    
+    TEST(alltypes.req_fixed64       == 1011);
+    TEST(alltypes.req_sfixed64      == -1012);
+    TEST(alltypes.req_double        == 1013.0f);
+    
+    TEST(strcmp(alltypes.req_string, "1014") == 0);
+    TEST(alltypes.req_bytes.size == 4);
+    TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0);
+    TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0);
+    TEST(alltypes.req_submsg.substuff2 == 1016);
+    TEST(alltypes.req_submsg.substuff3 == 3);
+    TEST(alltypes.req_enum == MyEnum_Truth);
+    
+    TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0);
+    TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0);
+    TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0);
+    TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0);
+    TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0);
+    TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0);
+    TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false);
+    
+    TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0);
+    TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0);
+    TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f);
+    
+    TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0);
+    TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0);
+    TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0);
+    
+    TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0');
+    TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0);
+    TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0);
+
+    TEST(alltypes.rep_submsg_count == 5);
+    TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0');
+    TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0);
+    TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3);
+    
+    TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero);
+    
+    if (mode == 0)
+    {
+        /* Expect default values */
+        TEST(alltypes.has_opt_int32     == false);
+        TEST(alltypes.opt_int32         == 4041);
+        TEST(alltypes.has_opt_int64     == false);
+        TEST(alltypes.opt_int64         == 4042);
+        TEST(alltypes.has_opt_uint32    == false);
+        TEST(alltypes.opt_uint32        == 4043);
+        TEST(alltypes.has_opt_uint64    == false);
+        TEST(alltypes.opt_uint64        == 4044);
+        TEST(alltypes.has_opt_sint32    == false);
+        TEST(alltypes.opt_sint32        == 4045);
+        TEST(alltypes.has_opt_sint64    == false);
+        TEST(alltypes.opt_sint64        == 4046);
+        TEST(alltypes.has_opt_bool      == false);
+        TEST(alltypes.opt_bool          == false);
+        
+        TEST(alltypes.has_opt_fixed32   == false);
+        TEST(alltypes.opt_fixed32       == 4048);
+        TEST(alltypes.has_opt_sfixed32  == false);
+        TEST(alltypes.opt_sfixed32      == 4049);
+        TEST(alltypes.has_opt_float     == false);
+        TEST(alltypes.opt_float         == 4050.0f);
+        
+        TEST(alltypes.has_opt_fixed64   == false);
+        TEST(alltypes.opt_fixed64       == 4051);
+        TEST(alltypes.has_opt_sfixed64  == false);
+        TEST(alltypes.opt_sfixed64      == 4052);
+        TEST(alltypes.has_opt_double    == false);
+        TEST(alltypes.opt_double        == 4053.0);
+        
+        TEST(alltypes.has_opt_string    == false);
+        TEST(strcmp(alltypes.opt_string, "4054") == 0);
+        TEST(alltypes.has_opt_bytes     == false);
+        TEST(alltypes.opt_bytes.size == 4);
+        TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0);
+        TEST(alltypes.has_opt_submsg    == false);
+        TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0);
+        TEST(alltypes.opt_submsg.substuff2 == 2);
+        TEST(alltypes.opt_submsg.substuff3 == 3);
+        TEST(alltypes.has_opt_enum     == false);
+        TEST(alltypes.opt_enum == MyEnum_Second);
+    }
+    else
+    {
+        /* Expect filled-in values */
+        TEST(alltypes.has_opt_int32     == true);
+        TEST(alltypes.opt_int32         == 3041);
+        TEST(alltypes.has_opt_int64     == true);
+        TEST(alltypes.opt_int64         == 3042);
+        TEST(alltypes.has_opt_uint32    == true);
+        TEST(alltypes.opt_uint32        == 3043);
+        TEST(alltypes.has_opt_uint64    == true);
+        TEST(alltypes.opt_uint64        == 3044);
+        TEST(alltypes.has_opt_sint32    == true);
+        TEST(alltypes.opt_sint32        == 3045);
+        TEST(alltypes.has_opt_sint64    == true);
+        TEST(alltypes.opt_sint64        == 3046);
+        TEST(alltypes.has_opt_bool      == true);
+        TEST(alltypes.opt_bool          == true);
+        
+        TEST(alltypes.has_opt_fixed32   == true);
+        TEST(alltypes.opt_fixed32       == 3048);
+        TEST(alltypes.has_opt_sfixed32  == true);
+        TEST(alltypes.opt_sfixed32      == 3049);
+        TEST(alltypes.has_opt_float     == true);
+        TEST(alltypes.opt_float         == 3050.0f);
+        
+        TEST(alltypes.has_opt_fixed64   == true);
+        TEST(alltypes.opt_fixed64       == 3051);
+        TEST(alltypes.has_opt_sfixed64  == true);
+        TEST(alltypes.opt_sfixed64      == 3052);
+        TEST(alltypes.has_opt_double    == true);
+        TEST(alltypes.opt_double        == 3053.0);
+        
+        TEST(alltypes.has_opt_string    == true);
+        TEST(strcmp(alltypes.opt_string, "3054") == 0);
+        TEST(alltypes.has_opt_bytes     == true);
+        TEST(alltypes.opt_bytes.size == 4);
+        TEST(memcmp(alltypes.opt_bytes.bytes, "3055", 4) == 0);
+        TEST(alltypes.has_opt_submsg    == true);
+        TEST(strcmp(alltypes.opt_submsg.substuff1, "3056") == 0);
+        TEST(alltypes.opt_submsg.substuff2 == 3056);
+        TEST(alltypes.opt_submsg.substuff3 == 3);
+        TEST(alltypes.has_opt_enum      == true);
+        TEST(alltypes.opt_enum == MyEnum_Truth);
+    }
+    
+    TEST(alltypes.end == 1099);
+    
+    return true;
+}
+
+int main(int argc, char **argv)
+{
+    /* Whether to expect the optional values or the default values. */
+    int mode = (argc > 1) ? atoi(argv[1]) : 0;
+    
+    /* Read the data into buffer */
+    uint8_t buffer[1024];
+    size_t count = fread(buffer, 1, sizeof(buffer), stdin);
+    
+    /* Construct a pb_istream_t for reading from the buffer */
+    pb_istream_t stream = pb_istream_from_buffer(buffer, count);
+    
+    /* Decode and print out the stuff */
+    if (!check_alltypes(&stream, mode))
+    {
+        printf("Parsing failed: %s\n", PB_GET_ERROR(&stream));
+        return 1;
+    } else {
+        return 0;
+    }
+}

tests/bc_encode.c

@@ -0,0 +1,131 @@
+/* Attempts to test all the datatypes supported by ProtoBuf.
+ * This is a backwards-compatibility test, using bc_alltypes.pb.h.
+ * It is similar to test_encode3, but duplicated in order to allow
+ * test_encode3 to test any new features introduced later.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pb_encode.h>
+#include "bc_alltypes.pb.h"
+
+int main(int argc, char **argv)
+{
+    int mode = (argc > 1) ? atoi(argv[1]) : 0;
+    
+    /* Initialize the structure with constants */
+    AllTypes alltypes = {0};
+    
+    alltypes.req_int32         = -1001;
+    alltypes.req_int64         = -1002;
+    alltypes.req_uint32        = 1003;
+    alltypes.req_uint64        = 1004;
+    alltypes.req_sint32        = -1005;
+    alltypes.req_sint64        = -1006;
+    alltypes.req_bool          = true;
+    
+    alltypes.req_fixed32       = 1008;
+    alltypes.req_sfixed32      = -1009;
+    alltypes.req_float         = 1010.0f;
+    
+    alltypes.req_fixed64       = 1011;
+    alltypes.req_sfixed64      = -1012;
+    alltypes.req_double        = 1013.0;
+    
+    strcpy(alltypes.req_string, "1014");
+    alltypes.req_bytes.size = 4;
+    memcpy(alltypes.req_bytes.bytes, "1015", 4);
+    strcpy(alltypes.req_submsg.substuff1, "1016");
+    alltypes.req_submsg.substuff2 = 1016;
+    alltypes.req_enum = MyEnum_Truth;
+    
+    alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001;
+    alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002;
+    alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003;
+    alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004;
+    alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005;
+    alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006;
+    alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true;
+    
+    alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008;
+    alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009;
+    alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f;
+    
+    alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011;
+    alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012;
+    alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0;
+    
+    alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014");
+    alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4;
+    memcpy(alltypes.rep_bytes[4].bytes, "2015", 4);
+
+    alltypes.rep_submsg_count = 5;
+    strcpy(alltypes.rep_submsg[4].substuff1, "2016");
+    alltypes.rep_submsg[4].substuff2 = 2016;
+    alltypes.rep_submsg[4].has_substuff3 = true;
+    alltypes.rep_submsg[4].substuff3 = 2016;
+    
+    alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth;
+    
+    if (mode != 0)
+    {
+        /* Fill in values for optional fields */
+        alltypes.has_opt_int32 = true;
+        alltypes.opt_int32         = 3041;
+        alltypes.has_opt_int64 = true;
+        alltypes.opt_int64         = 3042;
+        alltypes.has_opt_uint32 = true;
+        alltypes.opt_uint32        = 3043;
+        alltypes.has_opt_uint64 = true;
+        alltypes.opt_uint64        = 3044;
+        alltypes.has_opt_sint32 = true;
+        alltypes.opt_sint32        = 3045;
+        alltypes.has_opt_sint64 = true;
+        alltypes.opt_sint64        = 3046;
+        alltypes.has_opt_bool = true;
+        alltypes.opt_bool          = true;
+        
+        alltypes.has_opt_fixed32 = true;
+        alltypes.opt_fixed32       = 3048;
+        alltypes.has_opt_sfixed32 = true;
+        alltypes.opt_sfixed32      = 3049;
+        alltypes.has_opt_float = true;
+        alltypes.opt_float         = 3050.0f;
+        
+        alltypes.has_opt_fixed64 = true;
+        alltypes.opt_fixed64       = 3051;
+        alltypes.has_opt_sfixed64 = true;
+        alltypes.opt_sfixed64      = 3052;
+        alltypes.has_opt_double = true;
+        alltypes.opt_double        = 3053.0;
+        
+        alltypes.has_opt_string = true;
+        strcpy(alltypes.opt_string, "3054");
+        alltypes.has_opt_bytes = true;
+        alltypes.opt_bytes.size = 4;
+        memcpy(alltypes.opt_bytes.bytes, "3055", 4);
+        alltypes.has_opt_submsg = true;
+        strcpy(alltypes.opt_submsg.substuff1, "3056");
+        alltypes.opt_submsg.substuff2 = 3056;
+        alltypes.has_opt_enum = true;
+        alltypes.opt_enum = MyEnum_Truth;
+    }
+    
+    alltypes.end = 1099;
+    
+    uint8_t buffer[1024];
+    pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+    
+    /* Now encode it and check if we succeeded. */
+    if (pb_encode(&stream, AllTypes_fields, &alltypes))
+    {
+        fwrite(buffer, 1, stream.bytes_written, stdout);
+        return 0; /* Success */
+    }
+    else
+    {
+        fprintf(stderr, "Encoding failed!\n");
+        return 1; /* Failure */
+    }
+}

tests/callbacks2.proto

@@ -0,0 +1,9 @@
+// Test if including generated header file for this file + implicit include of
+// callbacks.pb.h still compiles. Used with test_compiles.c.
+import "callbacks.proto";
+
+message Callback2Message {
+    required TestMessage tstmsg = 1;
+    required SubMessage submsg = 2;
+}
+

tests/options.expected

@@ -0,0 +1,7 @@
+char filesize\[20\];
+char msgsize\[30\];
+char fieldsize\[40\];
+pb_callback_t int32_callback;
+\sEnumValue1 = 1
+Message5_EnumValue1
+} pb_packed my_packed_struct;

tests/options.proto

@@ -0,0 +1,73 @@
+/* Test nanopb option parsing.
+ * options.expected lists the patterns that are searched for in the output.
+ */
+
+import "nanopb.proto";
+
+// File level options
+option (nanopb_fileopt).max_size = 20;
+
+message Message1
+{
+    required string filesize = 1;
+}
+
+// Message level options
+message Message2
+{
+    option (nanopb_msgopt).max_size = 30;
+    required string msgsize = 1;
+}
+
+// Field level options
+message Message3
+{
+    required string fieldsize = 1 [(nanopb).max_size = 40];
+}
+
+// Forced callback field
+message Message4
+{
+    required int32 int32_callback = 1 [(nanopb).type = FT_CALLBACK];
+}
+
+// Short enum names
+enum Enum1
+{
+    option (nanopb_enumopt).long_names = false;
+    EnumValue1 = 1;
+    EnumValue2 = 2;
+}
+
+message EnumTest
+{
+    required Enum1 field = 1 [default = EnumValue2];
+}
+
+// Short enum names inside message
+message Message5
+{
+    enum Enum2
+    {
+       option (nanopb_enumopt).long_names = false;
+       EnumValue1 = 1;
+    }
+    required Enum2 field = 1 [default = EnumValue1];
+}
+
+// Packed structure
+message my_packed_struct
+{
+    option (nanopb_msgopt).packed_struct = true;
+    optional int32 myfield = 1;
+}
+
+// Message with ignored field
+// Note: doesn't really test if the field is missing in the output,
+// but atleast tests that the output compiles.
+message Message6
+{
+    required int32 field1 = 1;
+    optional int32 field2 = 2 [(nanopb).type = FT_IGNORE];
+}
+

tests/test_decode1.c

@@ -63,6 +63,12 @@ int main()
     uint8_t buffer[512];
     size_t count = fread(buffer, 1, sizeof(buffer), stdin);
     
+    if (!feof(stdin))
+    {
+    	printf("Message does not fit in buffer\n");
+    	return 1;
+    }
+    
     /* Construct a pb_istream_t for reading from the buffer */
     pb_istream_t stream = pb_istream_from_buffer(buffer, count);
     

tests/test_decode2.c

@@ -59,13 +59,6 @@ bool callback(pb_istream_t *stream, uint8_t *buf, size_t count)
     FILE *file = (FILE*)stream->state;
     bool status;
     
-    if (buf == NULL)
-    {
-       /* Skipping data */
-        while (count-- && fgetc(file) != EOF);
-        return count == 0;
-    }
-    
     status = (fread(buf, 1, count, file) == count);
     
     if (feof(file))

tests/test_decode3.c

@@ -26,20 +26,20 @@ bool check_alltypes(pb_istream_t *stream, int mode)
     if (!pb_decode(stream, AllTypes_fields, &alltypes))
         return false;
     
-    TEST(alltypes.req_int32         == 1001);
+    TEST(alltypes.req_int32         == -1001);
-    TEST(alltypes.req_int64         == 1002);
+    TEST(alltypes.req_int64         == -1002);
     TEST(alltypes.req_uint32        == 1003);
     TEST(alltypes.req_uint64        == 1004);
-    TEST(alltypes.req_sint32        == 1005);
+    TEST(alltypes.req_sint32        == -1005);
-    TEST(alltypes.req_sint64        == 1006);
+    TEST(alltypes.req_sint64        == -1006);
     TEST(alltypes.req_bool          == true);
     
     TEST(alltypes.req_fixed32       == 1008);
-    TEST(alltypes.req_sfixed32      == 1009);
+    TEST(alltypes.req_sfixed32      == -1009);
     TEST(alltypes.req_float         == 1010.0f);
     
     TEST(alltypes.req_fixed64       == 1011);
-    TEST(alltypes.req_sfixed64      == 1012);
+    TEST(alltypes.req_sfixed64      == -1012);
     TEST(alltypes.req_double        == 1013.0f);
     
     TEST(strcmp(alltypes.req_string, "1014") == 0);
@@ -50,20 +50,20 @@ bool check_alltypes(pb_istream_t *stream, int mode)
     TEST(alltypes.req_submsg.substuff3 == 3);
     TEST(alltypes.req_enum == MyEnum_Truth);
     
-    TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == 2001 && alltypes.rep_int32[0] == 0);
+    TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0);
-    TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == 2002 && alltypes.rep_int64[0] == 0);
+    TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0);
     TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0);
     TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0);
-    TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == 2005 && alltypes.rep_sint32[0] == 0);
+    TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0);
-    TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == 2006 && alltypes.rep_sint64[0] == 0);
+    TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0);
     TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false);
     
     TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0);
-    TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == 2009 && alltypes.rep_sfixed32[0] == 0);
+    TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0);
     TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f);
     
     TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0);
-    TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == 2012 && alltypes.rep_sfixed64[0] == 0);
+    TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0);
     TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0);
     
     TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0');

tests/test_encode3.c

@@ -14,20 +14,20 @@ int main(int argc, char **argv)
     /* Initialize the structure with constants */
     AllTypes alltypes = {0};
     
-    alltypes.req_int32         = 1001;
+    alltypes.req_int32         = -1001;
-    alltypes.req_int64         = 1002;
+    alltypes.req_int64         = -1002;
     alltypes.req_uint32        = 1003;
     alltypes.req_uint64        = 1004;
-    alltypes.req_sint32        = 1005;
+    alltypes.req_sint32        = -1005;
-    alltypes.req_sint64        = 1006;
+    alltypes.req_sint64        = -1006;
     alltypes.req_bool          = true;
     
     alltypes.req_fixed32       = 1008;
-    alltypes.req_sfixed32      = 1009;
+    alltypes.req_sfixed32      = -1009;
     alltypes.req_float         = 1010.0f;
     
     alltypes.req_fixed64       = 1011;
-    alltypes.req_sfixed64      = 1012;
+    alltypes.req_sfixed64      = -1012;
     alltypes.req_double        = 1013.0;
     
     strcpy(alltypes.req_string, "1014");
@@ -37,20 +37,20 @@ int main(int argc, char **argv)
     alltypes.req_submsg.substuff2 = 1016;
     alltypes.req_enum = MyEnum_Truth;
     
-    alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = 2001;
+    alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001;
-    alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = 2002;
+    alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002;
     alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003;
     alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004;
-    alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = 2005;
+    alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005;
-    alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = 2006;
+    alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006;
     alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true;
     
     alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008;
-    alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = 2009;
+    alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009;
     alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f;
     
     alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011;
-    alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = 2012;
+    alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012;
     alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0;
     
     alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014");

tests/test_multiple_files.c

@@ -0,0 +1,13 @@
+/*
+ * Tests if still compile if typedefs are redfefined in STATIC_ASSERTS when
+ * proto file includes another poto file
+ */
+
+#include <stdio.h>
+#include <pb_encode.h>
+#include "callbacks2.pb.h"
+
+int main()
+{
+	return 0;
+}

tools/set_version.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Run this from the top directory of nanopb tree.
+# e.g. user@localhost:~/nanopb$ tools/set_version.sh nanopb-0.1.9-dev
+# It sets the version number in pb.h and generator/nanopb_generator.py.
+
+sed -i -e 's/nanopb_version\s*=\s*"[^"]*"/nanopb_version = "'$1'"/' generator/nanopb_generator.py
+sed -i -e 's/#define\s*NANOPB_VERSION\s*.*/#define NANOPB_VERSION '$1'/' pb.h
+
+