Publishing nanopb-0.2.0

Patch courtesy of Michael Haberler.

CHANGELOG

@@ -1,4 +1,17 @@
-git master
+nanopb-0.2.0
+ NOTE: This release requires you to regenerate all .pb.c
+       files. Files generated by older versions will not
+       compile anymore.
+
+ Reformat generated .pb.c files using macros (issue 58)
+ Rename PB_HTYPE_ARRAY -> PB_HTYPE_REPEATED
+ Separate PB_HTYPE to PB_ATYPE and PB_HTYPE
+ Move STATIC_ASSERTs to .pb.c file
+ Added CMake file (by Pavel Ilin)
+ Add option to give file extension to generator (by Michael Haberler)
+ Documentation updates
+
+nanopb-0.1.9
  Fixed error message bugs (issues 52, 56)
  Sanitize #ifndef filename (issue 50)
  Performance improvements

cmake/FindNanopb.cmake

@@ -0,0 +1,224 @@
+# Locate and configure the nanopb library.
+#
+# The following varialbes have to be set:
+#
+#   NANOPB_SRC_ROOT_FOLDER  - Path to nanopb source folder
+#
+# The following variables can be set and are optional:
+#
+#
+#   PROTOBUF_SRC_ROOT_FOLDER - When compiling with MSVC, if this cache variable is set
+#                              the protobuf-default VS project build locations
+#                              (vsprojects/Debug & vsprojects/Release) will be searched
+#                              for libraries and binaries.
+#
+#   NANOPB_IMPORT_DIRS       - List of additional directories to be searched for
+#                              imported .proto files.
+#
+#   NANOPB_GENERATE_CPP_APPEND_PATH - By default -I will be passed to protoc
+#                                     for each directory where a proto file is referenced.
+#                                     Set to FALSE if you want to disable this behaviour.
+#
+# Defines the following variables:
+#
+#   NANOPB_FOUND - Found the nanopb library (source&header files, generator tool, protoc compiler tool)
+#   NANOPB_INCLUDE_DIRS - Include directories for Google Protocol Buffers
+#
+# The following cache variables are also available to set or use:
+#   NANOPB_GENERATOR_EXECUTABLE - The nanopb generator
+#   PROTOBUF_PROTOC_EXECUTABLE - The protoc compiler
+#
+#  ====================================================================
+#
+# NANOPB_GENERATE_CPP (public function)
+#   SRCS = Variable to define with autogenerated
+#          source files
+#   HDRS = Variable to define with autogenerated
+#          header files
+#   ARGN = proto files
+#
+#  ====================================================================
+#  Example:
+#
+#   set(NANOPB_SRC_ROOT_FOLDER "/path/to/nanopb")
+#   set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${NANOPB_SRC_ROOT_FOLDER}/cmake)
+#   find_package( Nanopb REQUIRED )
+#   include_directories(${NANOPB_INCLUDE_DIRS})
+#
+#   NANOPB_GENERATE_CPP(PROTO_SRCS PROTO_HDRS foo.proto)
+#
+#   include_directories(${CMAKE_CURRENT_BINARY_DIR})
+#   add_executable(bar bar.cc ${PROTO_SRCS} ${PROTO_HDRS})
+#
+#  ====================================================================
+
+#=============================================================================
+# Copyright 2009 Kitware, Inc.
+# Copyright 2009-2011 Philip Lowman <philip@yhbt.com>
+# Copyright 2008 Esben Mose Hansen, Ange Optimization ApS
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+#   notice, this list of conditions and the following disclaimer in the
+#   documentation and/or other materials provided with the distribution.
+#
+# * Neither the names of Kitware, Inc., the Insight Software Consortium,
+#   nor the names of their contributors may be used to endorse or promote
+#   products derived from this software without specific prior written
+#   permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#=============================================================================
+#
+# Changes
+# 2013.01.31 - Pavlo Ilin - used Modules/FindProtobuf.cmake from cmake 2.8.10 to
+#                           write FindNanopb.cmake
+#
+#=============================================================================
+
+
+function(NANOPB_GENERATE_CPP SRCS HDRS)
+  if(NOT ARGN)
+    return()
+  endif()
+
+  if(NANOPB_GENERATE_CPP_APPEND_PATH)
+    # Create an include path for each file specified
+    foreach(FIL ${ARGN})
+      get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+      get_filename_component(ABS_PATH ${ABS_FIL} PATH)
+
+      list(FIND _nanobp_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _nanobp_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  else()
+    set(_nanobp_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+
+  if(DEFINED NANOPB_IMPORT_DIRS)
+    foreach(DIR ${NANOPB_IMPORT_DIRS})
+      get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+      list(FIND _nanobp_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _nanobp_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  endif()
+
+  set(${SRCS})
+  set(${HDRS})
+  get_filename_component(GENERATOR_PATH ${NANOPB_GENERATOR_EXECUTABLE} PATH)
+
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb"
+      COMMAND  ${PROTOBUF_PROTOC_EXECUTABLE}
+      ARGS -I${GENERATOR_PATH} -I${CMAKE_CURRENT_BINARY_DIR} ${_nanobp_include_path} -o${FIL_WE}.pb ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM )
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
+      COMMAND python
+      ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb
+      DEPENDS ${FIL_WE}.pb
+      COMMENT "Running nanopb generator on ${FIL_WE}.pb"
+      VERBATIM )
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set(${SRCS} ${${SRCS}} ${NANOPB_SRCS} PARENT_SCOPE)
+  set(${HDRS} ${${HDRS}} ${NANOPB_HDRS} PARENT_SCOPE)
+
+endfunction()
+
+
+
+#
+# Main.
+#
+
+# By default have NANOPB_GENERATE_CPP macro pass -I to protoc
+# for each directory where a proto file is referenced.
+if(NOT DEFINED NANOPB_GENERATE_CPP_APPEND_PATH)
+  set(NANOPB_GENERATE_CPP_APPEND_PATH TRUE)
+endif()
+
+# Find the include directory
+find_path(NANOPB_INCLUDE_DIRS
+    pb.h
+    PATHS ${NANOPB_SRC_ROOT_FOLDER}
+)
+mark_as_advanced(NANOPB_INCLUDE_DIRS)
+
+# Find nanopb source files
+set(NANOPB_SRCS)
+set(NANOPB_HDRS)
+list(APPEND _nanopb_srcs pb_decode.c pb_encode.c)
+list(APPEND _nanopb_hdrs pb_decode.h pb_encode.h pb.h)
+
+foreach(FIL ${_nanopb_srcs})
+  find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_SRC_ROOT_FOLDER} ${NANOPB_INCLUDE_DIRS})
+  list(APPEND NANOPB_SRCS "${${FIL}__nano_pb_file}")
+  mark_as_advanced(${FIL}__nano_pb_file)
+endforeach()
+
+foreach(FIL ${_nanopb_hdrs})
+  find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_INCLUDE_DIRS})
+  mark_as_advanced(${FIL}__nano_pb_file)
+  list(APPEND NANOPB_HDRS "${${FIL}__nano_pb_file}")
+endforeach()
+
+# Find the protoc Executable
+find_program(PROTOBUF_PROTOC_EXECUTABLE
+    NAMES protoc
+    DOC "The Google Protocol Buffers Compiler"
+    PATHS
+    ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Release
+    ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Debug
+)
+mark_as_advanced(PROTOBUF_PROTOC_EXECUTABLE)
+
+# Find nanopb generator
+find_file(NANOPB_GENERATOR_EXECUTABLE
+    NAMES nanopb_generator.py
+    DOC "nanopb generator"
+    PATHS
+    ${NANOPB_SRC_ROOT_FOLDER}/generator
+)
+mark_as_advanced(NANOPB_GENERATOR_EXECUTABLE)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(NANOPB DEFAULT_MSG
+  NANOPB_INCLUDE_DIRS
+  NANOPB_SRCS NANOPB_HDRS
+  NANOPB_GENERATOR_EXECUTABLE
+  PROTOBUF_PROTOC_EXECUTABLE
+  )

docs/concepts.rst

@@ -255,16 +255,8 @@ For example this submessage in the Person.proto file::
 generates this field description array for the structure *Person_PhoneNumber*::
 
  const pb_field_t Person_PhoneNumber_fields[3] = {
-    {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING,
-    offsetof(Person_PhoneNumber, number), 0,
-    pb_membersize(Person_PhoneNumber, number), 0, 0},
-
-    {2, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT,
-    pb_delta(Person_PhoneNumber, type, number),
-    pb_delta(Person_PhoneNumber, has_type, type),
-    pb_membersize(Person_PhoneNumber, type), 0,
-    &Person_PhoneNumber_type_default},
-
+    PB_FIELD(  1, STRING  , REQUIRED, STATIC, Person_PhoneNumber, number, number, 0),
+    PB_FIELD(  2, ENUM    , OPTIONAL, STATIC, Person_PhoneNumber, type, number, &Person_PhoneNumber_type_default),
     PB_LAST_FIELD
  };
 
@@ -276,8 +268,8 @@ Most functions in nanopb return bool: *true* means success, *false* means failur
 
 The error messages help in guessing what is the underlying cause of the error. The most common error conditions are:
 
-1) Running out of memory. Because everything is allocated from the stack, nanopb can't detect this itself. Encoding or decoding the same type of a message always takes the same amount of stack space. Therefore, if it works once, it works always.
-2) Invalid field description. These are usually stored as constants, so if it works under the debugger, it always does.
+1) Running out of memory, i.e. stack overflow.
+2) Invalid field descriptors (would usually mean a bug in the generator).
 3) IO errors in your own stream callbacks.
 4) Errors that happen in your callback functions.
 5) Exceeding the max_size or bytes_left of a stream.

docs/index.rst

@@ -36,23 +36,26 @@ Features and limitations
 **Features**
 
 #) Pure C runtime
-#) Small code size (2–10 kB depending on processor)
-#) Small ram usage (typically 200 bytes)
+#) Small code size (2–10 kB depending on processor, plus any message definitions)
+#) Small ram usage (typically ~300 bytes, plus any message structs)
 #) Allows specifying maximum size for strings and arrays, so that they can be allocated statically.
 #) No malloc needed: everything can be allocated statically or on the stack.
 #) You can use either encoder or decoder alone to cut the code size in half.
+#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, packed arrays.
+#) Callback mechanism for handling messages larger than can fit in available RAM.
+#) Extensive set of tests.
 
 **Limitations**
 
 #) User must provide callbacks when decoding arrays or strings without maximum size. Malloc support could be added as a separate module.
-#) Some speed has been sacrificed for code size. For example varint calculations are always done in 64 bits.
+#) Some speed has been sacrificed for code size.
 #) Encoding is focused on writing to streams. For memory buffers only it could be made more efficient.
 #) The deprecated Protocol Buffers feature called "groups" is not supported.
 #) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file.
 #) Unknown fields are not preserved when decoding and re-encoding a message.
 #) Reflection (runtime introspection) is not supported. E.g. you can't request a field by giving its name in a string.
 #) Numeric arrays are always encoded as packed, even if not marked as packed in .proto. This causes incompatibility with decoders that do not support packed format.
-#) Cyclic references between messages are not supported. They could be supported in callback-mode if there was an option in the generator to set the mode.
+#) Cyclic references between messages are supported only in callback mode.
 
 Getting started
 ===============
@@ -104,10 +107,3 @@ Debugging and testing
 =====================
 Extensive unittests are included under the *tests* folder. Just type *make* there to run the tests.
 
-This also generates a file called *breakpoints* which includes all lines returning *false* in nanopb. You can use this in gdb by typing *source breakpoints*, after which gdb will break on first nanopb error.
-
-Wishlist
-========
-#) A specialized encoder for encoding to a memory buffer. Should serialize in reverse order to avoid having to determine submessage size beforehand.
-#) A cleaner rewrite of the Python-based source generator.
-#) Better performance for 16- and 8-bit platforms: use smaller datatypes where possible.

docs/reference.rst

@@ -37,22 +37,23 @@ pb_type_t
 ---------
 Defines the encoder/decoder behaviour that should be used for a field. ::
 
-    typedef enum { ... } pb_type_t;
+    typedef uint8_t pb_type_t;
 
-The low-order byte of the enumeration values defines the function that can be used for encoding and decoding the field data:
+The low-order nibble of the enumeration values defines the function that can be used for encoding and decoding the field data:
 
 ==================== ===== ================================================
 LTYPE identifier     Value Storage format
 ==================== ===== ================================================
 PB_LTYPE_VARINT      0x00  Integer.
 PB_LTYPE_SVARINT     0x01  Integer, zigzag encoded.
-PB_LTYPE_FIXED       0x02  Integer or floating point.
-PB_LTYPE_BYTES       0x03  Structure with *size_t* field and byte array.
-PB_LTYPE_STRING      0x04  Null-terminated string.
-PB_LTYPE_SUBMESSAGE  0x05  Submessage structure.
+PB_LTYPE_FIXED32     0x02  32-bit integer or floating point.
+PB_LTYPE_FIXED64     0x03  64-bit integer or floating point.
+PB_LTYPE_BYTES       0x04  Structure with *size_t* field and byte array.
+PB_LTYPE_STRING      0x05  Null-terminated string.
+PB_LTYPE_SUBMESSAGE  0x06  Submessage structure.
 ==================== ===== ================================================
 
-The high-order byte defines whether the field is required, optional, repeated or callback:
+The bits 4-5 define whether the field is required, optional or repeated:
 
 ==================== ===== ================================================
 HTYPE identifier     Value Field handling
@@ -60,13 +61,24 @@ HTYPE identifier     Value Field handling
 PB_HTYPE_REQUIRED    0x00  Verify that field exists in decoded message.
 PB_HTYPE_OPTIONAL    0x10  Use separate *has_<field>* boolean to specify
                            whether the field is present.
-PB_HTYPE_ARRAY       0x20  A repeated field with preallocated array.
+                           (Unless it is a callback)
+PB_HTYPE_REPEATED    0x20  A repeated field with preallocated array.
                            Separate *<field>_count* for number of items.
-PB_HTYPE_CALLBACK    0x30  A field with dynamic storage size, data is
-                           actually a pointer to a structure containing a
-                           callback function.
+                           (Unless it is a callback)
 ==================== ===== ================================================
 
+The bits 6-7 define the how the storage for the field is allocated:
+
+==================== ===== ================================================
+ATYPE identifier     Value Allocation method
+==================== ===== ================================================
+PB_ATYPE_STATIC      0x00  Statically allocated storage in the structure.
+PB_ATYPE_CALLBACK    0x40  A field with dynamic storage size. Struct field
+                           actually contains a pointer to a callback
+                           function.
+==================== ===== ================================================
+
+
 pb_field_t
 ----------
 Describes a single structure field with memory position in relation to others. The descriptions are usually autogenerated. ::
@@ -83,7 +95,7 @@ Describes a single structure field with memory position in relation to others. T
     } pb_packed;
 
 :tag:           Tag number of the field or 0 to terminate a list of fields.
-:type:          LTYPE and HTYPE of the field.
+:type:          LTYPE, HTYPE and ATYPE of the field.
 :data_offset:   Offset of field data, relative to the end of the previous field.
 :size_offset:   Offset of *bool* flag for optional fields or *size_t* count for arrays, relative to field data.
 :data_size:     Size of a single data entry, in bytes. For PB_LTYPE_BYTES, the size of the byte array inside the containing structure. For PB_HTYPE_CALLBACK, size of the C data type if known.

generator/nanopb_generator.py

@@ -1,5 +1,5 @@
 '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.'''
-nanopb_version = "0.2.0-dev"
+nanopb_version = "nanopb-0.2.0"
 
 try:
     import google.protobuf.descriptor_pb2 as descriptor
@@ -439,7 +439,7 @@ def make_identifier(headername):
             result += '_'
     return result
 
-def generate_header(dependencies, headername, enums, messages):
+def generate_header(dependencies, headername, enums, messages, options):
     '''Generate content for a header file.
     Generates strings, which should be concatenated and stored to file.
     '''
@@ -454,7 +454,7 @@ def generate_header(dependencies, headername, enums, messages):
     
     for dependency in dependencies:
         noext = os.path.splitext(dependency)[0]
-        yield '#include "%s.pb.h"\n' % noext
+        yield '#include "%s.%s.h"\n' % (noext,options.extension)
     
     yield '#ifdef __cplusplus\n'
     yield 'extern "C" {\n'
@@ -582,6 +582,8 @@ optparser = OptionParser(
              "Output will be written to file.pb.h and file.pb.c.")
 optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", default=[],
     help="Exclude file from generated #include list.")
+optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default="pb",
+    help="use extension instead of 'pb' for generated files.")
 optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False,
     help="Don't print anything except errors.")
 optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
@@ -638,8 +640,8 @@ def process(filenames, options):
         enums, messages = parse_file(fdesc.file[0], file_options)
         
         noext = os.path.splitext(filename)[0]
-        headername = noext + '.pb.h'
-        sourcename = noext + '.pb.c'
+        headername = noext + '.' + options.extension + '.h'
+        sourcename = noext + '.' + options.extension + '.c'
         headerbasename = os.path.basename(headername)
         
         if not options.quiet:
@@ -651,7 +653,7 @@ def process(filenames, options):
         dependencies = [d for d in fdesc.file[0].dependency if d not in excludes]
         
         header = open(headername, 'w')
-        for part in generate_header(dependencies, headerbasename, enums, messages):
+        for part in generate_header(dependencies, headerbasename, enums, messages, options):
             header.write(part)
 
         source = open(sourcename, 'w')

pb.h

@@ -6,7 +6,7 @@
  * see pb_encode.h or pb_decode.h
  */
 
-#define NANOPB_VERSION nanopb-0.2.0-dev
+#define NANOPB_VERSION nanopb-0.2.0
 
 #include <stdint.h>
 #include <stddef.h>