Improve alltypes_pointer test case

Allocation decision is now made before the field data type is decided.
This way the data type decisions can more cleanly account for the allocation
type, i.e. FT_DEFAULT logic etc.

Added pb_bytes_ptr_t for pointer-allocated bytes-fields. There is no point
generating separate structs for these, as they would all be of the same type.

.gitignore

@@ -5,17 +5,22 @@
 *.pb.c
 *.pb.h
 *.pb
+*.pyc
 *~
 *.tar.gz
 .sconsign.dblite
+config.log
+.sconf_temp
+tests/build
 julkaisu.txt
 docs/*.html
 docs/generator_flow.png
-example/client
+examples/simple/simple
-example/server
+examples/network_server/client
-example_avr_double/decode_double
+examples/network_server/server
-example_avr_double/encode_double
+examples/using_double_on_avr/decode_double
-example_avr_double/test_conversions
+examples/using_double_on_avr/encode_double
-example_unions/decode
+examples/using_double_on_avr/test_conversions
-example_unions/encode
+examples/using_union_messages/decode
+examples/using_union_messages/encode
 generator/nanopb_pb2.pyc

CHANGELOG

@@ -1,4 +1,13 @@
-nanopb-0.2.3
+nanopb-0.2.4 (2013-11-07)
+ Remove the deprecated NANOPB_INTERNALS functions from public API.
+ Document the security model.
+ Check array and bytes max sizes when encoding (issue 90)
+ Add #defines for maximum encoded message size (issue 89)
+ Add #define tags for extension fields (issue 93)
+ Fix MISRA C violations (issue 91)
+ Clean up pb_field_t definition with typedefs.
+
+nanopb-0.2.3 (2013-09-18)
  Improve compatibility by removing ternary operator from initializations (issue 88)
  Fix build error on Visual C++ (issue 84, patch by Markus Schwarzenberg)
  Don't stop on unsupported extension fields (issue 83)
@@ -7,7 +16,7 @@ nanopb-0.2.3
  Switch from Makefiles to scons for building the tests
  Make the tests buildable on Windows
 
-nanopb-0.2.2
+nanopb-0.2.2 (2013-08-18)
  Add support for extension fields (issue 17)
  Fix unknown fields in empty message (issue 78)
  Include the field tags in the generated .pb.h file.
@@ -16,7 +25,7 @@ nanopb-0.2.2
  Documentation improvements (issues 12, 77 and others)
  Improved tests
 
-nanopb-0.2.1
+nanopb-0.2.1 (2013-04-14)
  NOTE: The default callback function signature has changed.
        If you don't want to update your code, define PB_OLD_CALLBACK_STYLE.
  
@@ -33,7 +42,7 @@ nanopb-0.2.1
  Various new generator options
  Improved tests
 
-nanopb-0.2.0
+nanopb-0.2.0 (2013-03-02)
  NOTE: This release requires you to regenerate all .pb.c
        files. Files generated by older versions will not
        compile anymore.
@@ -46,7 +55,7 @@ nanopb-0.2.0
  Add option to give file extension to generator (by Michael Haberler)
  Documentation updates
 
-nanopb-0.1.9
+nanopb-0.1.9 (2013-02-13)
  Fixed error message bugs (issues 52, 56)
  Sanitize #ifndef filename (issue 50)
  Performance improvements
@@ -57,13 +66,13 @@ nanopb-0.1.9
  Added generator option to make message structs packed. (issue 49)
  Add more test cases.
 
-nanopb-0.1.8
+nanopb-0.1.8 (2012-12-13)
  Fix bugs in the enum short names introduced in 0.1.7 (issues 42, 43)
  Fix STATIC_ASSERT macro when using multiple .proto files. (issue 41)
  Fix missing initialization of istream.errmsg
  Make tests/Makefile work for non-gcc compilers (issue 40)
 
-nanopb-0.1.7
+nanopb-0.1.7 (2012-11-11)
  Remove "skip" mode from pb_istream_t callbacks. Example implementation had a bug. (issue 37)
  Add option to use shorter names for enum values (issue 38)
  Improve options support in generator (issues 12, 30)
@@ -73,19 +82,19 @@ nanopb-0.1.7
  Add buffer size check in example (issue 34)
  Fix build warnings on MS compilers (issue 33)
 
-nanopb-0.1.6
+nanopb-0.1.6 (2012-09-02)
  Reorganize the field decoder interface (issue 2)
  Improve performance in submessage decoding (issue 28)
  Implement error messages in the decoder side (issue 7)
  Extended testcases (alltypes test is now complete).
  Fix some compiler warnings (issues 25, 26, 27, 32).
 
-nanopb-0.1.5
+nanopb-0.1.5 (2012-08-04)
  Fix bug in decoder with packed arrays (issue 23).
  Extended testcases.
  Fix some compiler warnings.
 
-nanopb-0.1.4
+nanopb-0.1.4 (2012-07-05)
  Add compile-time options for easy-to-use >255 field support.
  Improve the detection of missing required fields.
  Added example on how to handle union messages.
@@ -93,20 +102,20 @@ nanopb-0.1.4
  Fix problems that stopped the code from compiling with some compilers.
  Fix some compiler warnings.
 
-nanopb-0.1.3
+nanopb-0.1.3 (2012-06-12)
  Refactor the field encoder interface.
  Improve generator error messages (issue 5)
  Add descriptor.proto into the #include exclusion list
  Fix some compiler warnings.
 
-nanopb-0.1.2
+nanopb-0.1.2 (2012-02-15)
  Make the generator to generate include for other .proto files (issue 4).
  Fixed generator not working on Windows (issue 3)
 
-nanopb-0.1.1
+nanopb-0.1.1 (2012-01-14)
  Fixed bug in encoder with 'bytes' fields (issue 1).
  Fixed a bug in the generator that caused a compiler error on sfixed32 and sfixed64 fields.
  Extended testcases.
 
-nanopb-0.1.0
+nanopb-0.1.0 (2012-01-06)
  First stable release.

cmake/FindNanopb.cmake

@@ -148,7 +148,7 @@ function(NANOPB_GENERATE_CPP SRCS HDRS)
              "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
       COMMAND python
       ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb
-      DEPENDS ${FIL_WE}.pb
+      DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb"
       COMMENT "Running nanopb generator on ${FIL_WE}.pb"
       VERBATIM )
   endforeach()

docs/Makefile

@@ -1,4 +1,4 @@
-all: index.html concepts.html reference.html \
+all: index.html concepts.html reference.html security.html \
 	generator_flow.png
 
 %.png: %.svg

docs/menu.rst

@@ -3,8 +3,9 @@
     1) `Overview`_
     2) `Concepts`_
     3) `API reference`_
+    4) `Security model`_
     
 .. _`Overview`: index.html
 .. _`Concepts`: concepts.html
 .. _`API reference`: reference.html
-
+.. _`Security model`: security.html

docs/reference.rst

@@ -25,7 +25,9 @@ __BIG_ENDIAN__                 Set this if your platform stores integers and
                                systems (different layout for ints and floats)
                                are currently not supported.
 NANOPB_INTERNALS               Set this to expose the field encoder functions
-                               that are hidden since nanopb-0.1.3.
+                               that are hidden since nanopb-0.1.3. Starting
+                               with nanopb-0.2.4, this flag does nothing. Use
+                               the newer functions that have better interface.
 PB_MAX_REQUIRED_FIELDS         Maximum number of required fields to check for
                                presence. Default value is 64. Increases stack
                                usage 1 byte per every 8 fields. Compiler

docs/security.rst

@@ -0,0 +1,79 @@
+======================
+Nanopb: Security model
+======================
+
+.. include :: menu.rst
+
+.. contents ::
+
+
+
+Importance of security in a Protocol Buffers library
+====================================================
+In the context of protocol buffers, security comes into play when decoding
+untrusted data. Naturally, if the attacker can modify the contents of a
+protocol buffers message, he can feed the application any values possible.
+Therefore the application itself must be prepared to receive untrusted values.
+
+Where nanopb plays a part is preventing the attacker from running arbitrary
+code on the target system. Mostly this means that there must not be any
+possibility to cause buffer overruns, memory corruption or invalid pointers
+by the means of crafting a malicious message.
+
+Division of trusted and untrusted data
+======================================
+The following data is regarded as **trusted**. It must be under the control of
+the application writer. Malicious data in these structures could cause
+security issues, such as execution of arbitrary code:
+
+1. Callback and extension fields in message structures given to pb_encode()
+   and pb_decode(). These fields are memory pointers, and are generated
+   depending on the .proto file.
+2. The automatically generated field definitions, i.e. *pb_field_t* lists.
+3. Contents of the *pb_istream_t* and *pb_ostream_t* structures (this does not
+   mean the contents of the stream itself, just the stream definition).
+
+The following data is regarded as **untrusted**. Invalid/malicious data in
+these will cause "garbage in, garbage out" behaviour. It will not cause
+buffer overflows, information disclosure or other security problems:
+
+1. All data read from *pb_istream_t*.
+2. All fields in message structures, except callbacks and extensions.
+   (Beginning with nanopb-0.2.4, in earlier versions the field sizes are partially unchecked.)
+
+Invariants
+==========
+The following invariants are maintained during operation, even if the
+untrusted data has been maliciously crafted:
+
+1. Nanopb will never read more than *bytes_left* bytes from *pb_istream_t*.
+2. Nanopb will never write more than *max_size* bytes to *pb_ostream_t*.
+3. Nanopb will never access memory out of bounds of the message structure.
+4. After pb_decode() returns successfully, the message structure will be
+   internally consistent:
+
+   - The *count* fields of arrays will not exceed the array size.
+   - The *size* field of bytes will not exceed the allocated size.
+   - All string fields will have null terminator.
+
+5. After pb_encode() returns successfully, the resulting message is a valid
+   protocol buffers message. (Except if user-defined callbacks write incorrect
+   data.)
+
+Further considerations
+======================
+Even if the nanopb library is free of any security issues, there are still
+several possible attack vectors that the application author must consider.
+The following list is not comprehensive:
+
+1. Stack usage may depend on the contents of the message. The message
+   definition places an upper bound on how much stack will be used. Tests
+   should be run with all fields present, to record the maximum possible
+   stack usage.
+2. Callbacks can do anything. The code for the callbacks must be carefully
+   checked if they are used with untrusted data.
+3. If using stream input, a maximum size should be set in *pb_istream_t* to
+   stop a denial of service attack from using an infinite message.
+4. If using network sockets as streams, a timeout should be set to stop
+   denial of service attacks.
+

generator/nanopb.proto

@@ -12,6 +12,7 @@ option java_package = "fi.kapsi.koti.jpa.nanopb";
 enum FieldType {
     FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible.
     FT_CALLBACK = 1; // Always generate a callback field.
+    FT_POINTER = 4; // Always generate a dynamically allocated field.
     FT_STATIC = 2; // Generate a static field or raise an exception if not possible.
     FT_IGNORE = 3; // Ignore the field completely.
 }

generator/nanopb_generator.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 
 '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.'''
-nanopb_version = "nanopb-0.2.3"
+nanopb_version = "nanopb-0.2.5-dev"
 
 try:
     import google.protobuf.descriptor_pb2 as descriptor
@@ -38,22 +38,22 @@ except:
 import time
 import os.path
 
-# Values are tuple (c type, pb type)
+# Values are tuple (c type, pb type, encoded size)
 FieldD = descriptor.FieldDescriptorProto
 datatypes = {
-    FieldD.TYPE_BOOL:       ('bool',     'BOOL'),
+    FieldD.TYPE_BOOL:       ('bool',     'BOOL',        1),
-    FieldD.TYPE_DOUBLE:     ('double',   'DOUBLE'),
+    FieldD.TYPE_DOUBLE:     ('double',   'DOUBLE',      8),
-    FieldD.TYPE_FIXED32:    ('uint32_t', 'FIXED32'),
+    FieldD.TYPE_FIXED32:    ('uint32_t', 'FIXED32',     4),
-    FieldD.TYPE_FIXED64:    ('uint64_t', 'FIXED64'),
+    FieldD.TYPE_FIXED64:    ('uint64_t', 'FIXED64',     8),
-    FieldD.TYPE_FLOAT:      ('float',    'FLOAT'),
+    FieldD.TYPE_FLOAT:      ('float',    'FLOAT',       4),
-    FieldD.TYPE_INT32:      ('int32_t',  'INT32'),
+    FieldD.TYPE_INT32:      ('int32_t',  'INT32',       5),
-    FieldD.TYPE_INT64:      ('int64_t',  'INT64'),
+    FieldD.TYPE_INT64:      ('int64_t',  'INT64',      10),
-    FieldD.TYPE_SFIXED32:   ('int32_t',  'SFIXED32'),
+    FieldD.TYPE_SFIXED32:   ('int32_t',  'SFIXED32',    4),
-    FieldD.TYPE_SFIXED64:   ('int64_t',  'SFIXED64'),
+    FieldD.TYPE_SFIXED64:   ('int64_t',  'SFIXED64',    8),
-    FieldD.TYPE_SINT32:     ('int32_t',  'SINT32'),
+    FieldD.TYPE_SINT32:     ('int32_t',  'SINT32',      5),
-    FieldD.TYPE_SINT64:     ('int64_t',  'SINT64'),
+    FieldD.TYPE_SINT64:     ('int64_t',  'SINT64',     10),
-    FieldD.TYPE_UINT32:     ('uint32_t', 'UINT32'),
+    FieldD.TYPE_UINT32:     ('uint32_t', 'UINT32',      5),
-    FieldD.TYPE_UINT64:     ('uint64_t', 'UINT64')
+    FieldD.TYPE_UINT64:     ('uint64_t', 'UINT64',     10)
 }
 
 class Names:
@@ -83,6 +83,55 @@ def names_from_type_name(type_name):
         raise NotImplementedError("Lookup of non-absolute type names is not supported")
     return Names(type_name[1:].split('.'))
 
+def varint_max_size(max_value):
+    '''Returns the maximum number of bytes a varint can take when encoded.'''
+    for i in range(1, 11):
+        if (max_value >> (i * 7)) == 0:
+            return i
+    raise ValueError("Value too large for varint: " + str(max_value))
+
+assert varint_max_size(0) == 1
+assert varint_max_size(127) == 1
+assert varint_max_size(128) == 2
+
+class EncodedSize:
+    '''Class used to represent the encoded size of a field or a message.
+    Consists of a combination of symbolic sizes and integer sizes.'''
+    def __init__(self, value = 0, symbols = []):
+        if isinstance(value, (str, Names)):
+            symbols = [str(value)]
+            value = 0
+        self.value = value
+        self.symbols = symbols
+    
+    def __add__(self, other):
+        if isinstance(other, (int, long)):
+            return EncodedSize(self.value + other, self.symbols)
+        elif isinstance(other, (str, Names)):
+            return EncodedSize(self.value, self.symbols + [str(other)])
+        elif isinstance(other, EncodedSize):
+            return EncodedSize(self.value + other.value, self.symbols + other.symbols)
+        else:
+            raise ValueError("Cannot add size: " + repr(other))
+
+    def __mul__(self, other):
+        if isinstance(other, (int, long)):
+            return EncodedSize(self.value * other, [str(other) + '*' + s for s in self.symbols])
+        else:
+            raise ValueError("Cannot multiply size: " + repr(other))
+
+    def __str__(self):
+        if not self.symbols:
+            return str(self.value)
+        else:
+            return '(' + str(self.value) + ' + ' + ' + '.join(self.symbols) + ')'
+
+    def upperlimit(self):
+        if not self.symbols:
+            return self.value
+        else:
+            return 2**32 - 1
+
 class Enum:
     def __init__(self, names, desc, enum_options):
         '''desc is EnumDescriptorProto'''
@@ -113,6 +162,7 @@ class Field:
         self.max_size = None
         self.max_count = None
         self.array_decl = ""
+        self.enc_size = None
         
         # Parse field options
         if field_options.HasField("max_size"):
@@ -139,33 +189,15 @@ class Field:
         else:
             raise NotImplementedError(desc.label)
         
-        # Decide the C data type to use in the struct.
+        # Check if the field can be implemented with static allocation
-        if datatypes.has_key(desc.type):
+        # i.e. whether the data size is known.
-            self.ctype, self.pbtype = datatypes[desc.type]
+        if desc.type == FieldD.TYPE_STRING and self.max_size is None:
-        elif desc.type == FieldD.TYPE_ENUM:
+            can_be_static = False
-            self.pbtype = 'ENUM'
-            self.ctype = names_from_type_name(desc.type_name)
-            if self.default is not None:
-                self.default = self.ctype + self.default
-        elif desc.type == FieldD.TYPE_STRING:
-            self.pbtype = 'STRING'
-            if self.max_size is None:
-                can_be_static = False
-            else:
-                self.ctype = 'char'
-                self.array_decl += '[%d]' % self.max_size
-        elif desc.type == FieldD.TYPE_BYTES:
-            self.pbtype = 'BYTES'
-            if self.max_size is None:
-                can_be_static = False
-            else:
-                self.ctype = self.struct_name + self.name + 't'
-        elif desc.type == FieldD.TYPE_MESSAGE:
-            self.pbtype = 'MESSAGE'
-            self.ctype = self.submsgname = names_from_type_name(desc.type_name)
-        else:
-            raise NotImplementedError(desc.type)
         
+        if desc.type == FieldD.TYPE_BYTES and self.max_size is None:
+            can_be_static = False
+        
+        # Decide how the field data will be allocated
         if field_options.type == nanopb_pb2.FT_DEFAULT:
             if can_be_static:
                 field_options.type = nanopb_pb2.FT_STATIC
@@ -173,28 +205,73 @@ class Field:
                 field_options.type = nanopb_pb2.FT_CALLBACK
         
         if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static:
-            raise Exception("Field %s is defined as static, but max_size or max_count is not given." % self.name)
+            raise Exception("Field %s is defined as static, but max_size or "
+                            "max_count is not given." % self.name)
         
         if field_options.type == nanopb_pb2.FT_STATIC:
             self.allocation = 'STATIC'
+        elif field_options.type == nanopb_pb2.FT_POINTER:
+            self.allocation = 'POINTER'
         elif field_options.type == nanopb_pb2.FT_CALLBACK:
             self.allocation = 'CALLBACK'
-            self.ctype = 'pb_callback_t'
-            self.array_decl = ''
         else:
             raise NotImplementedError(field_options.type)
         
+        # Decide the C data type to use in the struct.
+        if datatypes.has_key(desc.type):
+            self.ctype, self.pbtype, self.enc_size = datatypes[desc.type]
+        elif desc.type == FieldD.TYPE_ENUM:
+            self.pbtype = 'ENUM'
+            self.ctype = names_from_type_name(desc.type_name)
+            if self.default is not None:
+                self.default = self.ctype + self.default
+            self.enc_size = 5 # protoc rejects enum values > 32 bits
+        elif desc.type == FieldD.TYPE_STRING:
+            self.pbtype = 'STRING'
+            self.ctype = 'char'
+            if self.allocation == 'STATIC':
+                self.ctype = 'char'
+                self.array_decl += '[%d]' % self.max_size
+                self.enc_size = varint_max_size(self.max_size) + self.max_size
+        elif desc.type == FieldD.TYPE_BYTES:
+            self.pbtype = 'BYTES'
+            if self.allocation == 'STATIC':
+                self.ctype = self.struct_name + self.name + 't'
+                self.enc_size = varint_max_size(self.max_size) + self.max_size
+            elif self.allocation == 'POINTER':
+                self.ctype = 'pb_bytes_ptr_t'
+        elif desc.type == FieldD.TYPE_MESSAGE:
+            self.pbtype = 'MESSAGE'
+            self.ctype = self.submsgname = names_from_type_name(desc.type_name)
+            self.enc_size = None # Needs to be filled in after the message type is available
+        else:
+            raise NotImplementedError(desc.type)
+        
     def __cmp__(self, other):
         return cmp(self.tag, other.tag)
     
     def __str__(self):
-        if self.rules == 'OPTIONAL' and self.allocation == 'STATIC':
+        result = ''
-            result = '    bool has_' + self.name + ';\n'
+        if self.allocation == 'POINTER':
-        elif self.rules == 'REPEATED' and self.allocation == 'STATIC':
+            if self.rules == 'REPEATED':
-            result = '    size_t ' + self.name + '_count;\n'
+                result += '    size_t ' + self.name + '_count;\n'
+            
+            if self.pbtype == 'MESSAGE':
+                # Use struct definition, so recursive submessages are possible
+                result += '    struct _%s *%s;' % (self.ctype, self.name)
+            elif self.rules == 'REPEATED' and self.pbtype == 'STRING':
+                # String arrays need to be defined as pointers to pointers
+                result += '    %s **%s;' % (self.ctype, self.name)
+            else:
+                result += '    %s *%s;' % (self.ctype, self.name)
+        elif self.allocation == 'CALLBACK':
+            result += '    pb_callback_t %s;' % self.name
         else:
-            result = ''
+            if self.rules == 'OPTIONAL' and self.allocation == 'STATIC':
-        result += '    %s %s%s;' % (self.ctype, self.name, self.array_decl)
+                result += '    bool has_' + self.name + ';\n'
+            elif self.rules == 'REPEATED' and self.allocation == 'STATIC':
+                result += '    size_t ' + self.name + '_count;\n'
+            result += '    %s %s%s;' % (self.ctype, self.name, self.array_decl)
         return result
     
     def types(self):
@@ -249,7 +326,7 @@ class Field:
         result  = '    PB_FIELD2(%3d, ' % self.tag
         result += '%-8s, ' % self.pbtype
         result += '%s, ' % self.rules
-        result += '%s, ' % self.allocation
+        result += '%-8s, ' % self.allocation
         result += '%s, ' % ("FIRST" if not prev_field_name else "OTHER")
         result += '%s, ' % self.struct_name
         result += '%s, ' % self.name
@@ -277,6 +354,51 @@ class Field:
 
         return max(self.tag, self.max_size, self.max_count)        
 
+    def encoded_size(self, allmsgs):
+        '''Return the maximum size that this field can take when encoded,
+        including the field tag. If the size cannot be determined, returns
+        None.'''
+        
+        if self.allocation != 'STATIC':
+            return None
+        
+        if self.pbtype == 'MESSAGE':
+            for msg in allmsgs:
+                if msg.name == self.submsgname:
+                    encsize = msg.encoded_size(allmsgs)
+                    if encsize is None:
+                        return None # Submessage size is indeterminate
+                        
+                    # Include submessage length prefix
+                    encsize += varint_max_size(encsize.upperlimit())
+                    break
+            else:
+                # Submessage cannot be found, this currently occurs when
+                # the submessage type is defined in a different file.
+                # Instead of direct numeric value, reference the size that
+                # has been #defined in the other file.
+                encsize = EncodedSize(self.submsgname + 'size')
+
+                # We will have to make a conservative assumption on the length
+                # prefix size, though.
+                encsize += 5
+
+        elif self.enc_size is None:
+            raise RuntimeError("Could not determine encoded size for %s.%s"
+                               % (self.struct_name, self.name))
+        else:
+            encsize = EncodedSize(self.enc_size)
+        
+        encsize += varint_max_size(self.tag << 3) # Tag + wire type
+
+        if self.rules == 'REPEATED':
+            # Decoders must be always able to handle unpacked arrays.
+            # Therefore we have to reserve space for it, even though
+            # we emit packed arrays ourselves.
+            encsize *= self.max_count
+        
+        return encsize
+
 
 class ExtensionRange(Field):
     def __init__(self, struct_name, range_start, field_options):
@@ -306,6 +428,12 @@ class ExtensionRange(Field):
     def tags(self):
         return ''
         
+    def encoded_size(self, allmsgs):
+        # We exclude extensions from the count, because they cannot be known
+        # until runtime. Other option would be to return None here, but this
+        # way the value remains useful if extensions are not used.
+        return EncodedSize(0)
+
 class ExtensionField(Field):
     def __init__(self, struct_name, desc, field_options):
         self.fullname = struct_name + desc.name
@@ -318,6 +446,11 @@ class ExtensionField(Field):
             self.skip = False
             self.rules = 'OPTEXT'
 
+    def tags(self):
+        '''Return the #define for the tag number of this field.'''
+        identifier = '%s_tag' % self.fullname
+        return '#define %-40s %d\n' % (identifier, self.tag)
+
     def extension_decl(self):
         '''Declaration of the extension type in the .pb.h file'''
         if self.skip:
@@ -429,6 +562,18 @@ class Message:
         result += '    PB_LAST_FIELD\n};'
         return result
 
+    def encoded_size(self, allmsgs):
+        '''Return the maximum size that this message can take when encoded.
+        If the size cannot be determined, returns None.
+        '''
+        size = EncodedSize(0)
+        for field in self.fields:
+            fsize = field.encoded_size(allmsgs)
+            if fsize is None:
+                return None
+            size += fsize
+        
+        return size
 
 
 # ---------------------------------------------------------------------------
@@ -592,13 +737,24 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio
     for msg in sort_dependencies(messages):
         for field in msg.fields:
             yield field.tags()
+    for extension in extensions:
+        yield extension.tags()
     yield '\n'
     
     yield '/* Struct field encoding specification for nanopb */\n'
     for msg in messages:
         yield msg.fields_declaration() + '\n'
+    yield '\n'
     
-    yield '\n#ifdef __cplusplus\n'
+    yield '/* Maximum encoded size of messages (where known) */\n'
+    for msg in messages:
+        msize = msg.encoded_size(messages)
+        if msize is not None:
+            identifier = '%s_size' % msg.name
+            yield '#define %-40s %s\n' % (identifier, msize)
+    yield '\n'
+    
+    yield '#ifdef __cplusplus\n'
     yield '} /* extern "C" */\n'
     yield '#endif\n'
     

generator/nanopb_pb2.py

@@ -12,7 +12,7 @@ import google.protobuf.descriptor_pb2
 DESCRIPTOR = descriptor.FileDescriptor(
   name='nanopb.proto',
   package='',
-  serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*J\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions')
+  serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*Z\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\x0e\n\nFT_POINTER\x10\x04\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptionsB\x1a\n\x18\x66i.kapsi.koti.jpa.nanopb')
 
 _FIELDTYPE = descriptor.EnumDescriptor(
   name='FieldType',
@@ -29,23 +29,28 @@ _FIELDTYPE = descriptor.EnumDescriptor(
       options=None,
       type=None),
     descriptor.EnumValueDescriptor(
-      name='FT_STATIC', index=2, number=2,
+      name='FT_POINTER', index=2, number=4,
       options=None,
       type=None),
     descriptor.EnumValueDescriptor(
-      name='FT_IGNORE', index=3, number=3,
+      name='FT_STATIC', index=3, number=2,
+      options=None,
+      type=None),
+    descriptor.EnumValueDescriptor(
+      name='FT_IGNORE', index=4, number=3,
       options=None,
       type=None),
   ],
   containing_type=None,
   options=None,
   serialized_start=199,
-  serialized_end=273,
+  serialized_end=289,
 )
 
 
 FT_DEFAULT = 0
 FT_CALLBACK = 1
+FT_POINTER = 4
 FT_STATIC = 2
 FT_IGNORE = 3
 

pb.h

@@ -43,7 +43,7 @@
 
 /* Version of the nanopb library. Just in case you want to check it in
  * your own program. */
-#define NANOPB_VERSION nanopb-0.2.3
+#define NANOPB_VERSION nanopb-0.2.5-dev
 
 /* Include all the system headers needed by nanopb. You will need the
  * definitions of the following:
@@ -166,6 +166,7 @@ typedef uint8_t pb_type_t;
 /**** Field allocation types ****/
  
 #define PB_ATYPE_STATIC   0x00
+#define PB_ATYPE_POINTER  0x80
 #define PB_ATYPE_CALLBACK 0x40
 #define PB_ATYPE_MASK     0xC0
 
@@ -173,6 +174,20 @@ typedef uint8_t pb_type_t;
 #define PB_HTYPE(x) ((x) & PB_HTYPE_MASK)
 #define PB_LTYPE(x) ((x) & PB_LTYPE_MASK)
 
+/* Data type used for storing sizes of struct fields
+ * and array counts.
+ */
+#if defined(PB_FIELD_32BIT)
+    typedef uint32_t pb_size_t;
+    typedef int32_t pb_ssize_t;
+#elif defined(PB_FIELD_16BIT)
+    typedef uint16_t pb_size_t;
+    typedef int16_t pb_ssize_t;
+#else
+    typedef uint8_t pb_size_t;
+    typedef int8_t pb_ssize_t;
+#endif
+
 /* This structure is used in auto-generated constants
  * to specify struct fields.
  * You can change field sizes if you need structures
@@ -184,29 +199,12 @@ typedef uint8_t pb_type_t;
 PB_PACKED_STRUCT_START
 typedef struct _pb_field_t pb_field_t;
 struct _pb_field_t {
-
+    pb_size_t tag;
-#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
-    uint8_t tag;
     pb_type_t type;
-    uint8_t data_offset; /* Offset of field data, relative to previous field. */
+    pb_size_t data_offset; /* Offset of field data, relative to previous field. */
-    int8_t size_offset; /* Offset of array size or has-boolean, relative to data */
+    pb_ssize_t size_offset; /* Offset of array size or has-boolean, relative to data */
-    uint8_t data_size; /* Data size in bytes for a single item */
+    pb_size_t data_size; /* Data size in bytes for a single item */
-    uint8_t array_size; /* Maximum number of entries in array */
+    pb_size_t array_size; /* Maximum number of entries in array */
-#elif defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
-    uint16_t tag;
-    pb_type_t type;
-    uint8_t data_offset;
-    int8_t size_offset;
-    uint16_t data_size;
-    uint16_t array_size;
-#else
-    uint32_t tag;
-    pb_type_t type;
-    uint8_t data_offset;
-    int8_t size_offset;
-    uint32_t data_size;
-    uint32_t array_size;
-#endif
     
     /* Field definitions for submessage
      * OR default value for all other non-array, non-callback types
@@ -234,9 +232,17 @@ struct _pb_bytes_array_t {
     size_t size;
     uint8_t bytes[1];
 };
-
 typedef struct _pb_bytes_array_t pb_bytes_array_t;
 
+/* Same, except for pointer-type fields. There is no need to variable struct
+ * length in this case.
+ */
+struct _pb_bytes_ptr_t {
+    size_t size;
+    uint8_t *bytes;
+};
+typedef struct _pb_bytes_ptr_t pb_bytes_ptr_t;
+
 /* This structure is used for giving the callback function.
  * It is stored in the message structure and filled in by the method that
  * calls pb_decode.
@@ -373,6 +379,22 @@ struct _pb_extension_t {
     pb_membersize(st, m[0]), \
     pb_arraysize(st, m), ptr}
 
+/* Allocated fields carry the size of the actual data, not the pointer */
+#define PB_REQUIRED_POINTER(tag, st, m, fd, ltype, ptr) \
+    {tag, PB_ATYPE_POINTER | PB_HTYPE_REQUIRED | ltype, \
+    fd, 0, pb_membersize(st, m[0]), 0, ptr}
+
+/* Optional fields don't need a has_ variable, as information would be redundant */
+#define PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) \
+    {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \
+    fd, 0, pb_membersize(st, m[0]), 0, ptr}
+
+/* Repeated fields have a _count field and a pointer to array of pointers */
+#define PB_REPEATED_POINTER(tag, st, m, fd, ltype, ptr) \
+    {tag, PB_ATYPE_POINTER | PB_HTYPE_REPEATED | ltype, \
+    fd, pb_delta(st, m ## _count, m), \
+    pb_membersize(st, m[0]), 0, ptr}
+
 /* Callbacks are much like required fields except with special datatype. */
 #define PB_REQUIRED_CALLBACK(tag, st, m, fd, ltype, ptr) \
     {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REQUIRED | ltype, \

pb_decode.c

@@ -3,11 +3,13 @@
  * 2011 Petteri Aimonen <jpa@kapsi.fi>
  */
 
-/* The warn_unused_result attribute appeared first in gcc-3.4.0 */
+/* Use the GCC warn_unused_result attribute to check that all return values
+ * are propagated correctly. On other compilers and gcc before 3.4.0 just
+ * ignore the annotation.
+ */
 #if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4)
     #define checkreturn
 #else
-    /* Verify that we remember to check all return values for proper error propagation */
     #define checkreturn __attribute__((warn_unused_result))
 #endif
 
@@ -15,8 +17,46 @@
 #include "pb.h"
 #include "pb_decode.h"
 
+/**************************************
+ * Declarations internal to this file *
+ **************************************/
+
+/* Iterator for pb_field_t list */
+typedef struct {
+    const pb_field_t *start; /* Start of the pb_field_t array */
+    const pb_field_t *pos; /* Current position of the iterator */
+    unsigned field_index; /* Zero-based index of the field. */
+    unsigned required_field_index; /* Zero-based index that counts only the required fields */
+    void *dest_struct; /* Pointer to the destination structure to decode to */
+    void *pData; /* Pointer where to store current field value */
+    void *pSize; /* Pointer where to store the size of current array field */
+} pb_field_iterator_t;
+
 typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn;
 
+static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count);
+static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest);
+static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8_t *buf, size_t *size);
+static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct);
+static bool pb_field_next(pb_field_iterator_t *iter);
+static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag);
+static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
+static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
+static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
+static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);
+static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter);
+static bool checkreturn find_extension_field(pb_field_iterator_t *iter);
+static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct);
+static bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest);
+static bool checkreturn pb_skip_varint(pb_istream_t *stream);
+static bool checkreturn pb_skip_string(pb_istream_t *stream);
+
 /* --- Function pointers to field decoders ---
  * Order in the array must match pb_action_t LTYPE numbering.
  */
@@ -32,9 +72,9 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = {
     NULL /* extensions */
 };
 
-/**************
+/*******************************
- * pb_istream *
+ * pb_istream_t implementation *
- **************/
+ *******************************/
 
 static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count)
 {
@@ -84,6 +124,26 @@ bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count)
     return true;
 }
 
+/* Read a single byte from input stream. buf may not be NULL.
+ * This is an optimization for the varint decoding. */
+static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf)
+{
+    if (!stream->bytes_left)
+        PB_RETURN_ERROR(stream, "end-of-stream");
+
+#ifndef PB_BUFFER_ONLY
+    if (!stream->callback(stream, buf, 1))
+        PB_RETURN_ERROR(stream, "io error");
+#else
+    *buf = *(uint8_t*)stream->state;
+    stream->state = (uint8_t*)stream->state + 1;
+#endif
+
+    stream->bytes_left--;
+    
+    return true;    
+}
+
 pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize)
 {
     pb_istream_t stream;
@@ -109,7 +169,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest)
     uint8_t byte;
     uint32_t result;
     
-    if (!pb_read(stream, &byte, 1))
+    if (!pb_readbyte(stream, &byte))
         return false;
     
     if (!(byte & 0x80))
@@ -128,7 +188,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest)
             if (bitpos >= 32)
                 PB_RETURN_ERROR(stream, "varint overflow");
             
-            if (!pb_read(stream, &byte, 1))
+            if (!pb_readbyte(stream, &byte))
                 return false;
             
             result |= (uint32_t)(byte & 0x7F) << bitpos;
@@ -151,7 +211,7 @@ bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest)
         if (bitpos >= 64)
             PB_RETURN_ERROR(stream, "varint overflow");
         
-        if (!pb_read(stream, &byte, 1))
+        if (!pb_readbyte(stream, &byte))
             return false;
 
         result |= (uint64_t)(byte & 0x7F) << bitpos;
@@ -277,17 +337,6 @@ void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream)
 #endif
 }
 
-/* Iterator for pb_field_t list */
-typedef struct {
-    const pb_field_t *start; /* Start of the pb_field_t array */
-    const pb_field_t *pos; /* Current position of the iterator */
-    unsigned field_index; /* Zero-based index of the field. */
-    unsigned required_field_index; /* Zero-based index that counts only the required fields */
-    void *dest_struct; /* Pointer to the destination structure to decode to */
-    void *pData; /* Pointer where to store current field value */
-    void *pSize; /* Pointer where to store the size of current array field */
-} pb_field_iterator_t;
-
 static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct)
 {
     iter->start = iter->pos = fields;
@@ -478,7 +527,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
 
 /* Default handler for extension fields. Expects a pb_field_t structure
  * in extension->type->arg. */
-static bool checkreturn default_extension_handler(pb_istream_t *stream,
+static bool checkreturn default_extension_decoder(pb_istream_t *stream,
     pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type)
 {
     const pb_field_t *field = (const pb_field_t*)extension->type->arg;
@@ -513,7 +562,7 @@ static bool checkreturn decode_extension(pb_istream_t *stream,
         if (extension->type->decode)
             status = extension->type->decode(stream, extension, tag, wire_type);
         else
-            status = default_extension_handler(stream, extension, tag, wire_type);
+            status = default_extension_decoder(stream, extension, tag, wire_type);
 
         if (!status)
             return false;

pb_decode.h

@@ -131,25 +131,6 @@ bool pb_decode_fixed64(pb_istream_t *stream, void *dest);
 bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream);
 void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream);
 
-
-/*******************************
- * Internal / legacy functions *
- *******************************/
-
-#ifdef NANOPB_INTERNALS
-bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest);
-bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest);
-bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest);
-bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest);
-
-bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest);
-bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest);
-bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest);
-
-bool pb_skip_varint(pb_istream_t *stream);
-bool pb_skip_string(pb_istream_t *stream);
-#endif
-
 #ifdef __cplusplus
 } /* extern "C" */
 #endif

pb_encode.c

@@ -7,16 +7,34 @@
 #include "pb.h"
 #include "pb_encode.h"
 
-/* The warn_unused_result attribute appeared first in gcc-3.4.0 */
+/* Use the GCC warn_unused_result attribute to check that all return values
+ * are propagated correctly. On other compilers and gcc before 3.4.0 just
+ * ignore the annotation.
+ */
 #if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4)
     #define checkreturn
 #else
-    /* Verify that we remember to check all return values for proper error propagation */
     #define checkreturn __attribute__((warn_unused_result))
 #endif
 
+/**************************************
+ * Declarations internal to this file *
+ **************************************/
 typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src) checkreturn;
 
+static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count);
+static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, const void *pData, size_t count, pb_encoder_t func);
+static bool checkreturn encode_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData);
+static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension);
+static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData);
+static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src);
+
 /* --- Function pointers to field encoders ---
  * Order in the array must match pb_action_t LTYPE numbering.
  */
@@ -32,7 +50,9 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = {
     NULL /* extensions */
 };
 
-/* pb_ostream_t implementation */
+/*******************************
+ * pb_ostream_t implementation *
+ *******************************/
 
 static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count)
 {
@@ -49,7 +69,7 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize)
 {
     pb_ostream_t stream;
 #ifdef PB_BUFFER_ONLY
-    stream.callback = (void*)1; /* Just some marker value */
+    stream.callback = (void*)1; /* Just a marker value */
 #else
     stream.callback = &buf_write;
 #endif
@@ -82,7 +102,9 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count
     return true;
 }
 
-/* Main encoding stuff */
+/*************************
+ * Encode a single field *
+ *************************/
 
 /* Encode a static array. Handles the size calculations and possible packing. */
 static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field,
@@ -95,6 +117,9 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
     if (count == 0)
         return true;
 
+    if (PB_ATYPE(field->type) != PB_ATYPE_POINTER && count > field->array_size)
+        PB_RETURN_ERROR(stream, "array max size exceeded");
+    
     /* We always pack arrays if the datatype allows it. */
     if (PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE)
     {
@@ -145,8 +170,22 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
         {
             if (!pb_encode_tag_for_field(stream, field))
                 return false;
-            if (!func(stream, field, p))
+
-                return false;
+            /* Normally the data is stored directly in the array entries, but
+             * for pointer-type string fields, the array entries are actually
+             * string pointers. So we have to dereference once more to get to
+             * the character data. */
+            if (PB_ATYPE(field->type) == PB_ATYPE_POINTER &&
+                PB_LTYPE(field->type) == PB_LTYPE_STRING)
+            {
+                if (!func(stream, field, *(const void* const*)p))
+                    return false;      
+            }
+            else
+            {
+                if (!func(stream, field, p))
+                    return false;
+            }
             p = (const char*)p + field->data_size;
         }
     }
@@ -154,25 +193,38 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
     return true;
 }
 
-/* Encode a field with static allocation, i.e. one whose data is stored
+/* Encode a field with static or pointer allocation, i.e. one whose data
- * in the structure itself. */
+ * is available to the encoder directly. */
-static bool checkreturn encode_static_field(pb_ostream_t *stream,
+static bool checkreturn encode_basic_field(pb_ostream_t *stream,
     const pb_field_t *field, const void *pData)
 {
     pb_encoder_t func;
     const void *pSize;
-    bool dummy = true;
+    bool implicit_has = true;
     
     func = PB_ENCODERS[PB_LTYPE(field->type)];
     
     if (field->size_offset)
         pSize = (const char*)pData + field->size_offset;
     else
-        pSize = &dummy;
+        pSize = &implicit_has;
+
+    if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
+    {
+        /* pData is a pointer to the field, which contains pointer to
+         * the data. If the 2nd pointer is NULL, it is interpreted as if
+         * the has_field was false.
+         */
+        
+        pData = *(const void* const*)pData;
+        implicit_has = (pData != NULL);
+    }
 
     switch (PB_HTYPE(field->type))
     {
         case PB_HTYPE_REQUIRED:
+            if (!pData)
+                PB_RETURN_ERROR(stream, "missing required field");
             if (!pb_encode_tag_for_field(stream, field))
                 return false;
             if (!func(stream, field, pData))
@@ -230,7 +282,8 @@ static bool checkreturn encode_field(pb_ostream_t *stream,
     switch (PB_ATYPE(field->type))
     {
         case PB_ATYPE_STATIC:
-            return encode_static_field(stream, field, pData);
+        case PB_ATYPE_POINTER:
+            return encode_basic_field(stream, field, pData);
         
         case PB_ATYPE_CALLBACK:
             return encode_callback_field(stream, field, pData);
@@ -242,7 +295,7 @@ static bool checkreturn encode_field(pb_ostream_t *stream,
 
 /* Default handler for extension fields. Expects to have a pb_field_t
  * pointer in the extension->type->arg field. */
-static bool checkreturn default_extension_handler(pb_ostream_t *stream,
+static bool checkreturn default_extension_encoder(pb_ostream_t *stream,
     const pb_extension_t *extension)
 {
     const pb_field_t *field = (const pb_field_t*)extension->type->arg;
@@ -263,7 +316,7 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream,
         if (extension->type->encode)
             status = extension->type->encode(stream, extension);
         else
-            status = default_extension_handler(stream, extension);
+            status = default_extension_encoder(stream, extension);
 
         if (!status)
             return false;
@@ -274,6 +327,10 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream,
     return true;
 }
 
+/*********************
+ * Encode all fields *
+ *********************/
+
 bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct)
 {
     const pb_field_t *field = fields;
@@ -283,7 +340,10 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons
     while (field->tag != 0)
     {
         pData = (const char*)pData + prev_size + field->data_offset;
-        prev_size = field->data_size;
+        if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
+            prev_size = sizeof(const void*);
+        else
+            prev_size = field->data_size;
         
         /* Special case for static arrays */
         if (PB_ATYPE(field->type) == PB_ATYPE_STATIC &&
@@ -316,7 +376,9 @@ bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const
     return pb_encode_submessage(stream, fields, src_struct);
 }
 
-/* Helper functions */
+/********************
+ * Helper functions *
+ ********************/
 bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value)
 {
     uint8_t buffer[10];
@@ -434,7 +496,12 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie
     bool status;
     
     if (!pb_encode(&substream, fields, src_struct))
+    {
+#ifndef PB_NO_ERRMSG
+        stream->errmsg = substream.errmsg;
+#endif
         return false;
+    }
     
     size = substream.bytes_written;
     
@@ -517,17 +584,32 @@ bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, c
 
 bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
-    const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src;
+    if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
-    UNUSED(field);
+    {
-    return pb_encode_string(stream, bytes->bytes, bytes->size);
+        const pb_bytes_ptr_t *bytes = (const pb_bytes_ptr_t*)src;
+        return pb_encode_string(stream, bytes->bytes, bytes->size);
+    }
+    else
+    {
+        const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src;
+        if (bytes->size + offsetof(pb_bytes_array_t, bytes) > field->data_size)
+            PB_RETURN_ERROR(stream, "bytes size exceeded");
+
+        return pb_encode_string(stream, bytes->bytes, bytes->size);
+    }
 }
 
 bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
-    /* strnlen() is not always available, so just use a for-loop */
+    /* strnlen() is not always available, so just use a loop */
     size_t size = 0;
+    size_t max_size = field->data_size;
     const char *p = (const char*)src;
-    while (size < field->data_size && *p != '\0')
+    
+    if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
+        max_size = (size_t)-1;
+
+    while (size < max_size && *p != '\0')
     {
         size++;
         p++;

pb_encode.h

@@ -143,24 +143,6 @@ bool pb_encode_fixed64(pb_ostream_t *stream, const void *value);
  */
 bool pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct);
 
-
-/*******************************
- * Internal / legacy functions *
- *******************************/
-
-#ifdef NANOPB_INTERNALS
-bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-bool pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-bool pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-#endif
-
-/* This function is not recommended for new programs. Use pb_encode_submessage()
- * instead, it has the same functionality with a less confusing interface. */
-bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src);
-
 #ifdef __cplusplus
 } /* extern "C" */
 #endif

tests/SConstruct

@@ -26,7 +26,7 @@ if 'CXXFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CXXFLAGS'])
 add_nanopb_builders(env)
 
 # Path to the files shared by tests, and to the nanopb core.
-env.Append(CPPPATH = ["#../", "#common"])
+env.Append(CPPPATH = ["#../", "$COMMON"])
 
 # Path for finding nanopb.proto
 env.Append(PROTOCPATH = '#../generator')
@@ -78,6 +78,8 @@ if 'gcc' in env['CC']:
 
     # More strict checks on the nanopb core
     env.Append(CORECFLAGS = '-Wextra -Wcast-qual -Wlogical-op -Wconversion')
+    env.Append(CORECFLAGS = ' -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls')
+    env.Append(CORECFLAGS = ' -Wstack-protector')
 elif 'clang' in env['CC']:
     # CLang
     env.Append(CFLAGS = '-ansi -g -O0 -Wall -Werror')
@@ -108,5 +110,10 @@ elif 'cl' in env['CXX']:
     env.Append(CXXFLAGS = '/Zi /W2 /WX')
     
 # Now include the SConscript files from all subdirectories
-SConscript(Glob('*/SConscript'), exports = 'env')
+import os.path
+env['VARIANT_DIR'] = 'build'
+env['BUILD'] = '#' + env['VARIANT_DIR']
+env['COMMON'] = '#' + env['VARIANT_DIR'] + '/common'
+for subdir in Glob('*/SConscript'):
+    SConscript(subdir, exports = 'env', variant_dir = env['VARIANT_DIR'] + '/' + os.path.dirname(str(subdir)))
 

tests/alltypes/SConscript

@@ -3,9 +3,9 @@
 
 Import("env")
 
-env.NanopbProto("alltypes")
+env.NanopbProto(["alltypes", "alltypes.options"])
-enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "#common/pb_encode.o"])
+enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o"])
-dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "#common/pb_decode.o"])
+dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_alltypes.output"])

tests/alltypes/encode_alltypes.c

@@ -115,7 +115,7 @@ int main(int argc, char **argv)
     alltypes.end = 1099;
     
     {
-        uint8_t buffer[1024];
+        uint8_t buffer[AllTypes_size];
         pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
         
         /* Now encode it and check if we succeeded. */

tests/alltypes_pointer/SConscript

@@ -0,0 +1,18 @@
+# Encode the AllTypes message using pointers for all fields, and verify the
+# output against the normal AllTypes test case.
+
+Import("env")
+
+env.NanopbProto(["alltypes", "alltypes.options"])
+enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "$COMMON/pb_encode.o"])
+
+# Encode and compare results
+env.RunTest(enc)
+env.RunTest("decode_alltypes.output", ["$BUILD/alltypes/decode_alltypes", "encode_alltypes_pointer.output"])
+env.Compare(["encode_alltypes_pointer.output", "$BUILD/alltypes/encode_alltypes.output"])
+
+# Do the same thing with the optional fields present
+#env.RunTest("optionals.output", enc, ARGS = ['1'])
+#env.RunTest("optionals.decout", ["$BUILD/alltypes/decode_alltypes", "optionals.output"], ARGS = ['1'])
+#env.Compare(["optionals.output", "$BUILD/alltypes/optionals.output"])
+

tests/alltypes_pointer/alltypes.options

@@ -0,0 +1,3 @@
+# Generate all fields as pointers.
+* type:FT_POINTER
+

tests/alltypes_pointer/alltypes.proto

@@ -0,0 +1,93 @@
+message SubMessage {
+    required string substuff1 = 1 [default = "1"];
+    required int32 substuff2 = 2 [default = 2];
+    optional fixed32 substuff3 = 3 [default = 3];
+}
+
+message EmptyMessage {
+
+}
+
+enum MyEnum {
+    Zero = 0;
+    First = 1;
+    Second = 2;
+    Truth = 42;
+}
+
+message AllTypes {
+    required int32      req_int32   = 1;
+    required int64      req_int64   = 2;
+    required uint32     req_uint32  = 3;
+    required uint64     req_uint64  = 4;
+    required sint32     req_sint32  = 5;
+    required sint64     req_sint64  = 6;
+    required bool       req_bool    = 7;
+    
+    required fixed32    req_fixed32 = 8;
+    required sfixed32   req_sfixed32= 9;
+    required float      req_float   = 10;
+    
+    required fixed64    req_fixed64 = 11;
+    required sfixed64   req_sfixed64= 12;
+    required double     req_double  = 13;
+    
+    required string     req_string  = 14;
+    required bytes      req_bytes   = 15;
+    required SubMessage req_submsg  = 16;
+    required MyEnum     req_enum    = 17;
+    required EmptyMessage req_emptymsg = 18;
+    
+    
+    repeated int32      rep_int32   = 21;
+    repeated int64      rep_int64   = 22;
+    repeated uint32     rep_uint32  = 23;
+    repeated uint64     rep_uint64  = 24;
+    repeated sint32     rep_sint32  = 25;
+    repeated sint64     rep_sint64  = 26;
+    repeated bool       rep_bool    = 27;
+    
+    repeated fixed32    rep_fixed32 = 28;
+    repeated sfixed32   rep_sfixed32= 29;
+    repeated float      rep_float   = 30;
+    
+    repeated fixed64    rep_fixed64 = 31;
+    repeated sfixed64   rep_sfixed64= 32;
+    repeated double     rep_double  = 33;
+    
+    repeated string     rep_string  = 34;
+    repeated bytes      rep_bytes   = 35;
+    repeated SubMessage rep_submsg  = 36;
+    repeated MyEnum     rep_enum    = 37;
+    repeated EmptyMessage rep_emptymsg = 38;
+    
+    optional int32      opt_int32   = 41 [default = 4041];
+    optional int64      opt_int64   = 42 [default = 4042];
+    optional uint32     opt_uint32  = 43 [default = 4043];
+    optional uint64     opt_uint64  = 44 [default = 4044];
+    optional sint32     opt_sint32  = 45 [default = 4045];
+    optional sint64     opt_sint64  = 46 [default = 4046];
+    optional bool       opt_bool    = 47 [default = false];
+    
+    optional fixed32    opt_fixed32 = 48 [default = 4048];
+    optional sfixed32   opt_sfixed32= 49 [default = 4049];
+    optional float      opt_float   = 50 [default = 4050];
+    
+    optional fixed64    opt_fixed64 = 51 [default = 4051];
+    optional sfixed64   opt_sfixed64= 52 [default = 4052];
+    optional double     opt_double  = 53 [default = 4053];
+    
+    optional string     opt_string  = 54 [default = "4054"];
+    optional bytes      opt_bytes   = 55 [default = "4055"];
+    optional SubMessage opt_submsg  = 56;
+    optional MyEnum     opt_enum    = 57 [default = Second];
+    optional EmptyMessage opt_emptymsg = 58;
+
+    // Just to make sure that the size of the fields has been calculated
+    // properly, i.e. otherwise a bug in last field might not be detected.
+    required int32      end = 99;
+
+
+    extensions 200 to 255;
+}
+

tests/alltypes_pointer/encode_alltypes_pointer.c

@@ -0,0 +1,169 @@
+/* Attempts to test all the datatypes supported by ProtoBuf.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pb_encode.h>
+#include "alltypes.pb.h"
+#include "test_helpers.h"
+
+int main(int argc, char **argv)
+{
+    int mode = (argc > 1) ? atoi(argv[1]) : 0;
+    
+    /* Values for required fields */
+    int32_t     req_int32         = -1001;
+    int64_t     req_int64         = -1002;
+    uint32_t    req_uint32        = 1003;
+    uint64_t    req_uint64        = 1004;
+    int32_t     req_sint32        = -1005;
+    int64_t     req_sint64        = -1006;
+    bool        req_bool          = true;
+    uint32_t    req_fixed32       = 1008;
+    int32_t     req_sfixed32      = -1009;
+    float       req_float         = 1010.0f;
+    uint64_t    req_fixed64       = 1011;
+    int64_t     req_sfixed64      = -1012;
+    double      req_double        = 1013.0;
+    char*       req_string        = "1014";
+    pb_bytes_ptr_t req_bytes      = {4, (uint8_t*)"1015"};
+    static int32_t req_substuff   = 1016;
+    SubMessage  req_submsg        = {"1016", &req_substuff};
+    MyEnum      req_enum          = MyEnum_Truth;
+    EmptyMessage req_emptymsg     = {0};
+    
+    int32_t     end               = 1099;
+
+    /* Values for repeated fields */
+    int32_t     rep_int32[5]      = {0, 0, 0, 0, -2001};
+    int64_t     rep_int64[5]      = {0, 0, 0, 0, -2002};
+    uint32_t    rep_uint32[5]     = {0, 0, 0, 0, 2003};
+    uint64_t    rep_uint64[5]     = {0, 0, 0, 0, 2004};
+    int32_t     rep_sint32[5]     = {0, 0, 0, 0, -2005};
+    int64_t     rep_sint64[5]     = {0, 0, 0, 0, -2006};
+    bool        rep_bool[5]       = {false, false, false, false, true};
+    uint32_t    rep_fixed32[5]    = {0, 0, 0, 0, 2008};
+    int32_t     rep_sfixed32[5]   = {0, 0, 0, 0, -2009};
+    float       rep_float[5]      = {0, 0, 0, 0, 2010.0f};
+    uint64_t    rep_fixed64[5]    = {0, 0, 0, 0, 2011};
+    int64_t     rep_sfixed64[5]   = {0, 0, 0, 0, -2012};
+    double      rep_double[5]     = {0, 0, 0, 0, 2013.0f};
+    char*       rep_string[5]     = {"", "", "", "", "2014"};
+    pb_bytes_ptr_t rep_bytes[5]   = {{0,0}, {0,0}, {0,0}, {0,0}, {4, (uint8_t*)"2015"}};
+    static int32_t rep_sub2zero   = 0;
+    static int32_t rep_substuff2  = 2016;
+    static uint32_t rep_substuff3 = 2016;
+    SubMessage  rep_submsg[5]     = {{"", &rep_sub2zero},
+                                     {"", &rep_sub2zero},
+                                     {"", &rep_sub2zero},
+                                     {"", &rep_sub2zero},
+                                     {"2016", &rep_substuff2, &rep_substuff3}};
+    MyEnum      rep_enum[5]       = {0, 0, 0, 0, MyEnum_Truth};
+    EmptyMessage rep_emptymsg[5]  = {{0}, {0}, {0}, {0}, {0}};
+
+    /* Values for optional fields */
+    int32_t     opt_int32         = 3041;
+    int64_t     opt_int64         = 3042;
+    uint32_t    opt_uint32        = 3043;
+    uint64_t    opt_uint64        = 3044;
+    int32_t     opt_sint32        = 3045;
+    int64_t     opt_sint64        = 3046;
+    bool        opt_bool          = true;
+    uint32_t    opt_fixed32       = 3048;
+    int32_t     opt_sfixed32      = 3049;
+    float       opt_float         = 3050.0f;
+    uint64_t    opt_fixed64       = 3051;
+    int64_t     opt_sfixed64      = 3052;
+    double      opt_double        = 3053.0;
+    char*       opt_string        = "3054";
+    pb_bytes_ptr_t opt_bytes      = {4, (uint8_t*)"3055"};
+    static int32_t opt_substuff   = 3056;
+    SubMessage  opt_submsg        = {"3056", &opt_substuff};
+    MyEnum      opt_enum          = MyEnum_Truth;
+    EmptyMessage opt_emptymsg     = {0};
+
+    /* Initialize the message struct with pointers to the fields. */
+    AllTypes alltypes = {0};
+
+    alltypes.req_int32         = &req_int32;
+    alltypes.req_int64         = &req_int64;
+    alltypes.req_uint32        = &req_uint32;
+    alltypes.req_uint64        = &req_uint64;
+    alltypes.req_sint32        = &req_sint32;
+    alltypes.req_sint64        = &req_sint64;
+    alltypes.req_bool          = &req_bool;
+    alltypes.req_fixed32       = &req_fixed32;
+    alltypes.req_sfixed32      = &req_sfixed32;
+    alltypes.req_float         = &req_float;
+    alltypes.req_fixed64       = &req_fixed64;
+    alltypes.req_sfixed64      = &req_sfixed64;
+    alltypes.req_double        = &req_double;
+    alltypes.req_string        = req_string;
+    alltypes.req_bytes         = &req_bytes;
+    alltypes.req_submsg        = &req_submsg;
+    alltypes.req_enum          = &req_enum;
+    alltypes.req_emptymsg      = &req_emptymsg;
+    
+    alltypes.rep_int32_count    = 5; alltypes.rep_int32     = rep_int32;
+    alltypes.rep_int64_count    = 5; alltypes.rep_int64     = rep_int64;
+    alltypes.rep_uint32_count   = 5; alltypes.rep_uint32    = rep_uint32;
+    alltypes.rep_uint64_count   = 5; alltypes.rep_uint64    = rep_uint64;
+    alltypes.rep_sint32_count   = 5; alltypes.rep_sint32    = rep_sint32;
+    alltypes.rep_sint64_count   = 5; alltypes.rep_sint64    = rep_sint64;
+    alltypes.rep_bool_count     = 5; alltypes.rep_bool      = rep_bool;
+    alltypes.rep_fixed32_count  = 5; alltypes.rep_fixed32   = rep_fixed32;
+    alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32  = rep_sfixed32;
+    alltypes.rep_float_count    = 5; alltypes.rep_float     = rep_float;
+    alltypes.rep_fixed64_count  = 5; alltypes.rep_fixed64   = rep_fixed64;
+    alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64  = rep_sfixed64;
+    alltypes.rep_double_count   = 5; alltypes.rep_double    = rep_double;
+    alltypes.rep_string_count   = 5; alltypes.rep_string    = rep_string;
+    alltypes.rep_bytes_count    = 5; alltypes.rep_bytes     = rep_bytes;
+    alltypes.rep_submsg_count   = 5; alltypes.rep_submsg    = rep_submsg;
+    alltypes.rep_enum_count     = 5; alltypes.rep_enum      = rep_enum;
+    alltypes.rep_emptymsg_count = 5; alltypes.rep_emptymsg  = rep_emptymsg;
+    
+    if (mode != 0)
+    {
+        /* Fill in values for optional fields */
+        alltypes.opt_int32         = &opt_int32;
+        alltypes.opt_int64         = &opt_int64;
+        alltypes.opt_uint32        = &opt_uint32;
+        alltypes.opt_uint64        = &opt_uint64;
+        alltypes.opt_sint32        = &opt_sint32;
+        alltypes.opt_sint64        = &opt_sint64;
+        alltypes.opt_bool          = &opt_bool;
+        alltypes.opt_fixed32       = &opt_fixed32;
+        alltypes.opt_sfixed32      = &opt_sfixed32;
+        alltypes.opt_float         = &opt_float;
+        alltypes.opt_fixed64       = &opt_fixed64;
+        alltypes.opt_sfixed64      = &opt_sfixed64;
+        alltypes.opt_double        = &opt_double;
+        alltypes.opt_string        = opt_string;
+        alltypes.opt_bytes         = &opt_bytes;
+        alltypes.opt_submsg        = &opt_submsg;
+        alltypes.opt_enum          = &opt_enum;
+        alltypes.opt_emptymsg      = &opt_emptymsg;
+    }
+    
+    alltypes.end = &end;
+    
+    {
+        uint8_t buffer[4096];
+        pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        
+        /* Now encode it and check if we succeeded. */
+        if (pb_encode(&stream, AllTypes_fields, &alltypes))
+        {
+            SET_BINARY_MODE(stdout);
+            fwrite(buffer, 1, stream.bytes_written, stdout);
+            return 0; /* Success */
+        }
+        else
+        {
+            fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
+            return 1; /* Failure */
+        }
+    }
+}

tests/backwards_compatibility/SConscript

@@ -3,8 +3,8 @@
 
 Import("env")
 
-enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "#common/pb_encode.o"])
+enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_encode.o"])
-dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "#common/pb_decode.o"])
+dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_decode.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_legacy.output"])

tests/basic_buffer/SConscript

@@ -2,11 +2,11 @@
 
 Import("env")
 
-enc = env.Program(["encode_buffer.c", "#common/person.pb.c", "#common/pb_encode.o"])
+enc = env.Program(["encode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"])
-dec = env.Program(["decode_buffer.c", "#common/person.pb.c", "#common/pb_decode.o"])
+dec = env.Program(["decode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_buffer.output"])
-env.Decode(["encode_buffer.output", "#common/person.proto"], MESSAGE = "Person")
+env.Decode(["encode_buffer.output", "$COMMON/person.proto"], MESSAGE = "Person")
 env.Compare(["decode_buffer.output", "encode_buffer.decoded"])
 

tests/basic_buffer/decode_buffer.c

@@ -60,7 +60,7 @@ bool print_person(pb_istream_t *stream)
 
 int main()
 {
-    uint8_t buffer[512];
+    uint8_t buffer[Person_size];
     pb_istream_t stream;
     size_t count;
     

tests/basic_buffer/encode_buffer.c

@@ -10,7 +10,7 @@
 
 int main()
 {
-    uint8_t buffer[512];
+    uint8_t buffer[Person_size];
     pb_ostream_t stream;
     
     /* Initialize the structure with constants */

tests/basic_stream/SConscript

@@ -2,11 +2,11 @@
 
 Import("env")
 
-enc = env.Program(["encode_stream.c", "#common/person.pb.c", "#common/pb_encode.o"])
+enc = env.Program(["encode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"])
-dec = env.Program(["decode_stream.c", "#common/person.pb.c", "#common/pb_decode.o"])
+dec = env.Program(["decode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_stream.output"])
-env.Decode(["encode_stream.output", "#common/person.proto"], MESSAGE = "Person")
+env.Decode(["encode_stream.output", "$COMMON/person.proto"], MESSAGE = "Person")
 env.Compare(["decode_stream.output", "encode_stream.decoded"])
 

tests/buffer_only/SConscript

@@ -6,10 +6,10 @@ Import("env")
 c = Copy("$TARGET", "$SOURCE")
 env.Command("pb_encode.c", "#../pb_encode.c", c)
 env.Command("pb_decode.c", "#../pb_decode.c", c)
-env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c)
+env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c)
-env.Command("alltypes.pb.c", "#alltypes/alltypes.pb.c", c)
+env.Command("alltypes.pb.c", "$BUILD/alltypes/alltypes.pb.c", c)
-env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
+env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c)
-env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c)
 
 # Define the compilation options
 opts = env.Clone()

tests/callbacks/SConscript

@@ -3,8 +3,8 @@
 Import("env")
 
 env.NanopbProto("callbacks")
-enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "#common/pb_encode.o"])
+enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_encode.o"])
-dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "#common/pb_decode.o"])
+dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_decode.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_callbacks.output"])

tests/common/unittestproto.proto

@@ -12,6 +12,10 @@ message StringMessage {
     required string data = 1 [(nanopb).max_size = 10];
 }
 
+message BytesMessage {
+    required bytes data = 1 [(nanopb).max_size = 16];
+}
+
 message CallbackArray {
     // We cheat a bit and use this message for testing other types, too.
     // Nanopb does not care about the actual defined data type for callback

tests/cxx_main_program/SConscript

@@ -7,10 +7,10 @@ Import("env")
 c = Copy("$TARGET", "$SOURCE")
 env.Command("pb_encode.cxx", "#../pb_encode.c", c)
 env.Command("pb_decode.cxx", "#../pb_decode.c", c)
-env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c)
+env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c)
-env.Command("alltypes.pb.cxx", "#alltypes/alltypes.pb.c", c)
+env.Command("alltypes.pb.cxx", "$BUILD/alltypes/alltypes.pb.c", c)
-env.Command("encode_alltypes.cxx", "#alltypes/encode_alltypes.c", c)
+env.Command("encode_alltypes.cxx", "$BUILD/alltypes/encode_alltypes.c", c)
-env.Command("decode_alltypes.cxx", "#alltypes/decode_alltypes.c", c)
+env.Command("decode_alltypes.cxx", "$BUILD/alltypes/decode_alltypes.c", c)
 
 # Now build and run the test normally.
 enc = env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx"])

tests/decode_unittests/SConscript

@@ -1,4 +1,4 @@
 Import('env')
-p = env.Program(["decode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_decode.o"])
+p = env.Program(["decode_unittests.c", "$COMMON/unittestproto.pb.c"])
 env.RunTest(p)
 

tests/decode_unittests/decode_unittests.c

@@ -1,8 +1,8 @@
-#define NANOPB_INTERNALS
+/* This includes the whole .c file to get access to static functions. */
+#include "pb_decode.c"
 
 #include <stdio.h>
 #include <string.h>
-#include "pb_decode.h"
 #include "unittests.h"
 #include "unittestproto.pb.h"
 

tests/encode_unittests/SConscript

@@ -1,5 +1,5 @@
 # Build and run the stand-alone unit tests for the nanopb encoder part.
 
 Import('env')
-p = env.Program(["encode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_encode.o"])
+p = env.Program(["encode_unittests.c", "$COMMON/unittestproto.pb.c"])
 env.RunTest(p)

tests/encode_unittests/encode_unittests.c

@@ -1,8 +1,8 @@
-#define NANOPB_INTERNALS
+/* This includes the whole .c file to get access to static functions. */
+#include "pb_encode.c"
 
 #include <stdio.h>
 #include <string.h>
-#include "pb_encode.h"
 #include "unittests.h"
 #include "unittestproto.pb.h"
 
@@ -172,9 +172,9 @@ int main()
         struct { size_t size; uint8_t bytes[5]; } value = {5, {'x', 'y', 'z', 'z', 'y'}};
     
         COMMENT("Test pb_enc_bytes")
-        TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x05xyzzy"))
+        TEST(WRITES(pb_enc_bytes(&s, &BytesMessage_fields[0], &value), "\x05xyzzy"))
         value.size = 0;
-        TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x00"))
+        TEST(WRITES(pb_enc_bytes(&s, &BytesMessage_fields[0], &value), "\x00"))
     }
     
     {
@@ -223,6 +223,20 @@ int main()
         TEST(!pb_encode(&s, FloatArray_fields, &msg))
     }
     
+    {
+        uint8_t buffer[50];
+        pb_ostream_t s;
+        FloatArray msg = {1, {99.0f}};
+        
+        COMMENT("Test array size limit in pb_encode")
+        
+        s = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        TEST((msg.data_count = 10) && pb_encode(&s, FloatArray_fields, &msg))
+        
+        s = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        TEST((msg.data_count = 11) && !pb_encode(&s, FloatArray_fields, &msg))
+    }
+    
     {
         uint8_t buffer[10];
         pb_ostream_t s;
@@ -244,6 +258,20 @@ int main()
                     "\x0A\x07\x0A\x05\x01\x02\x03\x04\x05"))
     }
     
+    {
+        uint8_t buffer[32];
+        pb_ostream_t s;
+        BytesMessage msg = {{3, "xyz"}};
+        
+        COMMENT("Test pb_encode with bytes message.")
+        TEST(WRITES(pb_encode(&s, BytesMessage_fields, &msg),
+                    "\x0A\x03xyz"))
+        
+        msg.data.size = 17; /* More than maximum */
+        TEST(!pb_encode(&s, BytesMessage_fields, &msg))
+    }
+        
+    
     {
         uint8_t buffer[20];
         pb_ostream_t s;
@@ -280,6 +308,19 @@ int main()
         TEST(!pb_encode(&s, CallbackContainerContainer_fields, &msg2))
     }
     
+    {
+        uint8_t buffer[StringMessage_size];
+        pb_ostream_t s;
+        StringMessage msg = {"0123456789"};
+        
+        s = pb_ostream_from_buffer(buffer, sizeof(buffer));
+        
+        COMMENT("Test that StringMessage_size is correct")
+
+        TEST(pb_encode(&s, StringMessage_fields, &msg));
+        TEST(s.bytes_written == StringMessage_size);
+    }
+    
     if (status != 0)
         fprintf(stdout, "\n\nSome tests FAILED!\n");
     

tests/extensions/SConscript

@@ -4,12 +4,12 @@ Import("env")
 
 # We use the files from the alltypes test case
 incpath = env.Clone()
-incpath.Append(PROTOCPATH = '#alltypes')
+incpath.Append(PROTOCPATH = '$BUILD/alltypes')
-incpath.Append(CPPPATH = '#alltypes')
+incpath.Append(CPPPATH = '$BUILD/alltypes')
 
-incpath.NanopbProto("extensions")
+incpath.NanopbProto(["extensions", "extensions.options"])
-enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb$OBJSUFFIX", "#common/pb_encode.o"])
+enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_encode.o"])
-dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb$OBJSUFFIX", "#common/pb_decode.o"])
+dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_decode.o"])
 
 env.RunTest(enc)
 env.RunTest([dec, "encode_extensions.output"])

tests/extensions/encode_extensions.c

@@ -46,5 +46,9 @@ int main(int argc, char **argv)
         fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
         return 1; /* Failure */
     }
+    
+    /* Check that the field tags are properly generated */
+    (void)AllTypes_extensionfield1_tag;
+    (void)ExtensionMessage_AllTypes_extensionfield2_tag;
 }
 

tests/extra_fields/SConscript

@@ -2,13 +2,13 @@
 
 Import("env")
 
-dec = env.GetBuildPath('#basic_buffer/${PROGPREFIX}decode_buffer${PROGSUFFIX}')
+dec = env.GetBuildPath('$BUILD/basic_buffer/${PROGPREFIX}decode_buffer${PROGSUFFIX}')
 env.RunTest('person_with_extra_field.output', [dec, "person_with_extra_field.pb"])
 env.Compare(["person_with_extra_field.output", "person_with_extra_field.expected"])
 
-dec = env.GetBuildPath('#basic_stream/${PROGPREFIX}decode_stream${PROGSUFFIX}')
+dec = env.GetBuildPath('$BUILD/basic_stream/${PROGPREFIX}decode_stream${PROGSUFFIX}')
 env.RunTest('person_with_extra_field_stream.output', [dec, "person_with_extra_field.pb"])
 env.Compare(["person_with_extra_field_stream.output", "person_with_extra_field.expected"])
 
-dec2 = env.GetBuildPath('#alltypes/${PROGPREFIX}decode_alltypes${PROGSUFFIX}')
+dec2 = env.GetBuildPath('$BUILD/alltypes/${PROGPREFIX}decode_alltypes${PROGSUFFIX}')
 env.RunTest('alltypes_with_extra_fields.output', [dec2, 'alltypes_with_extra_fields.pb'])

tests/field_size_16/SConscript

@@ -7,10 +7,10 @@ Import("env")
 c = Copy("$TARGET", "$SOURCE")
 env.Command("pb_encode.c", "#../pb_encode.c", c)
 env.Command("pb_decode.c", "#../pb_decode.c", c)
-env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
+env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c)
-env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c)
 
-env.NanopbProto("alltypes")
+env.NanopbProto(["alltypes", "alltypes.options"])
 
 # Define the compilation options
 opts = env.Clone()

tests/field_size_32/SConscript

@@ -7,10 +7,10 @@ Import("env")
 c = Copy("$TARGET", "$SOURCE")
 env.Command("pb_encode.c", "#../pb_encode.c", c)
 env.Command("pb_decode.c", "#../pb_decode.c", c)
-env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
+env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c)
-env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c)
 
-env.NanopbProto("alltypes")
+env.NanopbProto(["alltypes", "alltypes.options"])
 
 # Define the compilation options
 opts = env.Clone()

tests/message_sizes/SConscript

@@ -0,0 +1,11 @@
+# Test the generation of message size #defines
+
+Import('env')
+
+incpath = env.Clone()
+incpath.Append(PROTOCPATH = '#message_sizes')
+
+incpath.NanopbProto("messages1")
+incpath.NanopbProto("messages2")
+
+incpath.Program(['dummy.c', 'messages1.pb.c', 'messages2.pb.c'])

tests/message_sizes/dummy.c

@@ -0,0 +1,9 @@
+/* Just test that the file can be compiled successfully. */
+
+#include "messages2.pb.h"
+
+int main()
+{
+    return xmit_size;
+}
+

tests/message_sizes/messages1.proto

@@ -0,0 +1,27 @@
+enum MessageStatus {
+    FAIL = 0;
+    OK = 1;
+};
+
+message MessageInfo {
+   required fixed32 msg_id = 1;
+   optional fixed32 interface_id = 2;
+}
+
+message MessageResponseInfo {
+   required fixed64 interface_id = 1;
+   required fixed32 seq = 2;
+   required fixed32 msg_id = 3;
+}
+
+message MessageHeader {
+   required MessageInfo info = 1;
+   optional MessageResponseInfo response_info = 2;
+   optional MessageResponse response = 3;
+}
+
+message MessageResponse {
+   required MessageStatus status = 1;
+   required fixed32 seq = 2;
+}
+

tests/message_sizes/messages2.proto

@@ -0,0 +1,8 @@
+import 'nanopb.proto';
+import 'messages1.proto';
+
+message xmit {
+   required MessageHeader header = 1;
+   required bytes data = 2 [(nanopb).max_size = 128];
+}
+

tests/missing_fields/SConscript

@@ -3,6 +3,6 @@
 Import("env")
 
 env.NanopbProto("missing_fields")
-test = env.Program(["missing_fields.c", "missing_fields.pb.c", "#common/pb_encode.o", "#common/pb_decode.o"])
+test = env.Program(["missing_fields.c", "missing_fields.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_decode.o"])
 env.RunTest(test)
 

tests/no_errmsg/SConscript

@@ -6,10 +6,10 @@ Import("env")
 c = Copy("$TARGET", "$SOURCE")
 env.Command("pb_encode.c", "#../pb_encode.c", c)
 env.Command("pb_decode.c", "#../pb_decode.c", c)
-env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c)
+env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c)
-env.Command("alltypes.pb.c", "#alltypes/alltypes.pb.c", c)
+env.Command("alltypes.pb.c", "$BUILD/alltypes/alltypes.pb.c", c)
-env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c)
+env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c)
-env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c)
+env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c)
 
 # Define the compilation options
 opts = env.Clone()