Fixed pb_get_message_size

Code reduction optimization by sharing common field descriptor information for numeric field types
2012-02-27 14:46:38 -08:00 · 2012-02-24 13:46:36 -08:00
17 changed files with 883 additions and 279 deletions
--- a/9
+++ b/9
@@ -7,3 +7,12 @@ protobuf-compiler python-protobuf libprotobuf-dev

 To run the tests, run make under the tests folder.
 If it completes without error, everything is fine. 
+
+Code size optimization is currently only supported for 32-bit 
+architecture. If you want to run on 64-bit architecture,
+you must disable code size optimization by providing -n option to
+code generator nanopb_generator.py.  
+
+For testing purpose, you may need to pass -m32 compiler to gcc if 
+you're running on a 64-bit machine unless -n option is used for code 
+generation. 
--- a/example/Makefile
+++ b/example/Makefile
@@ -1,14 +1,24 @@
 CFLAGS=-ansi -Wall -Werror -I .. -g -O0
-DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h
+DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_field.c ../pb_encode.h ../pb.h ../pb_field.h
+
+CC_VER := $(shell gcc --version | grep gcc)
+ifneq "$(CC_VER)" ""
+CFLAGS += -m32
+endif
+ifndef PB_PATH
+PBPATHOPT=-I/usr/include -I/usr/local/include
+else
+PBPATHOPT=-I$(PB_PATH)
+endif

 all: server client

 clean:
-	rm -f server client fileproto.pb.c fileproto.pb.h
+	rm -f server client fileproto.pb.c fileproto.pb.h fileproto.pb

 %: %.c $(DEPS) fileproto.pb.h fileproto.pb.c
-	$(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c fileproto.pb.c common.c
+	$(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c ../pb_field.c fileproto.pb.c common.c

 fileproto.pb.c fileproto.pb.h: fileproto.proto ../generator/nanopb_generator.py
-	protoc -I. -I../generator -I/usr/include -ofileproto.pb $<
+	protoc -I. -I../generator $(PBPATHOPT) -ofileproto.pb $<
 	python ../generator/nanopb_generator.py fileproto.pb
--- a/example/common.h
+++ b/example/common.h
@@ -7,3 +7,4 @@ pb_ostream_t pb_ostream_from_socket(int fd);
 pb_istream_t pb_istream_from_socket(int fd);

 #endif
+
--- a/generator/Makefile
+++ b/generator/Makefile
@@ -1,2 +1,8 @@
+ifndef PB_PATH
+PBPATHOPT=-I/usr/include -I/usr/local/include
+else
+PBPATHOPT=-I$(PB_PATH)
+endif
+
 nanopb_pb2.py: nanopb.proto
-	protoc --python_out=. -I /usr/include -I . nanopb.proto
+	protoc --python_out=. $(PBPATHOPT) -I . nanopb.proto
--- a/generator/nanopb_generator.py
+++ b/generator/nanopb_generator.py
@@ -4,22 +4,22 @@ import google.protobuf.descriptor_pb2 as descriptor
 import nanopb_pb2
 import os.path

-# Values are tuple (c type, pb ltype)
+# Values are tuple (c type, pb ltype, memory aligned flag, common field info type)
 FieldD = descriptor.FieldDescriptorProto
 datatypes = {
-    FieldD.TYPE_BOOL: ('bool', 'PB_LTYPE_VARINT'),
-    FieldD.TYPE_DOUBLE: ('double', 'PB_LTYPE_FIXED64'),
-    FieldD.TYPE_FIXED32: ('uint32_t', 'PB_LTYPE_FIXED32'),
-    FieldD.TYPE_FIXED64: ('uint64_t', 'PB_LTYPE_FIXED64'),
-    FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED32'),
-    FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT'),
-    FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT'),
-    FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED32'),
-    FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED64'),
-    FieldD.TYPE_SINT32: ('int32_t', 'PB_LTYPE_SVARINT'),
-    FieldD.TYPE_SINT64: ('int64_t', 'PB_LTYPE_SVARINT'),
-    FieldD.TYPE_UINT32: ('uint32_t', 'PB_LTYPE_VARINT'),
-    FieldD.TYPE_UINT64: ('uint64_t', 'PB_LTYPE_VARINT')
+    FieldD.TYPE_BOOL: ('bool', 'PB_LTYPE_VARINT', False, 'BOOL'),
+    FieldD.TYPE_DOUBLE: ('double', 'PB_LTYPE_FIXED64', True, 'FIXED64'),
+    FieldD.TYPE_FIXED32: ('uint32_t', 'PB_LTYPE_FIXED32', True, 'FIXED32'),
+    FieldD.TYPE_FIXED64: ('uint64_t', 'PB_LTYPE_FIXED64', True, 'FIXED64'),
+    FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED32', True, 'FIXED32'),
+    FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT', True, 'INT32'),
+    FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT', True, 'INT64'),
+    FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED32', True, 'FIXED32'),
+    FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED64', True, 'FIXED64'),
+    FieldD.TYPE_SINT32: ('int32_t', 'PB_LTYPE_SVARINT', True, 'SINT32'),
+    FieldD.TYPE_SINT64: ('int64_t', 'PB_LTYPE_SVARINT', True, "SINT64"),
+    FieldD.TYPE_UINT32: ('uint32_t', 'PB_LTYPE_VARINT', True, 'INT32'),
+    FieldD.TYPE_UINT64: ('uint64_t', 'PB_LTYPE_VARINT', True, 'INT64')
 }

 class Names:
@@ -70,6 +70,9 @@ class Field:
        self.max_size = None
        self.max_count = None
        self.array_decl = ""
+        self.is_last_field = False
+        self.aligned = False
+        self.info_index = None
        
        # Parse nanopb-specific field options
        if desc.options.HasExtension(nanopb_pb2.nanopb):
@@ -104,10 +107,12 @@ class Field:
        # defining how to decode an individual value.
        # CTYPE is the name of the c type to use in the struct.
        if datatypes.has_key(desc.type):
-            self.ctype, self.ltype = datatypes[desc.type]
+            self.ctype, self.ltype, self.aligned, self.common_info_name = datatypes[desc.type]
        elif desc.type == FieldD.TYPE_ENUM:
            self.ltype = 'PB_LTYPE_VARINT'
            self.ctype = names_from_type_name(desc.type_name)
+            self.aligned = True
+            self.common_info_name = 'INT32'
            if self.default is not None:
                self.default = self.ctype + self.default
        elif desc.type == FieldD.TYPE_STRING:
@@ -192,10 +197,28 @@ class Field:
        else:
            return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default)

+    def pb_field_key_t(self):
+        '''Return the pb_field_t field key initializer to use in the constant array.
+        '''
+        result = '    {%d' % self.tag
+        if self.is_last_field:
+            result += ' | PB_LAST_FIELD'
+            
+        if self.info_index is not None:
+            result += ', %d}' % self.info_index
+        else:
+            if 'REQUIRED' in self.htype:
+                prefix = 'REQUIRED'
+            elif 'OPTIONAL' in self.htype:
+                prefix = 'OPTIONAL'
+            result += ', %s_%s_INFO}' % (prefix, self.common_info_name)
+        return result
+    
    def pb_field_t(self, prev_field_name):
-        '''Return the pb_field_t initializer to use in the constant array.
+        '''Return the pb_field_t field info initializer to use in the constant array.
        prev_field_name is the name of the previous field or None.
        '''
+        ''' result = '    {0, '''
        result = '    {%d, ' % self.tag        
        result += self.htype
        if self.ltype is not None:
@@ -232,12 +255,24 @@ class Field:
        
        return result
    
+    def should_generate_field_info(self, prev):
+        return  g_no_optimization or not ((prev is None or prev.aligned) and (self.ltype in ['PB_LTYPE_VARINT', 'PB_LTYPE_SVARINT', 'PB_LTYPE_FIXED32', 'PB_LTYPE_FIXED64']) and (self.htype in ['PB_HTYPE_REQUIRED', 'PB_HTYPE_OPTIONAL']))
+    
 class Message:
    def __init__(self, names, desc):
        self.name = names
        self.fields = [Field(self.name, f) for f in desc.field]
        self.ordered_fields = self.fields[:]
        self.ordered_fields.sort()
+        if self.ordered_fields:
+            self.ordered_fields[-1].is_last_field = True
+        self.num_info = 0
+        prev = None
+        for f in self.ordered_fields:
+            if f.should_generate_field_info(prev):
+                f.info_index = self.num_info
+                self.num_info += 1
+            prev = f
        
    def get_dependencies(self):
        '''Get list of type names that this structure refers to.'''
@@ -266,19 +301,48 @@ class Message:
        return result

    def fields_declaration(self):
-        result = 'extern const pb_field_t %s_fields[%d];' % (self.name, len(self.fields) + 1)
+        result = 'extern const pb_field_info_t %s_fields[1];' % self.name
+         
        return result

    def fields_definition(self):
-        result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, len(self.fields) + 1)
+        num_fields = len(self.fields)
        
+        result = 'static const pb_field_key_t %s_field_keys[%d] = {\n' % (self.name, num_fields)
+
+        for field in self.ordered_fields:
+            result += field.pb_field_key_t()
+            if not field.is_last_field:
+                result += ',\n'
+            prev = field                
+        result += '\n};\n\n'
+    
+        if self.num_info > 0:
+            result += 'static const pb_field_t %s_field_info[%d] = {\n' % (self.name, self.num_info)    
            prev = None
            for field in self.ordered_fields:
+                if field.info_index is not None:
                    result += field.pb_field_t(prev)
+                    if field.info_index < self.num_info - 1:
                        result += ',\n\n'
                prev = field.name
+            result += '\n};\n\n'
+        
+        decl = 'const pb_field_info_t %s_fields[1] = { {' % self.name
+        
+        if num_fields == 0:
+            result = decl
+            result += '\n    0,'
+            result += '\n    0' 
+        else:
+            result += decl
+            result += '\n    &%s_field_keys[0],' % self.name
+            if self.num_info > 0:
+                result += '\n    &%s_field_info[0]' % self.name
+            else:
+                result += '\n    0' 
+        result += '\n}};\n' 
        
-        result += '    PB_LAST_FIELD\n};'
        return result

 def iterate_messages(desc, names = Names()):
@@ -401,19 +465,25 @@ def generate_source(headername, enums, messages):
 if __name__ == '__main__':
    import sys
    import os.path
+    from optparse import OptionParser
+    usage = "usage: %prog [options] file.pb\n" + \
+            "where file.pb has been compiled from .proto by:\n" + \
+            "    protoc -ofile.pb file.proto\n" + \
+            "Output fill be written to file.pb.h and file.pb.c"

-    if len(sys.argv) != 2:
-        print "Usage: " + sys.argv[0] + " file.pb"
-        print "where file.pb has been compiled from .proto by:"
-        print "protoc -ofile.pb file.proto"
-        print "Output fill be written to file.pb.h and file.pb.c"
+    parser = OptionParser(usage=usage)
+    parser.add_option('-n', '--no-opt', action='store_true', dest='no_optimization', default=False, help='disable code reduction optimization as 64-bit architecture is not supported [default=optimization on]')
+    (options, args) = parser.parse_args()
+    
+    if len(args) != 1:
+        parser.print_help()
        sys.exit(1)
-    
-    data = open(sys.argv[1], 'rb').read()
+    g_no_optimization = options.no_optimization
+    data = open(args[0], 'rb').read()
    fdesc = descriptor.FileDescriptorSet.FromString(data)
    enums, messages = parse_file(fdesc.file[0])
    
-    noext = os.path.splitext(sys.argv[1])[0]
+    noext = os.path.splitext(args[0])[0]
    headername = noext + '.pb.h'
    sourcename = noext + '.pb.c'
    headerbasename = os.path.basename(headername)
--- a/pb.h
+++ b/pb.h
@@ -5,112 +5,7 @@
 * Most of these are quite low-level stuff. For the high-level interface,
 * see pb_encode.h or pb_decode.h
 */
-
-#include <stdint.h>
-#include <stddef.h>
-#include <stdbool.h>
-
-#ifdef __GNUC__
-/* This just reduces memory requirements, but is not required. */
-#define pb_packed __attribute__((packed))
-#else
-#define pb_packed
-#endif
-
-/* List of possible field types. These are used in the autogenerated code.
- * Least-significant 4 bits tell the scalar type
- * Most-significant 4 bits specify repeated/required/packed etc.
- * 
- * INT32 and UINT32 are treated the same, as are (U)INT64 and (S)FIXED*
- * These types are simply casted to correct field type when they are
- * assigned to the memory pointer.
- * SINT* is different, though, because it is zig-zag coded.
- */
-
-typedef enum {
-    /************************
-     * Field contents types *
-     ************************/
-    
-    /* Numeric types */
-    PB_LTYPE_VARINT = 0x00, /* int32, uint32, int64, uint64, bool, enum */
-    PB_LTYPE_SVARINT = 0x01, /* sint32, sint64 */
-    PB_LTYPE_FIXED32 = 0x02, /* fixed32, sfixed32, float */
-    PB_LTYPE_FIXED64 = 0x03, /* fixed64, sfixed64, double */
-    
-    /* Marker for last packable field type. */
-    PB_LTYPE_LAST_PACKABLE = 0x03,
-    
-    /* Byte array with pre-allocated buffer.
-     * data_size is the length of the allocated PB_BYTES_ARRAY structure. */
-    PB_LTYPE_BYTES = 0x04,
-    
-    /* String with pre-allocated buffer.
-     * data_size is the maximum length. */
-    PB_LTYPE_STRING = 0x05,
-    
-    /* Submessage
-     * submsg_fields is pointer to field descriptions */
-    PB_LTYPE_SUBMESSAGE = 0x06,
-    
-    /* Number of declared LTYPES */
-    PB_LTYPES_COUNT = 7,
-    
-    /******************
-     * Modifier flags *
-     ******************/
-    
-    /* Just the basic, write data at data_offset */
-    PB_HTYPE_REQUIRED = 0x00,
-    
-    /* Write true at size_offset */
-    PB_HTYPE_OPTIONAL = 0x10,
-    
-    /* Read to pre-allocated array
-     * Maximum number of entries is array_size,
-     * actual number is stored at size_offset */
-    PB_HTYPE_ARRAY = 0x20,
-    
-    /* Works for all required/optional/repeated fields.
-     * data_offset points to pb_callback_t structure.
-     * LTYPE should be 0 (it is ignored, but sometimes
-     * used to speculatively index an array). */
-    PB_HTYPE_CALLBACK = 0x30
-} pb_packed pb_type_t;
-
-#define PB_HTYPE(x) ((x) & 0xF0)
-#define PB_LTYPE(x) ((x) & 0x0F)
-
-/* This structure is used in auto-generated constants
- * to specify struct fields.
- * You can change field sizes here if you need structures
- * larger than 256 bytes or field tags larger than 256.
- * The compiler should complain if your .proto has such
- * structures ("initializer too large for type").
- */
-typedef struct _pb_field_t pb_field_t;
-struct _pb_field_t {
-    uint8_t tag;
-    pb_type_t type;
-    uint8_t data_offset; /* Offset of field data, relative to previous field. */
-    int8_t size_offset; /* Offset of array size or has-boolean, relative to data */
-    uint8_t data_size; /* Data size in bytes for a single item */
-    uint8_t array_size; /* Maximum number of entries in array */
-    
-    /* Field definitions for submessage
-     * OR default value for all other non-array, non-callback types
-     * If null, then field will zeroed. */
-    const void *ptr;
-} pb_packed;
-
-/* This structure is used for 'bytes' arrays.
- * It has the number of bytes in the beginning, and after that an array.
- * Note that actual structs used will have a different length of bytes array.
- */
-typedef struct {
-    size_t size;
-    uint8_t bytes[1];
-} pb_bytes_array_t;
+#include <pb_field.h>

 /* This structure is used for giving the callback function.
 * It is stored in the message structure and filled in by the method that
@@ -151,12 +46,4 @@ typedef enum {
    PB_WT_32BIT  = 5
 } pb_wire_type_t;

-/* These macros are used to declare pb_field_t's in the constant array. */
-#define pb_membersize(st, m) (sizeof ((st*)0)->m)
-#define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0]))
-#define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2))
-#define pb_delta_end(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2))
-#define PB_LAST_FIELD {0,0,0,0}
-
-
 #endif
--- a/pb_decode.c
+++ b/pb_decode.c
@@ -183,82 +183,26 @@ static bool checkreturn make_string_substream(pb_istream_t *stream, pb_istream_t
    return true;
 }

-/* Iterator for pb_field_t list */
-typedef struct {
-    const pb_field_t *start;
-    const pb_field_t *current;
-    int field_index;
-    void *dest_struct;
-    void *pData;
-    void *pSize;
-} pb_field_iterator_t;
-
-static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct)
-{
-    iter->start = iter->current = fields;
-    iter->field_index = 0;
-    iter->pData = (char*)dest_struct + iter->current->data_offset;
-    iter->pSize = (char*)iter->pData + iter->current->size_offset;
-    iter->dest_struct = dest_struct;
-}
-
-static bool pb_field_next(pb_field_iterator_t *iter)
-{
-    bool notwrapped = true;
-    size_t prev_size = iter->current->data_size;
-    
-    if (PB_HTYPE(iter->current->type) == PB_HTYPE_ARRAY)
-        prev_size *= iter->current->array_size;
-    
-    iter->current++;
-    iter->field_index++;
-    if (iter->current->tag == 0)
-    {
-        iter->current = iter->start;
-        iter->field_index = 0;
-        iter->pData = iter->dest_struct;
-        prev_size = 0;
-        notwrapped = false;
-    }
-    
-    iter->pData = (char*)iter->pData + prev_size + iter->current->data_offset;
-    iter->pSize = (char*)iter->pData + iter->current->size_offset;
-    return notwrapped;
-}
-
-static bool checkreturn pb_field_find(pb_field_iterator_t *iter, int tag)
-{
-    int start = iter->field_index;
-    
-    do {
-        if (iter->current->tag == tag)
-            return true;
-        pb_field_next(iter);
-    } while (iter->field_index != start);
-    
-    return false;
-}
-
 /*************************
 * Decode a single field *
 *************************/

 static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter)
 {
-    pb_decoder_t func = PB_DECODERS[PB_LTYPE(iter->current->type)];
+    pb_decoder_t func = PB_DECODERS[PB_LTYPE(iter->current.type)];
    
-    switch (PB_HTYPE(iter->current->type))
+    switch (PB_HTYPE(iter->current.type))
    {
        case PB_HTYPE_REQUIRED:
-            return func(stream, iter->current, iter->pData);
+            return func(stream, &iter->current, iter->pData);
            
        case PB_HTYPE_OPTIONAL:
            *(bool*)iter->pSize = true;
-            return func(stream, iter->current, iter->pData);
+            return func(stream, &iter->current, iter->pData);
    
        case PB_HTYPE_ARRAY:
            if (wire_type == PB_WT_STRING
-                && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE)
+                && PB_LTYPE(iter->current.type) <= PB_LTYPE_LAST_PACKABLE)
            {
                /* Packed array */
                size_t *size = (size_t*)iter->pSize;
@@ -266,10 +210,10 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
                if (!make_string_substream(stream, &substream))
                    return false;
                
-                while (substream.bytes_left && *size < iter->current->array_size)
+                while (substream.bytes_left && *size < iter->current.array_size)
                {
-                    void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size);
-                    if (!func(&substream, iter->current, pItem))
+                    void *pItem = (uint8_t*)iter->pData + iter->current.data_size * (*size);
+                    if (!func(&substream, &iter->current, pItem))
                        return false;
                    (*size)++;
                }
@@ -279,12 +223,12 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
            {
                /* Repeated field */
                size_t *size = (size_t*)iter->pSize;
-                void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size);
-                if (*size >= iter->current->array_size)
+                void *pItem = (uint8_t*)iter->pData + iter->current.data_size * (*size);
+                if (*size >= iter->current.array_size)
                    return false;
                
                (*size)++;
-                return func(stream, iter->current, pItem);
+                return func(stream, &iter->current, pItem);
            }
        
        case PB_HTYPE_CALLBACK:
@@ -303,7 +247,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
                
                while (substream.bytes_left)
                {
-                    if (!pCallback->funcs.decode(&substream, iter->current, pCallback->arg))
+                    if (!pCallback->funcs.decode(&substream, &iter->current, pCallback->arg))
                        return false;
                }
                
@@ -324,7 +268,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
                    return false;
                substream = pb_istream_from_buffer(buffer, size);
                
-                return pCallback->funcs.decode(&substream, iter->current, pCallback->arg);
+                return pCallback->funcs.decode(&substream, &iter->current, pCallback->arg);
            }
        }
        
@@ -334,44 +278,44 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t
 }

 /* Initialize message fields to default values, recursively */
-static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct)
+static void pb_message_set_to_defaults(const pb_field_info_t *fields, void *dest_struct)
 {
    pb_field_iterator_t iter;
+
+    if (fields->field_keys == NULL) return;
+
    pb_field_init(&iter, fields, dest_struct);
    
    /* Initialize size/has fields and apply default values */
    do
    {
-        if (iter.current->tag == 0)
-            continue;
-        
        /* Initialize the size field for optional/repeated fields to 0. */
-        if (PB_HTYPE(iter.current->type) == PB_HTYPE_OPTIONAL)
+        if (PB_HTYPE(iter.current.type) == PB_HTYPE_OPTIONAL)
        {
            *(bool*)iter.pSize = false;
        }
-        else if (PB_HTYPE(iter.current->type) == PB_HTYPE_ARRAY)
+        else if (PB_HTYPE(iter.current.type) == PB_HTYPE_ARRAY)
        {
            *(size_t*)iter.pSize = 0;
            continue; /* Array is empty, no need to initialize contents */
        }
        
        /* Initialize field contents to default value */
-        if (PB_HTYPE(iter.current->type) == PB_HTYPE_CALLBACK)
+        if (PB_HTYPE(iter.current.type) == PB_HTYPE_CALLBACK)
        {
            continue; /* Don't overwrite callback */
        }
-        else if (PB_LTYPE(iter.current->type) == PB_LTYPE_SUBMESSAGE)
+        else if (PB_LTYPE(iter.current.type) == PB_LTYPE_SUBMESSAGE)
        {
-            pb_message_set_to_defaults(iter.current->ptr, iter.pData);
+            pb_message_set_to_defaults(iter.current.ptr, iter.pData);
        }
-        else if (iter.current->ptr != NULL)
+        else if (iter.current.ptr != NULL)
        {
-            memcpy(iter.pData, iter.current->ptr, iter.current->data_size);
+            memcpy(iter.pData, iter.current.ptr, iter.current.data_size);
        }
        else
        {
-            memset(iter.pData, 0, iter.current->data_size);
+            memset(iter.pData, 0, iter.current.data_size);
        }
    } while (pb_field_next(&iter));
 }
@@ -380,10 +324,11 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str
 * Decode all fields *
 *********************/

-bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct)
+bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_info_t *fields, void *dest_struct)
 {
    uint32_t fields_seen = 0; /* Used to check for required fields */
    pb_field_iterator_t iter;
+    const pb_field_key_t *field_keys = fields->field_keys;
    int i;
    
    pb_message_set_to_defaults(fields, dest_struct);
@@ -424,13 +369,14 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void
    }
    
    /* Check that all required fields (mod 31) were present. */
-    for (i = 0; fields[i].tag != 0; i++)
+    for (i = 0; field_keys != NULL; i++)
    {
-        if (PB_HTYPE(fields[i].type) == PB_HTYPE_REQUIRED &&
+        if (PB_HTYPE(PB_FIELD_INFO(&field_keys[i], fields->field_info)->type) == PB_HTYPE_REQUIRED &&
            !(fields_seen & (1 << (i & 31))))
        {
            return false;
        }
+        if (PB_IS_LAST(&field_keys[i])) break;
    }
    
    return true;
@@ -523,8 +469,7 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo
    if (!pb_decode_varint32(stream, &size))
        return false;
    
-    /* Check length, noting the null terminator */
-    if (size > field->data_size - 1)
+    if ((int) size > field->data_size - 1)
        return false;
    
    status = pb_read(stream, (uint8_t*)dest, size);
@@ -543,7 +488,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field
    if (field->ptr == NULL)
        return false;
    
-    status = pb_decode(&substream, (pb_field_t*)field->ptr, dest);
+    status = pb_decode(&substream, (pb_field_info_t*)field->ptr, dest);
    stream->state = substream.state;
    return status;
 }
--- a/pb_decode.h
+++ b/pb_decode.h
@@ -42,7 +42,7 @@ bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count);
 * Returns true on success, false on any failure.
 * The actual struct pointed to by dest must match the description in fields.
 */
-bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct);
+bool pb_decode(pb_istream_t *stream, const pb_field_info_t *fields, void *dest_struct);

 /* --- Helper functions ---
 * You may want to use these from your caller or callbacks.
@@ -68,4 +68,6 @@ bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest);
 bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest);
 bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest);

+#define PB_ISTREAM_INIT {0,0,0}
+
 #endif
--- a/pb_encode.c
+++ b/pb_encode.c
@@ -75,7 +75,7 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count
 static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field,
                         const void *pData, size_t count, pb_encoder_t func)
 {
-    int i;
+    size_t i;
    const void *p;
    size_t size;
    
@@ -98,7 +98,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
        }
        else
        {
-            pb_ostream_t sizestream = {0};
+            pb_ostream_t sizestream = PB_OSTREAM_INIT;
            p = pData;
            for (i = 0; i < count; i++)
            {
@@ -140,15 +140,19 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
    return true;
 }

-bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct)
+bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_info_t *fields, const void *src_struct)
 {
-    const pb_field_t *field = fields;
+    const pb_field_t *field;
+    const pb_field_key_t *field_key = fields->field_keys;
    const void *pData = src_struct;
    const void *pSize;
    size_t prev_size = 0;
    
-    while (field->tag != 0)
+    while(field_key != NULL)
    {
+        pb_field_t f = *(PB_FIELD_INFO(field_key, fields->field_info));
+        f.tag = PB_TAG_VAL(field_key);
+        field = &f;
        pb_encoder_t func = PB_ENCODERS[PB_LTYPE(field->type)];
        pData = (const char*)pData + prev_size + field->data_offset;
        pSize = (const char*)pData + field->size_offset;
@@ -193,13 +197,20 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons
                break;
            }
        }
-    
-        field++;
+        if (PB_IS_LAST(field_key++)) break;
    }
    
    return true;
 }

+int checkreturn pb_get_message_size(const pb_field_info_t *fields, const void *src_struct)
+{
+    pb_ostream_t ostream = PB_OSTREAM_INIT;
+    if (pb_encode(&ostream, fields, src_struct))
+        return ostream.bytes_written;
+    return -1;
+}
+
 /* Helper functions */
 bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value)
 {
@@ -345,14 +356,14 @@ bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, co

 bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src)
 {
-    pb_ostream_t substream = {0};
+    pb_ostream_t substream = {0, 0, 0, 0};
    size_t size;
    bool status;
    
    if (field->ptr == NULL)
        return false;
    
-    if (!pb_encode(&substream, (pb_field_t*)field->ptr, src))
+    if (!pb_encode(&substream, (pb_field_info_t*)field->ptr, src))
        return false;
    
    size = substream.bytes_written;
@@ -373,7 +384,7 @@ bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field
    substream.max_size = size;
    substream.bytes_written = 0;
    
-    status = pb_encode(&substream, (pb_field_t*)field->ptr, src);
+    status = pb_encode(&substream, (pb_field_info_t*)field->ptr, src);
    
    stream->bytes_written += substream.bytes_written;
    stream->state = substream.state;
--- a/pb_encode.h
+++ b/pb_encode.h
@@ -42,7 +42,9 @@ bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count);
 * The actual struct pointed to by src_struct must match the description in fields.
 * All required fields in the struct are assumed to have been filled in.
 */
-bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct);
+bool pb_encode(pb_ostream_t *stream, const pb_field_info_t *fields, const void *src_struct);
+
+int pb_get_message_size(const pb_field_info_t *fields, const void *src_struct);

 /* --- Helper functions ---
 * You may want to use these from your caller or callbacks.
@@ -69,4 +71,5 @@ bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src
 bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src);
 bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src);

+#define PB_OSTREAM_INIT {0,0,0,0}
 #endif
--- a/pb_field.c
+++ b/pb_field.c
@@ -0,0 +1,98 @@
+#include "pb_field.h"
+
+#ifdef __GNUC__
+/* Verify that we remember to check all return values for proper error propagation */
+#define checkreturn __attribute__((warn_unused_result))
+#else
+#define checkreturn
+#endif
+
+/**
+ * pb_common_aligned_field_info array holds the common field info that can be shared
+ * by any numeric fields whose previous field is memory aligned
+ */
+const pb_field_t pb_common_aligned_field_info[NUM_COMMON_FIELD_INFO] = {
+	/* REQUIRED bool */
+    {0, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, 0, 0, sizeof(bool), 0, 0},
+    /* OPTIONAL bool */
+    {0, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, 1, -1, sizeof(bool), 0, 0},
+	/* REQUIRED int32, uint32, enum */
+    {0, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, 0, 0, sizeof(uint32_t), 0, 0},
+    /* OPTIONAL int32, uint32, enum */
+    {0, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, 4, -4, sizeof(uint32_t), 0, 0},
+	/* REQUIRED int64, uint64 */
+    {0, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, 0, 0, sizeof(uint64_t), 0, 0},
+    /* OPTIONAL int64, uint64 */
+    {0, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, 4, -4, sizeof(uint64_t), 0, 0},
+    /* REQUIRED sint32 */
+    {0, PB_HTYPE_REQUIRED | PB_LTYPE_SVARINT, 0, 0, sizeof(int32_t), 0, 0},
+    /* OPTIONAL sint32 */
+    {0, PB_HTYPE_OPTIONAL | PB_LTYPE_SVARINT, 4, -4, sizeof(int32_t), 0, 0},
+    /* REQUIRED sint64 */
+    {0, PB_HTYPE_REQUIRED | PB_LTYPE_SVARINT, 0, 0, sizeof(int64_t), 0, 0},
+    /* OPTIONAL sint64 */
+    {0, PB_HTYPE_OPTIONAL | PB_LTYPE_SVARINT, 4, -4, sizeof(int64_t), 0, 0},
+    /* REQUIRED fixed32, sfixed32, float */
+    {0, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED32, 0, 0, sizeof(uint32_t), 0, 0},
+    /* OPTIONAL fixed32, sfixed32, float */
+    {0, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED32, 4, -4, sizeof(uint32_t), 0, 0},
+    /* REQUIRED fixed64, sfixed64, double */
+    {0, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED64, 0, 0, sizeof(uint64_t), 0, 0},
+    /* OPTIONAL fixed64, sfixed64, double */
+    {0, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED64, 4, -4, sizeof(uint64_t), 0, 0}
+};
+
+void pb_field_init(pb_field_iterator_t *iter, const pb_field_info_t *fields, void *dest_struct)
+{
+    if (fields->field_keys == NULL) return;
+    iter->start_key = iter->current_key = fields->field_keys;
+    iter->field_info = fields->field_info;
+    iter->current = *(PB_FIELD_INFO(iter->current_key, iter->field_info));
+    iter->current.tag = PB_TAG_VAL(iter->current_key);
+    iter->field_index = 0;
+    iter->pData = (char*)dest_struct + iter->current.data_offset;
+    iter->pSize = (char*)iter->pData + iter->current.size_offset;
+    iter->dest_struct = dest_struct;
+}
+
+bool pb_field_next(pb_field_iterator_t *iter)
+{
+    bool notwrapped = true;
+    size_t prev_size = iter->current.data_size;
+
+    if (PB_HTYPE(iter->current.type) == PB_HTYPE_ARRAY)
+        prev_size *= iter->current.array_size;
+
+    iter->field_index++;
+    if (PB_IS_LAST(iter->current_key))
+    {
+        iter->current_key = iter->start_key;
+        iter->current = *(PB_FIELD_INFO(iter->current_key, iter->field_info));
+        iter->current.tag = PB_TAG_VAL(iter->current_key);
+        iter->field_index = 0;
+        iter->pData = iter->dest_struct;
+        prev_size = 0;
+        notwrapped = false;
+    } else {
+        iter->current_key++;
+        iter->current = *(PB_FIELD_INFO(iter->current_key, iter->field_info));
+        iter->current.tag = PB_TAG_VAL(iter->current_key);
+    }
+
+    iter->pData = (char*)iter->pData + prev_size + iter->current.data_offset;
+    iter->pSize = (char*)iter->pData + iter->current.size_offset;
+    return notwrapped;
+}
+
+bool checkreturn pb_field_find(pb_field_iterator_t *iter, int tag)
+{
+    int start = iter->field_index;
+
+    do {
+        if (iter->current.tag == tag)
+            return true;
+        pb_field_next(iter);
+    } while (iter->field_index != start);
+
+    return false;
+}
--- a/pb_field.h
+++ b/pb_field.h
@@ -0,0 +1,211 @@
+#ifndef _PB_FIELD_H_
+#define _PB_FIELD_H_
+
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+
+
+#ifdef __GNUC__
+/* This just reduces memory requirements, but is not required. */
+#define pb_packed __attribute__((packed))
+#else
+#define pb_packed
+#endif
+
+/* List of possible field types. These are used in the autogenerated code.
+ * Least-significant 4 bits tell the scalar type
+ * Most-significant 4 bits specify repeated/required/packed etc.
+ *
+ * INT32 and UINT32 are treated the same, as are (U)INT64 and (S)FIXED*
+ * These types are simply casted to correct field type when they are
+ * assigned to the memory pointer.
+ * SINT* is different, though, because it is zig-zag coded.
+ */
+
+typedef enum {
+    /************************
+     * Field contents types *
+     ************************/
+
+    /* Numeric types */
+    PB_LTYPE_VARINT = 0x00, /* int32, uint32, int64, uint64, bool, enum */
+    PB_LTYPE_SVARINT = 0x01, /* sint32, sint64 */
+    PB_LTYPE_FIXED32 = 0x02, /* fixed32, sfixed32, float */
+    PB_LTYPE_FIXED64 = 0x03, /* fixed64, sfixed64, double */
+
+    /* Marker for last packable field type. */
+    PB_LTYPE_LAST_PACKABLE = 0x03,
+
+    /* Byte array with pre-allocated buffer.
+     * data_size is the length of the allocated PB_BYTES_ARRAY structure. */
+    PB_LTYPE_BYTES = 0x04,
+
+    /* String with pre-allocated buffer.
+     * data_size is the maximum length. */
+    PB_LTYPE_STRING = 0x05,
+
+    /* Submessage
+     * submsg_fields is pointer to field descriptions */
+    PB_LTYPE_SUBMESSAGE = 0x06,
+
+    /* Number of declared LTYPES */
+    PB_LTYPES_COUNT = 7,
+
+    /******************
+     * Modifier flags *
+     ******************/
+
+    /* Just the basic, write data at data_offset */
+    PB_HTYPE_REQUIRED = 0x00,
+
+    /* Write true at size_offset */
+    PB_HTYPE_OPTIONAL = 0x10,
+
+    /* Read to pre-allocated array
+     * Maximum number of entries is array_size,
+     * actual number is stored at size_offset */
+    PB_HTYPE_ARRAY = 0x20,
+
+    /* Works for all required/optional/repeated fields.
+     * data_offset points to pb_callback_t structure.
+     * LTYPE should be 0 (it is ignored, but sometimes
+     * used to speculatively index an array). */
+    PB_HTYPE_CALLBACK = 0x30
+
+} pb_packed pb_type_t;
+
+#define PB_HTYPE(x) ((x) & 0xF0)
+#define PB_LTYPE(x) ((x) & 0x0F)
+
+/* This structure is used for 'bytes' arrays.
+ * It has the number of bytes in the beginning, and after that an array.
+ * Note that actual structs used will have a different length of bytes array.
+ */
+typedef struct {
+    size_t size;
+    uint8_t bytes[1];
+} pb_bytes_array_t;
+
+/* The next three pb_field_ structures are used in auto-generated constants
+ * to specify struct fields.
+ */
+
+typedef struct _pb_field_key_t pb_field_key_t;
+typedef struct _pb_field_t pb_field_t;
+typedef struct _pb_field_info_t pb_field_info_t;
+/* pb_field_key_t is a structure for storing the tag and an index into an array
+ * that stores the actual field info. This approach is used to reduce code size
+ * as many numeric fields have the same field info.
+ *
+ * A uint8_t tag will limit the field tag to 128 as the MSB bit is used to indicate
+ * whether or not it's the last field in the message. You may change it to uint16_t
+ * if you want to specify larger field tag. Please also change the corresponding
+ * tag in pb_field_t
+ */
+typedef uint8_t tag_t;
+struct _pb_field_key_t {
+    tag_t tag;     /* The MSB bit of the tag is used to indicate it's the last field in the message */
+    /* The MSB bit of the info_index is used to indicate the field info is retrieved
+     * from the global pb_common_aligned_field_info array instead of the auto-generated field
+     * info.
+     */
+    uint8_t info_index;
+} pb_packed;
+
+/* pb_field_t is a structure for storing the pb field descriptor information
+ * You can change field sizes here if you need structures
+ * larger than 256 bytes
+ */
+struct _pb_field_t {
+    tag_t tag;
+    pb_type_t type;
+    uint8_t data_offset; /* Offset of field data, relative to previous field. */
+    int8_t size_offset; /* Offset of array size or has-boolean, relative to data */
+    uint8_t data_size; /* Data size in bytes for a single item */
+    uint8_t array_size; /* Maximum number of entries in array */
+
+    /* Field definitions for submessage
+     * OR default value for all other non-array, non-callback types
+     * If null, then field will zeroed. */
+    const void *ptr;
+} pb_packed;
+
+/* pb_field_info_t is the structure for holding together the key and the field info arrays.
+ * This is the structure that will be passed to pb_encode and pb_decode
+ */
+struct _pb_field_info_t {
+    const pb_field_key_t *field_keys;
+    const pb_field_t *field_info;
+};
+
+/* Iterator for pb_field_t list */
+typedef struct {
+    const pb_field_key_t *start_key;
+    const pb_field_key_t *current_key;
+    pb_field_t current;
+    const pb_field_t *field_info;
+    int field_index;
+    void *dest_struct;
+    void *pData;
+    void *pSize;
+} pb_field_iterator_t;
+
+/* Functions for iterating through the field */
+void pb_field_init(pb_field_iterator_t *iter, const pb_field_info_t *fields, void *dest_struct);
+bool pb_field_next(pb_field_iterator_t *iter);
+bool pb_field_find(pb_field_iterator_t *iter, int tag);
+
+/* These macros are used to declare pb_field_t's in the constant array. */
+#define pb_membersize(st, m) (sizeof ((st*)0)->m)
+#define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0]))
+#define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2))
+#define pb_delta_end(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2))
+
+/* The MSB bit of the pb_field_key_t.tag indicates whether or not it's the last element */
+#define PB_LAST_FIELD (0x1 << (pb_membersize(pb_field_key_t, tag) * 8 - 1))
+#define PB_IS_LAST(key) ((key)->tag & PB_LAST_FIELD)
+#define PB_TAG_VAL(key) ((key)->tag & ~PB_LAST_FIELD)
+
+/* The MSB bit of the pb_field_key_tag.info_index indicates whether it's a common or generated field info */
+#define PB_COMMON_INFO_FLAG (0x1 << (pb_membersize(pb_field_key_t, info_index) * 8 -1))
+
+/* A common field info is retrieved from the globally declared pb_common_aligned_field_info array or
+ * locally auto-generated field info array */
+#define PB_FIELD_INFO(key, info) (((key)->info_index & PB_COMMON_INFO_FLAG) ? \
+    &pb_common_aligned_field_info[(key)->info_index ^ PB_COMMON_INFO_FLAG] : \
+    &((info)[(key)->info_index]))
+
+/* Defines the indices for pb_common_aligned_field_info array
+ * Note that MSB bit PB_COMMON_INFO_FLAG is set to indicate the index
+ * should be used on pb_common_aligned_field_info instead of generated
+ * array
+ * */
+enum {
+    REQUIRED_BOOL_INFO = 0 | PB_COMMON_INFO_FLAG,
+    OPTIONAL_BOOL_INFO = 1 | PB_COMMON_INFO_FLAG,
+    REQUIRED_INT32_INFO = 2 | PB_COMMON_INFO_FLAG,
+    OPTIONAL_INT32_INFO = 3 | PB_COMMON_INFO_FLAG,
+    REQUIRED_INT64_INFO = 4 | PB_COMMON_INFO_FLAG,
+    OPTIONAL_INT64_INFO = 5 | PB_COMMON_INFO_FLAG,
+    REQUIRED_SINT32_INFO = 6 | PB_COMMON_INFO_FLAG,
+    OPTIONAL_SINT32_INFO = 7 | PB_COMMON_INFO_FLAG,
+    REQUIRED_SINT64_INFO = 8 | PB_COMMON_INFO_FLAG,
+    OPTIONAL_SINT64_INFO = 9 | PB_COMMON_INFO_FLAG,
+    REQUIRED_FIXED32_INFO = 10 | PB_COMMON_INFO_FLAG,
+    OPTIONAL_FIXED32_INFO = 11 | PB_COMMON_INFO_FLAG,
+    REQUIRED_FIXED64_INFO = 12 | PB_COMMON_INFO_FLAG,
+    OPTIONAL_FIXED64_INFO = 13 | PB_COMMON_INFO_FLAG,
+
+    /* leave this enum to determine num of elements */
+    MAX_COMMON_FIELD_INFO
+};
+
+#define NUM_COMMON_FIELD_INFO (MAX_COMMON_FIELD_INFO ^ PB_COMMON_INFO_FLAG)
+
+/* pb_common_aligned_field_info array holds the common field info that can be shared
+ * by any numeric fields whose previous field is memory aligned
+ */
+extern const pb_field_t pb_common_aligned_field_info[NUM_COMMON_FIELD_INFO];
+
+#endif /* _PB_FIELD_H_ */
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,12 +1,23 @@
 CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage
 LDFLAGS=--coverage
-DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h
-TESTS=test_decode1 test_encode1 decode_unittests encode_unittests
+DEPS=../pb_decode.h ../pb_encode.h ../pb.h ../pb_field.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h aligntype.pb.h
+TESTS=test_decode1 test_encode1 test_encode2 test_decode2 test_encode3 test_decode3 test_encode4 test_decode4 decode_unittests encode_unittests test_encode_callbacks test_decode_callbacks
+
+CC_VER := $(shell gcc --version | grep gcc)
+ifneq "$(CC_VER)" ""
+CFLAGS += -m32
+LDFLAGS += -m32
+endif
+ifndef PB_PATH
+PBPATHOPT=-I/usr/include -I/usr/local/include
+else
+PBPATHOPT=-I$(PB_PATH)
+endif

 all: breakpoints $(TESTS) run_unittests

 clean:
-	rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno
+	rm -f breakpoints $(TESTS) *.pb *.pb.c *.pb.h *.o *.gcda *.gcno

 %.o: %.c
 %.o: %.c $(DEPS)
@@ -16,20 +27,24 @@ pb_encode.o: ../pb_encode.c $(DEPS)
 	$(CC) $(CFLAGS) -c -o $@ $<
 pb_decode.o: ../pb_decode.c $(DEPS)
 	$(CC) $(CFLAGS) -c -o $@ $<
+pb_field.o: ../pb_field.c $(DEPS)
+	$(CC) $(CFLAGS) -c -o $@ $<

-test_decode1: test_decode1.o pb_decode.o person.pb.o
-test_decode2: test_decode2.o pb_decode.o person.pb.o
-test_decode3: test_decode3.o pb_decode.o alltypes.pb.o
-test_encode1: test_encode1.o pb_encode.o person.pb.o
-test_encode2: test_encode2.o pb_encode.o person.pb.o
-test_encode3: test_encode3.o pb_encode.o alltypes.pb.o
-test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o
-test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o
-decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o
-encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o
+test_decode1: test_decode1.o pb_decode.o pb_field.o person.pb.o
+test_decode2: test_decode2.o pb_decode.o pb_field.o person.pb.o
+test_decode3: test_decode3.o pb_decode.o pb_field.o alltypes.pb.o
+test_decode4: test_decode4.o pb_decode.o pb_field.o aligntype.pb.o
+test_encode1: test_encode1.o pb_encode.o pb_field.o person.pb.o
+test_encode2: test_encode2.o pb_encode.o pb_field.o person.pb.o
+test_encode3: test_encode3.o pb_encode.o pb_field.o alltypes.pb.o
+test_encode4: test_encode4.o pb_encode.o pb_field.o aligntype.pb.o
+test_decode_callbacks: test_decode_callbacks.o pb_decode.o pb_field.o callbacks.pb.o
+test_encode_callbacks: test_encode_callbacks.o pb_encode.o pb_field.o callbacks.pb.o
+decode_unittests: decode_unittests.o pb_decode.o pb_field.o unittestproto.pb.o
+encode_unittests: encode_unittests.o pb_encode.o pb_field.o unittestproto.pb.o

 %.pb: %.proto
-	protoc -I. -I../generator -I/usr/include -o$@ $<
+	protoc -I. -I../generator $(PBPATHOPT) -o$@ $<

 %.pb.c %.pb.h: %.pb ../generator/nanopb_generator.py
 	python ../generator/nanopb_generator.py $<
@@ -41,26 +56,29 @@ coverage: run_unittests
 	gcov pb_encode.gcda
 	gcov pb_decode.gcda

-run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks
+run_unittests: $(TESTS)
 	rm -f *.gcda
 	
 	./decode_unittests > /dev/null
 	./encode_unittests > /dev/null
 	
 	[ "`./test_encode1 | ./test_decode1`" = \
-	"`./test_encode1 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ]
+	"`./test_encode1 | protoc --decode=Person -I. -I../generator $(PBPATHOPT) person.proto`" ]

 	[ "`./test_encode2 | ./test_decode1`" = \
-	"`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ]
+	"`./test_encode2 | protoc --decode=Person -I. -I../generator $(PBPATHOPT) person.proto`" ]

 	[ "`./test_encode2 | ./test_decode2`" = \
-	"`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ]
+	"`./test_encode2 | protoc --decode=Person -I. -I../generator $(PBPATHOPT) person.proto`" ]
 	
 	[ "`./test_encode_callbacks | ./test_decode_callbacks`" = \
 	"`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ]

 	./test_encode3 | ./test_decode3
-	./test_encode3 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null
+	./test_encode3 | protoc --decode=AllTypes -I. -I../generator $(PBPATHOPT) alltypes.proto >/dev/null
+
+	./test_encode4 | ./test_decode4
+	./test_encode4 | protoc --decode=AlignTypes -I. -I../generator $(PBPATHOPT) aligntype.proto >/dev/null

 run_fuzztest: test_decode2
 	bash -c 'I=1; while true; do cat /dev/urandom | ./test_decode2 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done'
--- a/tests/aligntype.proto
+++ b/tests/aligntype.proto
@@ -0,0 +1,80 @@
+import "nanopb.proto";
+
+message SubMessage {
+    required string substuff1 = 1 [(nanopb).max_size = 16];
+    required int32 substuff2 = 2;
+}
+
+enum MyEnum {
+    First = 1;
+    Second = 2;
+    Truth = 42;
+}
+
+message AlignTypes {
+    required bool       req_bool    = 1;
+    required int32      req_int32   = 2;
+    required SubMessage req_submsg  = 3;
+    required int64      req_int64   = 4;
+    required string     req_string  = 5 [(nanopb).max_size = 16];
+    required MyEnum     req_enum    = 6;
+    required bytes      req_bytes   = 7 [(nanopb).max_size = 16];
+    required sint32     req_sint32  = 8;
+    required bool       req_bool_2  = 9;
+    required sint64     req_sint64  = 10;
+    required bool       req_bool_3  = 11;
+    required fixed32    req_fixed32 = 12;
+    required bool       req_bool_4  = 13;
+    required fixed64    req_fixed64 = 14;
+    required bool       req_bool_5  = 15;
+    required float      req_float   = 16;
+    required bool       req_bool_6  = 17;
+    required double     req_double  = 18;
+
+    optional bool       opt_bool    = 19;
+    optional int32      opt_int32   = 20;
+    optional SubMessage opt_submsg  = 21;
+    optional int64      opt_int64   = 22;
+    optional string     opt_string  = 23 [(nanopb).max_size = 16];
+    optional MyEnum     opt_enum    = 24;
+    optional bytes      opt_bytes   = 25 [(nanopb).max_size = 16];
+    optional sint32     opt_sint32  = 26;
+    optional bool       opt_bool_2  = 27;
+    optional sint64     opt_sint64  = 28;
+    optional bool       opt_bool_3  = 29;
+    optional fixed32    opt_fixed32 = 30;
+    optional bool       opt_bool_4  = 31;
+    optional fixed64    opt_fixed64 = 32;
+    optional bool       opt_bool_5  = 33;
+    optional float      opt_float   = 34;
+    optional bool       opt_bool_6  = 35;
+    optional double     opt_double  = 36;
+
+    required bool       req_bool_aligned    = 37; 
+    required int32      req_int32_aligned   = 38;
+    required int64      req_int64_aligned   = 39;
+    required MyEnum     req_enum_aligned    = 40;
+    required sint32     req_sint32_aligned  = 41;
+    required sint64     req_sint64_aligned  = 42;
+    required fixed32    req_fixed32_aligned = 43;
+    required fixed64    req_fixed64_aligned = 44;
+    required float      req_float_aligned   = 45;
+    required double     req_double_aligned  = 46;
+
+    optional bool       opt_bool_aligned    = 47;
+    optional int32      opt_int32_aligned   = 48;
+    optional int64      opt_int64_aligned   = 49;
+    optional MyEnum     opt_enum_aligned    = 50;
+    optional sint32     opt_sint32_aligned  = 51;
+    optional sint64     opt_sint64_aligned  = 52;
+    optional fixed32    opt_fixed32_aligned = 53;
+    optional fixed64    opt_fixed64_aligned = 54;
+    optional float      opt_float_aligned   = 55;
+    optional double     opt_double_aligned  = 56;
+
+
+    // Just to make sure that the size of the fields has been calculated
+    // properly, i.e. otherwise a bug in last field might not be detected.
+    required int32      end = 99;
+}
+
--- a/tests/test_decode4.c
+++ b/tests/test_decode4.c
@@ -0,0 +1,139 @@
+/* Tests the decoding of all types. Currently only in the 'required' variety.
+ * This is the counterpart of test_encode3.
+ * Run e.g. ./test_encode3 | ./test_decode3
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <pb_decode.h>
+#include "aligntype.pb.h"
+
+#define TEST(x) if (!(x)) { \
+    printf("Test " #x " failed.\n"); \
+    return false; \
+    }
+
+/* This function is called once from main(), it handles
+   the decoding and checks the fields. */
+bool check_aligntypes(pb_istream_t *stream)
+{
+    AlignTypes aligntypes = {};
+    
+    if (!pb_decode(stream, AlignTypes_fields, &aligntypes))
+        return false;
+    
+    TEST(aligntypes.req_bool          == true);
+    TEST(aligntypes.req_int32         == 1001);
+    TEST(strcmp(aligntypes.req_submsg.substuff1, "1002") == 0);
+    TEST(aligntypes.req_submsg.substuff2 == 1002);
+    TEST(aligntypes.req_int64         == 1003);
+    TEST(strcmp(aligntypes.req_string, "1004") == 0);
+    TEST(aligntypes.req_enum == MyEnum_Truth);
+    TEST(aligntypes.req_bytes.size == 4);
+    TEST(memcmp(aligntypes.req_bytes.bytes, "1005", 4) == 0);
+    TEST(aligntypes.req_sint32        == 1006);
+    TEST(aligntypes.req_bool_2        == true);
+    TEST(aligntypes.req_sint64        == 1007);
+    TEST(aligntypes.req_bool_3        == true);
+    TEST(aligntypes.req_fixed32       == 1008);
+    TEST(aligntypes.req_bool_4        == true);
+    TEST(aligntypes.req_fixed64       == 1009);
+    TEST(aligntypes.req_bool_5        == true);
+    TEST(aligntypes.req_float         == 1010.0f);
+    TEST(aligntypes.req_bool_5        == true);
+    TEST(aligntypes.req_double        == 1011.0f);
+   
+    TEST(aligntypes.has_opt_bool      == true);
+    TEST(aligntypes.opt_bool          == true);
+    TEST(aligntypes.has_opt_int32     == true);
+    TEST(aligntypes.opt_int32         == 1001);
+    TEST(aligntypes.has_opt_submsg    == true);
+    TEST(strcmp(aligntypes.opt_submsg.substuff1, "1002") == 0);
+    TEST(aligntypes.opt_submsg.substuff2 == 1002);
+    TEST(aligntypes.has_opt_int64     == true);
+    TEST(aligntypes.opt_int64         == 1003);
+    TEST(aligntypes.has_opt_string    == true);
+    TEST(strcmp(aligntypes.opt_string, "1004") == 0);
+    TEST(aligntypes.has_opt_enum      == true);
+    TEST(aligntypes.opt_enum == MyEnum_Truth);
+    TEST(aligntypes.has_opt_bytes     == true);
+    TEST(aligntypes.opt_bytes.size == 4);
+    TEST(memcmp(aligntypes.opt_bytes.bytes, "1005", 4) == 0);
+    TEST(aligntypes.has_opt_sint32    == true);
+    TEST(aligntypes.opt_sint32        == 1006);
+    TEST(aligntypes.has_opt_bool_2    == true);
+    TEST(aligntypes.opt_bool_2        == true);
+    TEST(aligntypes.has_opt_sint64    == true);
+    TEST(aligntypes.opt_sint64        == 1007);
+    TEST(aligntypes.has_opt_bool_3    == true);
+    TEST(aligntypes.opt_bool_3        == true);
+    TEST(aligntypes.has_opt_fixed32   == true);
+    TEST(aligntypes.opt_fixed32       == 1008);
+    TEST(aligntypes.has_opt_bool_4    == true);
+    TEST(aligntypes.opt_bool_4        == true);
+    TEST(aligntypes.has_opt_fixed64   == true);
+    TEST(aligntypes.opt_fixed64       == 1009);
+    TEST(aligntypes.has_opt_bool_5    == true);
+    TEST(aligntypes.opt_bool_5        == true);
+    TEST(aligntypes.has_opt_float     == true);
+    TEST(aligntypes.opt_float         == 1010.0f);
+    TEST(aligntypes.has_opt_bool_6    == true);
+    TEST(aligntypes.opt_bool_6        == true);
+    TEST(aligntypes.has_opt_double    == true);
+    TEST(aligntypes.opt_double        == 1011.0f);
+
+    TEST(aligntypes.req_bool_aligned   == true);
+    TEST(aligntypes.req_int32_aligned  == 1001);
+    TEST(aligntypes.req_int64_aligned  == 1003);
+    TEST(aligntypes.req_enum_aligned   == MyEnum_Truth);
+    TEST(aligntypes.req_sint32_aligned == 1006);
+    TEST(aligntypes.req_sint64_aligned == 1007);
+    TEST(aligntypes.req_fixed32_aligned == 1008);
+    TEST(aligntypes.req_fixed64_aligned == 1009);
+    TEST(aligntypes.req_float_aligned  == 1010.0f);
+    TEST(aligntypes.req_double_aligned == 1011.0f);
+
+    TEST(aligntypes.has_opt_bool_aligned      == true);
+    TEST(aligntypes.opt_bool_aligned          == true);
+    TEST(aligntypes.has_opt_int32_aligned     == true);
+    TEST(aligntypes.opt_int32_aligned         == 1001);
+    TEST(aligntypes.has_opt_int64_aligned     == true);
+    TEST(aligntypes.opt_int64_aligned         == 1003);
+    TEST(aligntypes.has_opt_enum_aligned      == true);
+    TEST(aligntypes.opt_enum_aligned == MyEnum_Truth);
+    TEST(aligntypes.has_opt_sint32_aligned    == true);
+    TEST(aligntypes.opt_sint32_aligned        == 1006);
+    TEST(aligntypes.has_opt_sint64_aligned    == true);
+    TEST(aligntypes.opt_sint64_aligned        == 1007);
+    TEST(aligntypes.has_opt_fixed32_aligned   == true);
+    TEST(aligntypes.opt_fixed32_aligned       == 1008);
+    TEST(aligntypes.has_opt_fixed64_aligned   == true);
+    TEST(aligntypes.opt_fixed64_aligned       == 1009);
+    TEST(aligntypes.has_opt_float_aligned     == true);
+    TEST(aligntypes.opt_float_aligned         == 1010.0f);
+    TEST(aligntypes.has_opt_double_aligned    == true);
+    TEST(aligntypes.opt_double_aligned        == 1011.0f);
+
+    TEST(aligntypes.end == 1099);
+    
+    return true;
+}
+
+int main()
+{
+    /* Read the data into buffer */
+    uint8_t buffer[512];
+    size_t count = fread(buffer, 1, sizeof(buffer), stdin);
+    
+    /* Construct a pb_istream_t for reading from the buffer */
+    pb_istream_t stream = pb_istream_from_buffer(buffer, count);
+    
+    /* Decode and print out the stuff */
+    if (!check_aligntypes(&stream))
+    {
+        printf("Parsing failed.\n");
+        return 1;
+    } else {
+        return 0;
+    }
+}
--- a/tests/test_encode4.c
+++ b/tests/test_encode4.c
@@ -0,0 +1,117 @@
+/* Attempts to test all the datatypes supported by ProtoBuf.
+ * Currently only tests the 'required' variety.
+ */
+
+#include <stdio.h>
+#include <pb_encode.h>
+#include "aligntype.pb.h"
+
+int main()
+{
+    /* Initialize the structure with constants */
+    AlignTypes aligntypes = {
+        true,
+        1001,
+        {"1002", 1002},
+        1003,
+        "1004",
+        MyEnum_Truth,
+        {4, "1005"},
+        1006,
+        true,
+        1007,
+        true,
+        1008,
+        true,
+        1009,
+        true,
+        1010.0f,
+        true,
+        1011.0f,
+
+        true,
+        true,
+        true,
+        1001,
+        true,
+        {"1002", 1002},
+        true,
+        1003,
+        true,
+        "1004",
+        true,
+        MyEnum_Truth,
+        true,
+        {4, "1005"},
+        true,
+        1006,
+        true,
+        true,
+        true,
+        1007,
+        true,
+        true,
+        true,
+        1008,
+        true,
+        true,
+        true,
+        1009,
+        true,
+        true,
+        true,
+        1010.0f,
+        true,
+        true,
+        true,
+        1011.0f,
+        
+        true,
+        1001,
+        1003,
+        MyEnum_Truth,
+        1006,
+        1007,
+        1008,
+        1009,
+        1010.0f,
+        1011.0f,
+
+        true,
+        true,
+        true,
+        1001,
+        true,
+        1003,
+        true,
+        MyEnum_Truth,
+        true,
+        1006,
+        true,
+        1007,
+        true,
+        1008,
+        true,
+        1009,
+        true,
+        1010.0f,
+        true,
+        1011.0f,
+
+        1099
+    };
+    
+    uint8_t buffer[512];
+    pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+    
+    /* Now encode it and check if we succeeded. */
+    if (pb_encode(&stream, AlignTypes_fields, &aligntypes))
+    {
+        fwrite(buffer, 1, stream.bytes_written, stdout);
+        return 0; /* Success */
+    }
+    else
+    {
+        return 1; /* Failure */
+    }
+}
--- a/tests/testperson.pb
+++ b/tests/testperson.pb
@@ -1,3 +0,0 @@
-
-Test Person7foobar@foobar.com"
-555-12345678
Author	SHA1	Message	Date
Daniel Kan	6d010fb7c4	Fixed pb_get_message_size	2012-02-27 14:46:38 -08:00
Daniel Kan	34aa2031c4	Code reduction optimization by sharing common field descriptor information for numeric field types	2012-02-24 13:46:36 -08:00