diff --git a/README b/README index 0fc79c4..da8b879 100644 --- a/README +++ b/README @@ -6,4 +6,13 @@ To compile the library, you'll need these libraries: protobuf-compiler python-protobuf libprotobuf-dev To run the tests, run make under the tests folder. -If it completes without error, everything is fine. +If it completes without error, everything is fine. + +Code size optimization is currently only supported for 32-bit +architecture. If you want to run on 64-bit architecture, +you must disable code size optimization by providing -n option to +code generator nanopb_generator.py. + +For testing purpose, you may need to pass -m32 compiler to gcc if +you're running on a 64-bit machine unless -n option is used for code +generation. diff --git a/example/Makefile b/example/Makefile index 14dd9fa..aac2908 100644 --- a/example/Makefile +++ b/example/Makefile @@ -1,14 +1,24 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 -DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h +DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_field.c ../pb_encode.h ../pb.h ../pb_field.h + +CC_VER := $(shell gcc --version | grep gcc) +ifneq "$(CC_VER)" "" +CFLAGS += -m32 +endif +ifndef PB_PATH +PBPATHOPT=-I/usr/include -I/usr/local/include +else +PBPATHOPT=-I$(PB_PATH) +endif all: server client clean: - rm -f server client fileproto.pb.c fileproto.pb.h + rm -f server client fileproto.pb.c fileproto.pb.h fileproto.pb %: %.c $(DEPS) fileproto.pb.h fileproto.pb.c - $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c fileproto.pb.c common.c + $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c ../pb_field.c fileproto.pb.c common.c fileproto.pb.c fileproto.pb.h: fileproto.proto ../generator/nanopb_generator.py - protoc -I. -I../generator -I/usr/include -ofileproto.pb $< + protoc -I. -I../generator $(PBPATHOPT) -ofileproto.pb $< python ../generator/nanopb_generator.py fileproto.pb diff --git a/example/common.h b/example/common.h index 8dab3b7..bdd8a14 100644 --- a/example/common.h +++ b/example/common.h @@ -6,4 +6,5 @@ pb_ostream_t pb_ostream_from_socket(int fd); pb_istream_t pb_istream_from_socket(int fd); -#endif \ No newline at end of file +#endif + diff --git a/generator/Makefile b/generator/Makefile index 161ef38..4b512ad 100644 --- a/generator/Makefile +++ b/generator/Makefile @@ -1,2 +1,8 @@ +ifndef PB_PATH +PBPATHOPT=-I/usr/include -I/usr/local/include +else +PBPATHOPT=-I$(PB_PATH) +endif + nanopb_pb2.py: nanopb.proto - protoc --python_out=. -I /usr/include -I . nanopb.proto + protoc --python_out=. $(PBPATHOPT) -I . nanopb.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2cc92f0..38ecee4 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -4,22 +4,22 @@ import google.protobuf.descriptor_pb2 as descriptor import nanopb_pb2 import os.path -# Values are tuple (c type, pb ltype) +# Values are tuple (c type, pb ltype, memory aligned flag, common field info type) FieldD = descriptor.FieldDescriptorProto datatypes = { - FieldD.TYPE_BOOL: ('bool', 'PB_LTYPE_VARINT'), - FieldD.TYPE_DOUBLE: ('double', 'PB_LTYPE_FIXED64'), - FieldD.TYPE_FIXED32: ('uint32_t', 'PB_LTYPE_FIXED32'), - FieldD.TYPE_FIXED64: ('uint64_t', 'PB_LTYPE_FIXED64'), - FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED32'), - FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT'), - FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT'), - FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED32'), - FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED64'), - FieldD.TYPE_SINT32: ('int32_t', 'PB_LTYPE_SVARINT'), - FieldD.TYPE_SINT64: ('int64_t', 'PB_LTYPE_SVARINT'), - FieldD.TYPE_UINT32: ('uint32_t', 'PB_LTYPE_VARINT'), - FieldD.TYPE_UINT64: ('uint64_t', 'PB_LTYPE_VARINT') + FieldD.TYPE_BOOL: ('bool', 'PB_LTYPE_VARINT', False, 'BOOL'), + FieldD.TYPE_DOUBLE: ('double', 'PB_LTYPE_FIXED64', True, 'FIXED64'), + FieldD.TYPE_FIXED32: ('uint32_t', 'PB_LTYPE_FIXED32', True, 'FIXED32'), + FieldD.TYPE_FIXED64: ('uint64_t', 'PB_LTYPE_FIXED64', True, 'FIXED64'), + FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED32', True, 'FIXED32'), + FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT', True, 'INT32'), + FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT', True, 'INT64'), + FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED32', True, 'FIXED32'), + FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED64', True, 'FIXED64'), + FieldD.TYPE_SINT32: ('int32_t', 'PB_LTYPE_SVARINT', True, 'SINT32'), + FieldD.TYPE_SINT64: ('int64_t', 'PB_LTYPE_SVARINT', True, "SINT64"), + FieldD.TYPE_UINT32: ('uint32_t', 'PB_LTYPE_VARINT', True, 'INT32'), + FieldD.TYPE_UINT64: ('uint64_t', 'PB_LTYPE_VARINT', True, 'INT64') } class Names: @@ -70,6 +70,9 @@ class Field: self.max_size = None self.max_count = None self.array_decl = "" + self.is_last_field = False + self.aligned = False + self.info_index = None # Parse nanopb-specific field options if desc.options.HasExtension(nanopb_pb2.nanopb): @@ -104,10 +107,12 @@ class Field: # defining how to decode an individual value. # CTYPE is the name of the c type to use in the struct. if datatypes.has_key(desc.type): - self.ctype, self.ltype = datatypes[desc.type] + self.ctype, self.ltype, self.aligned, self.common_info_name = datatypes[desc.type] elif desc.type == FieldD.TYPE_ENUM: self.ltype = 'PB_LTYPE_VARINT' self.ctype = names_from_type_name(desc.type_name) + self.aligned = True + self.common_info_name = 'INT32' if self.default is not None: self.default = self.ctype + self.default elif desc.type == FieldD.TYPE_STRING: @@ -191,15 +196,33 @@ class Field: return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl) else: return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default) + + def pb_field_key_t(self): + '''Return the pb_field_t field key initializer to use in the constant array. + ''' + result = ' {%d' % self.tag + if self.is_last_field: + result += ' | PB_LAST_FIELD' + + if self.info_index is not None: + result += ', %d}' % self.info_index + else: + if 'REQUIRED' in self.htype: + prefix = 'REQUIRED' + elif 'OPTIONAL' in self.htype: + prefix = 'OPTIONAL' + result += ', %s_%s_INFO}' % (prefix, self.common_info_name) + return result def pb_field_t(self, prev_field_name): - '''Return the pb_field_t initializer to use in the constant array. + '''Return the pb_field_t field info initializer to use in the constant array. prev_field_name is the name of the previous field or None. ''' - result = ' {%d, ' % self.tag + ''' result = ' {0, ''' + result = ' {%d, ' % self.tag result += self.htype if self.ltype is not None: - result += ' | ' + self.ltype + result += ' | ' + self.ltype result += ',\n' if prev_field_name is None: @@ -231,14 +254,26 @@ class Field: result += '\n &%s_default}' % (self.struct_name + self.name) return result - + + def should_generate_field_info(self, prev): + return g_no_optimization or not ((prev is None or prev.aligned) and (self.ltype in ['PB_LTYPE_VARINT', 'PB_LTYPE_SVARINT', 'PB_LTYPE_FIXED32', 'PB_LTYPE_FIXED64']) and (self.htype in ['PB_HTYPE_REQUIRED', 'PB_HTYPE_OPTIONAL'])) + class Message: def __init__(self, names, desc): self.name = names self.fields = [Field(self.name, f) for f in desc.field] self.ordered_fields = self.fields[:] self.ordered_fields.sort() - + if self.ordered_fields: + self.ordered_fields[-1].is_last_field = True + self.num_info = 0 + prev = None + for f in self.ordered_fields: + if f.should_generate_field_info(prev): + f.info_index = self.num_info + self.num_info += 1 + prev = f + def get_dependencies(self): '''Get list of type names that this structure refers to.''' return [str(field.ctype) for field in self.fields] @@ -266,19 +301,48 @@ class Message: return result def fields_declaration(self): - result = 'extern const pb_field_t %s_fields[%d];' % (self.name, len(self.fields) + 1) + result = 'extern const pb_field_info_t %s_fields[1];' % self.name + return result def fields_definition(self): - result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, len(self.fields) + 1) + num_fields = len(self.fields) - prev = None + result = 'static const pb_field_key_t %s_field_keys[%d] = {\n' % (self.name, num_fields) + for field in self.ordered_fields: - result += field.pb_field_t(prev) - result += ',\n\n' - prev = field.name + result += field.pb_field_key_t() + if not field.is_last_field: + result += ',\n' + prev = field + result += '\n};\n\n' + + if self.num_info > 0: + result += 'static const pb_field_t %s_field_info[%d] = {\n' % (self.name, self.num_info) + prev = None + for field in self.ordered_fields: + if field.info_index is not None: + result += field.pb_field_t(prev) + if field.info_index < self.num_info - 1: + result += ',\n\n' + prev = field.name + result += '\n};\n\n' + + decl = 'const pb_field_info_t %s_fields[1] = { {' % self.name + + if num_fields == 0: + result = decl + result += '\n 0,' + result += '\n 0' + else: + result += decl + result += '\n &%s_field_keys[0],' % self.name + if self.num_info > 0: + result += '\n &%s_field_info[0]' % self.name + else: + result += '\n 0' + result += '\n}};\n' - result += ' PB_LAST_FIELD\n};' return result def iterate_messages(desc, names = Names()): @@ -310,7 +374,7 @@ def parse_file(fdesc): enums.append(Enum(base_name, enum)) for names, message in iterate_messages(fdesc, base_name): - messages.append(Message(names, message)) + messages.append(Message(names,message)) for enum in message.enum_type: enums.append(Enum(names, enum)) @@ -401,19 +465,25 @@ def generate_source(headername, enums, messages): if __name__ == '__main__': import sys import os.path + from optparse import OptionParser + usage = "usage: %prog [options] file.pb\n" + \ + "where file.pb has been compiled from .proto by:\n" + \ + " protoc -ofile.pb file.proto\n" + \ + "Output fill be written to file.pb.h and file.pb.c" + + parser = OptionParser(usage=usage) + parser.add_option('-n', '--no-opt', action='store_true', dest='no_optimization', default=False, help='disable code reduction optimization as 64-bit architecture is not supported [default=optimization on]') + (options, args) = parser.parse_args() - if len(sys.argv) != 2: - print "Usage: " + sys.argv[0] + " file.pb" - print "where file.pb has been compiled from .proto by:" - print "protoc -ofile.pb file.proto" - print "Output fill be written to file.pb.h and file.pb.c" + if len(args) != 1: + parser.print_help() sys.exit(1) - - data = open(sys.argv[1], 'rb').read() + g_no_optimization = options.no_optimization + data = open(args[0], 'rb').read() fdesc = descriptor.FileDescriptorSet.FromString(data) enums, messages = parse_file(fdesc.file[0]) - noext = os.path.splitext(sys.argv[1])[0] + noext = os.path.splitext(args[0])[0] headername = noext + '.pb.h' sourcename = noext + '.pb.c' headerbasename = os.path.basename(headername) diff --git a/pb.h b/pb.h index ea94efc..11c0c2a 100644 --- a/pb.h +++ b/pb.h @@ -5,112 +5,7 @@ * Most of these are quite low-level stuff. For the high-level interface, * see pb_encode.h or pb_decode.h */ - -#include -#include -#include - -#ifdef __GNUC__ -/* This just reduces memory requirements, but is not required. */ -#define pb_packed __attribute__((packed)) -#else -#define pb_packed -#endif - -/* List of possible field types. These are used in the autogenerated code. - * Least-significant 4 bits tell the scalar type - * Most-significant 4 bits specify repeated/required/packed etc. - * - * INT32 and UINT32 are treated the same, as are (U)INT64 and (S)FIXED* - * These types are simply casted to correct field type when they are - * assigned to the memory pointer. - * SINT* is different, though, because it is zig-zag coded. - */ - -typedef enum { - /************************ - * Field contents types * - ************************/ - - /* Numeric types */ - PB_LTYPE_VARINT = 0x00, /* int32, uint32, int64, uint64, bool, enum */ - PB_LTYPE_SVARINT = 0x01, /* sint32, sint64 */ - PB_LTYPE_FIXED32 = 0x02, /* fixed32, sfixed32, float */ - PB_LTYPE_FIXED64 = 0x03, /* fixed64, sfixed64, double */ - - /* Marker for last packable field type. */ - PB_LTYPE_LAST_PACKABLE = 0x03, - - /* Byte array with pre-allocated buffer. - * data_size is the length of the allocated PB_BYTES_ARRAY structure. */ - PB_LTYPE_BYTES = 0x04, - - /* String with pre-allocated buffer. - * data_size is the maximum length. */ - PB_LTYPE_STRING = 0x05, - - /* Submessage - * submsg_fields is pointer to field descriptions */ - PB_LTYPE_SUBMESSAGE = 0x06, - - /* Number of declared LTYPES */ - PB_LTYPES_COUNT = 7, - - /****************** - * Modifier flags * - ******************/ - - /* Just the basic, write data at data_offset */ - PB_HTYPE_REQUIRED = 0x00, - - /* Write true at size_offset */ - PB_HTYPE_OPTIONAL = 0x10, - - /* Read to pre-allocated array - * Maximum number of entries is array_size, - * actual number is stored at size_offset */ - PB_HTYPE_ARRAY = 0x20, - - /* Works for all required/optional/repeated fields. - * data_offset points to pb_callback_t structure. - * LTYPE should be 0 (it is ignored, but sometimes - * used to speculatively index an array). */ - PB_HTYPE_CALLBACK = 0x30 -} pb_packed pb_type_t; - -#define PB_HTYPE(x) ((x) & 0xF0) -#define PB_LTYPE(x) ((x) & 0x0F) - -/* This structure is used in auto-generated constants - * to specify struct fields. - * You can change field sizes here if you need structures - * larger than 256 bytes or field tags larger than 256. - * The compiler should complain if your .proto has such - * structures ("initializer too large for type"). - */ -typedef struct _pb_field_t pb_field_t; -struct _pb_field_t { - uint8_t tag; - pb_type_t type; - uint8_t data_offset; /* Offset of field data, relative to previous field. */ - int8_t size_offset; /* Offset of array size or has-boolean, relative to data */ - uint8_t data_size; /* Data size in bytes for a single item */ - uint8_t array_size; /* Maximum number of entries in array */ - - /* Field definitions for submessage - * OR default value for all other non-array, non-callback types - * If null, then field will zeroed. */ - const void *ptr; -} pb_packed; - -/* This structure is used for 'bytes' arrays. - * It has the number of bytes in the beginning, and after that an array. - * Note that actual structs used will have a different length of bytes array. - */ -typedef struct { - size_t size; - uint8_t bytes[1]; -} pb_bytes_array_t; +#include /* This structure is used for giving the callback function. * It is stored in the message structure and filled in by the method that @@ -151,12 +46,4 @@ typedef enum { PB_WT_32BIT = 5 } pb_wire_type_t; -/* These macros are used to declare pb_field_t's in the constant array. */ -#define pb_membersize(st, m) (sizeof ((st*)0)->m) -#define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) -#define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) -#define pb_delta_end(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) -#define PB_LAST_FIELD {0,0,0,0} - - #endif diff --git a/pb_decode.c b/pb_decode.c index 1e2fea0..ef982ac 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -183,82 +183,26 @@ static bool checkreturn make_string_substream(pb_istream_t *stream, pb_istream_t return true; } -/* Iterator for pb_field_t list */ -typedef struct { - const pb_field_t *start; - const pb_field_t *current; - int field_index; - void *dest_struct; - void *pData; - void *pSize; -} pb_field_iterator_t; - -static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct) -{ - iter->start = iter->current = fields; - iter->field_index = 0; - iter->pData = (char*)dest_struct + iter->current->data_offset; - iter->pSize = (char*)iter->pData + iter->current->size_offset; - iter->dest_struct = dest_struct; -} - -static bool pb_field_next(pb_field_iterator_t *iter) -{ - bool notwrapped = true; - size_t prev_size = iter->current->data_size; - - if (PB_HTYPE(iter->current->type) == PB_HTYPE_ARRAY) - prev_size *= iter->current->array_size; - - iter->current++; - iter->field_index++; - if (iter->current->tag == 0) - { - iter->current = iter->start; - iter->field_index = 0; - iter->pData = iter->dest_struct; - prev_size = 0; - notwrapped = false; - } - - iter->pData = (char*)iter->pData + prev_size + iter->current->data_offset; - iter->pSize = (char*)iter->pData + iter->current->size_offset; - return notwrapped; -} - -static bool checkreturn pb_field_find(pb_field_iterator_t *iter, int tag) -{ - int start = iter->field_index; - - do { - if (iter->current->tag == tag) - return true; - pb_field_next(iter); - } while (iter->field_index != start); - - return false; -} - /************************* * Decode a single field * *************************/ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) { - pb_decoder_t func = PB_DECODERS[PB_LTYPE(iter->current->type)]; + pb_decoder_t func = PB_DECODERS[PB_LTYPE(iter->current.type)]; - switch (PB_HTYPE(iter->current->type)) + switch (PB_HTYPE(iter->current.type)) { case PB_HTYPE_REQUIRED: - return func(stream, iter->current, iter->pData); + return func(stream, &iter->current, iter->pData); case PB_HTYPE_OPTIONAL: *(bool*)iter->pSize = true; - return func(stream, iter->current, iter->pData); + return func(stream, &iter->current, iter->pData); case PB_HTYPE_ARRAY: if (wire_type == PB_WT_STRING - && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE) + && PB_LTYPE(iter->current.type) <= PB_LTYPE_LAST_PACKABLE) { /* Packed array */ size_t *size = (size_t*)iter->pSize; @@ -266,10 +210,10 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t if (!make_string_substream(stream, &substream)) return false; - while (substream.bytes_left && *size < iter->current->array_size) + while (substream.bytes_left && *size < iter->current.array_size) { - void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size); - if (!func(&substream, iter->current, pItem)) + void *pItem = (uint8_t*)iter->pData + iter->current.data_size * (*size); + if (!func(&substream, &iter->current, pItem)) return false; (*size)++; } @@ -279,12 +223,12 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t { /* Repeated field */ size_t *size = (size_t*)iter->pSize; - void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size); - if (*size >= iter->current->array_size) + void *pItem = (uint8_t*)iter->pData + iter->current.data_size * (*size); + if (*size >= iter->current.array_size) return false; (*size)++; - return func(stream, iter->current, pItem); + return func(stream, &iter->current, pItem); } case PB_HTYPE_CALLBACK: @@ -303,7 +247,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t while (substream.bytes_left) { - if (!pCallback->funcs.decode(&substream, iter->current, pCallback->arg)) + if (!pCallback->funcs.decode(&substream, &iter->current, pCallback->arg)) return false; } @@ -324,7 +268,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t return false; substream = pb_istream_from_buffer(buffer, size); - return pCallback->funcs.decode(&substream, iter->current, pCallback->arg); + return pCallback->funcs.decode(&substream, &iter->current, pCallback->arg); } } @@ -334,44 +278,44 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t } /* Initialize message fields to default values, recursively */ -static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct) +static void pb_message_set_to_defaults(const pb_field_info_t *fields, void *dest_struct) { pb_field_iterator_t iter; + + if (fields->field_keys == NULL) return; + pb_field_init(&iter, fields, dest_struct); /* Initialize size/has fields and apply default values */ do { - if (iter.current->tag == 0) - continue; - /* Initialize the size field for optional/repeated fields to 0. */ - if (PB_HTYPE(iter.current->type) == PB_HTYPE_OPTIONAL) + if (PB_HTYPE(iter.current.type) == PB_HTYPE_OPTIONAL) { *(bool*)iter.pSize = false; } - else if (PB_HTYPE(iter.current->type) == PB_HTYPE_ARRAY) + else if (PB_HTYPE(iter.current.type) == PB_HTYPE_ARRAY) { *(size_t*)iter.pSize = 0; continue; /* Array is empty, no need to initialize contents */ } /* Initialize field contents to default value */ - if (PB_HTYPE(iter.current->type) == PB_HTYPE_CALLBACK) + if (PB_HTYPE(iter.current.type) == PB_HTYPE_CALLBACK) { continue; /* Don't overwrite callback */ } - else if (PB_LTYPE(iter.current->type) == PB_LTYPE_SUBMESSAGE) + else if (PB_LTYPE(iter.current.type) == PB_LTYPE_SUBMESSAGE) { - pb_message_set_to_defaults(iter.current->ptr, iter.pData); + pb_message_set_to_defaults(iter.current.ptr, iter.pData); } - else if (iter.current->ptr != NULL) + else if (iter.current.ptr != NULL) { - memcpy(iter.pData, iter.current->ptr, iter.current->data_size); + memcpy(iter.pData, iter.current.ptr, iter.current.data_size); } else { - memset(iter.pData, 0, iter.current->data_size); + memset(iter.pData, 0, iter.current.data_size); } } while (pb_field_next(&iter)); } @@ -380,10 +324,11 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str * Decode all fields * *********************/ -bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_info_t *fields, void *dest_struct) { uint32_t fields_seen = 0; /* Used to check for required fields */ pb_field_iterator_t iter; + const pb_field_key_t *field_keys = fields->field_keys; int i; pb_message_set_to_defaults(fields, dest_struct); @@ -424,13 +369,14 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void } /* Check that all required fields (mod 31) were present. */ - for (i = 0; fields[i].tag != 0; i++) + for (i = 0; field_keys != NULL; i++) { - if (PB_HTYPE(fields[i].type) == PB_HTYPE_REQUIRED && + if (PB_HTYPE(PB_FIELD_INFO(&field_keys[i], fields->field_info)->type) == PB_HTYPE_REQUIRED && !(fields_seen & (1 << (i & 31)))) { return false; } + if (PB_IS_LAST(&field_keys[i])) break; } return true; @@ -511,7 +457,7 @@ bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, voi /* Check length, noting the space taken by the size_t header. */ if (x->size > field->data_size - offsetof(pb_bytes_array_t, bytes)) - return false; + return false; return pb_read(stream, x->bytes, x->size); } @@ -523,8 +469,7 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo if (!pb_decode_varint32(stream, &size)) return false; - /* Check length, noting the null terminator */ - if (size > field->data_size - 1) + if ((int) size > field->data_size - 1) return false; status = pb_read(stream, (uint8_t*)dest, size); @@ -543,7 +488,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field if (field->ptr == NULL) return false; - status = pb_decode(&substream, (pb_field_t*)field->ptr, dest); + status = pb_decode(&substream, (pb_field_info_t*)field->ptr, dest); stream->state = substream.state; return status; } diff --git a/pb_decode.h b/pb_decode.h index f12b190..34ae267 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -42,7 +42,7 @@ bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); * Returns true on success, false on any failure. * The actual struct pointed to by dest must match the description in fields. */ -bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); +bool pb_decode(pb_istream_t *stream, const pb_field_info_t *fields, void *dest_struct); /* --- Helper functions --- * You may want to use these from your caller or callbacks. @@ -68,4 +68,6 @@ bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest); +#define PB_ISTREAM_INIT {0,0,0} + #endif diff --git a/pb_encode.c b/pb_encode.c index 995eb3d..3e8fe65 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -75,7 +75,7 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, const void *pData, size_t count, pb_encoder_t func) { - int i; + size_t i; const void *p; size_t size; @@ -98,7 +98,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie } else { - pb_ostream_t sizestream = {0}; + pb_ostream_t sizestream = PB_OSTREAM_INIT; p = pData; for (i = 0; i < count; i++) { @@ -140,15 +140,19 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie return true; } -bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) +bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_info_t *fields, const void *src_struct) { - const pb_field_t *field = fields; + const pb_field_t *field; + const pb_field_key_t *field_key = fields->field_keys; const void *pData = src_struct; const void *pSize; size_t prev_size = 0; - while (field->tag != 0) + while(field_key != NULL) { + pb_field_t f = *(PB_FIELD_INFO(field_key, fields->field_info)); + f.tag = PB_TAG_VAL(field_key); + field = &f; pb_encoder_t func = PB_ENCODERS[PB_LTYPE(field->type)]; pData = (const char*)pData + prev_size + field->data_offset; pSize = (const char*)pData + field->size_offset; @@ -193,13 +197,20 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons break; } } - - field++; + if (PB_IS_LAST(field_key++)) break; } return true; } +int checkreturn pb_get_message_size(const pb_field_info_t *fields, const void *src_struct) +{ + pb_ostream_t ostream = PB_OSTREAM_INIT; + if (pb_encode(&ostream, fields, &src_struct)) + return ostream.bytes_written; + return -1; +} + /* Helper functions */ bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value) { @@ -345,14 +356,14 @@ bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, co bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - pb_ostream_t substream = {0}; + pb_ostream_t substream = {0, 0, 0, 0}; size_t size; bool status; if (field->ptr == NULL) return false; - if (!pb_encode(&substream, (pb_field_t*)field->ptr, src)) + if (!pb_encode(&substream, (pb_field_info_t*)field->ptr, src)) return false; size = substream.bytes_written; @@ -373,7 +384,7 @@ bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field substream.max_size = size; substream.bytes_written = 0; - status = pb_encode(&substream, (pb_field_t*)field->ptr, src); + status = pb_encode(&substream, (pb_field_info_t*)field->ptr, src); stream->bytes_written += substream.bytes_written; stream->state = substream.state; diff --git a/pb_encode.h b/pb_encode.h index cec3913..40f8719 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -42,7 +42,9 @@ bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); * The actual struct pointed to by src_struct must match the description in fields. * All required fields in the struct are assumed to have been filled in. */ -bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); +bool pb_encode(pb_ostream_t *stream, const pb_field_info_t *fields, const void *src_struct); + +int pb_get_message_size(const pb_field_info_t *fields, const void *src_struct); /* --- Helper functions --- * You may want to use these from your caller or callbacks. @@ -69,4 +71,5 @@ bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src); bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src); +#define PB_OSTREAM_INIT {0,0,0,0} #endif diff --git a/pb_field.c b/pb_field.c new file mode 100644 index 0000000..9d252e1 --- /dev/null +++ b/pb_field.c @@ -0,0 +1,98 @@ +#include "pb_field.h" + +#ifdef __GNUC__ +/* Verify that we remember to check all return values for proper error propagation */ +#define checkreturn __attribute__((warn_unused_result)) +#else +#define checkreturn +#endif + +/** + * pb_common_aligned_field_info array holds the common field info that can be shared + * by any numeric fields whose previous field is memory aligned + */ +const pb_field_t pb_common_aligned_field_info[NUM_COMMON_FIELD_INFO] = { + /* REQUIRED bool */ + {0, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, 0, 0, sizeof(bool), 0, 0}, + /* OPTIONAL bool */ + {0, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, 1, -1, sizeof(bool), 0, 0}, + /* REQUIRED int32, uint32, enum */ + {0, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, 0, 0, sizeof(uint32_t), 0, 0}, + /* OPTIONAL int32, uint32, enum */ + {0, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, 4, -4, sizeof(uint32_t), 0, 0}, + /* REQUIRED int64, uint64 */ + {0, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, 0, 0, sizeof(uint64_t), 0, 0}, + /* OPTIONAL int64, uint64 */ + {0, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, 4, -4, sizeof(uint64_t), 0, 0}, + /* REQUIRED sint32 */ + {0, PB_HTYPE_REQUIRED | PB_LTYPE_SVARINT, 0, 0, sizeof(int32_t), 0, 0}, + /* OPTIONAL sint32 */ + {0, PB_HTYPE_OPTIONAL | PB_LTYPE_SVARINT, 4, -4, sizeof(int32_t), 0, 0}, + /* REQUIRED sint64 */ + {0, PB_HTYPE_REQUIRED | PB_LTYPE_SVARINT, 0, 0, sizeof(int64_t), 0, 0}, + /* OPTIONAL sint64 */ + {0, PB_HTYPE_OPTIONAL | PB_LTYPE_SVARINT, 4, -4, sizeof(int64_t), 0, 0}, + /* REQUIRED fixed32, sfixed32, float */ + {0, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED32, 0, 0, sizeof(uint32_t), 0, 0}, + /* OPTIONAL fixed32, sfixed32, float */ + {0, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED32, 4, -4, sizeof(uint32_t), 0, 0}, + /* REQUIRED fixed64, sfixed64, double */ + {0, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED64, 0, 0, sizeof(uint64_t), 0, 0}, + /* OPTIONAL fixed64, sfixed64, double */ + {0, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED64, 4, -4, sizeof(uint64_t), 0, 0} +}; + +void pb_field_init(pb_field_iterator_t *iter, const pb_field_info_t *fields, void *dest_struct) +{ + if (fields->field_keys == NULL) return; + iter->start_key = iter->current_key = fields->field_keys; + iter->field_info = fields->field_info; + iter->current = *(PB_FIELD_INFO(iter->current_key, iter->field_info)); + iter->current.tag = PB_TAG_VAL(iter->current_key); + iter->field_index = 0; + iter->pData = (char*)dest_struct + iter->current.data_offset; + iter->pSize = (char*)iter->pData + iter->current.size_offset; + iter->dest_struct = dest_struct; +} + +bool pb_field_next(pb_field_iterator_t *iter) +{ + bool notwrapped = true; + size_t prev_size = iter->current.data_size; + + if (PB_HTYPE(iter->current.type) == PB_HTYPE_ARRAY) + prev_size *= iter->current.array_size; + + iter->field_index++; + if (PB_IS_LAST(iter->current_key)) + { + iter->current_key = iter->start_key; + iter->current = *(PB_FIELD_INFO(iter->current_key, iter->field_info)); + iter->current.tag = PB_TAG_VAL(iter->current_key); + iter->field_index = 0; + iter->pData = iter->dest_struct; + prev_size = 0; + notwrapped = false; + } else { + iter->current_key++; + iter->current = *(PB_FIELD_INFO(iter->current_key, iter->field_info)); + iter->current.tag = PB_TAG_VAL(iter->current_key); + } + + iter->pData = (char*)iter->pData + prev_size + iter->current.data_offset; + iter->pSize = (char*)iter->pData + iter->current.size_offset; + return notwrapped; +} + +bool checkreturn pb_field_find(pb_field_iterator_t *iter, int tag) +{ + int start = iter->field_index; + + do { + if (iter->current.tag == tag) + return true; + pb_field_next(iter); + } while (iter->field_index != start); + + return false; +} diff --git a/pb_field.h b/pb_field.h new file mode 100644 index 0000000..89f76e6 --- /dev/null +++ b/pb_field.h @@ -0,0 +1,211 @@ +#ifndef _PB_FIELD_H_ +#define _PB_FIELD_H_ + +#include +#include +#include + + +#ifdef __GNUC__ +/* This just reduces memory requirements, but is not required. */ +#define pb_packed __attribute__((packed)) +#else +#define pb_packed +#endif + +/* List of possible field types. These are used in the autogenerated code. + * Least-significant 4 bits tell the scalar type + * Most-significant 4 bits specify repeated/required/packed etc. + * + * INT32 and UINT32 are treated the same, as are (U)INT64 and (S)FIXED* + * These types are simply casted to correct field type when they are + * assigned to the memory pointer. + * SINT* is different, though, because it is zig-zag coded. + */ + +typedef enum { + /************************ + * Field contents types * + ************************/ + + /* Numeric types */ + PB_LTYPE_VARINT = 0x00, /* int32, uint32, int64, uint64, bool, enum */ + PB_LTYPE_SVARINT = 0x01, /* sint32, sint64 */ + PB_LTYPE_FIXED32 = 0x02, /* fixed32, sfixed32, float */ + PB_LTYPE_FIXED64 = 0x03, /* fixed64, sfixed64, double */ + + /* Marker for last packable field type. */ + PB_LTYPE_LAST_PACKABLE = 0x03, + + /* Byte array with pre-allocated buffer. + * data_size is the length of the allocated PB_BYTES_ARRAY structure. */ + PB_LTYPE_BYTES = 0x04, + + /* String with pre-allocated buffer. + * data_size is the maximum length. */ + PB_LTYPE_STRING = 0x05, + + /* Submessage + * submsg_fields is pointer to field descriptions */ + PB_LTYPE_SUBMESSAGE = 0x06, + + /* Number of declared LTYPES */ + PB_LTYPES_COUNT = 7, + + /****************** + * Modifier flags * + ******************/ + + /* Just the basic, write data at data_offset */ + PB_HTYPE_REQUIRED = 0x00, + + /* Write true at size_offset */ + PB_HTYPE_OPTIONAL = 0x10, + + /* Read to pre-allocated array + * Maximum number of entries is array_size, + * actual number is stored at size_offset */ + PB_HTYPE_ARRAY = 0x20, + + /* Works for all required/optional/repeated fields. + * data_offset points to pb_callback_t structure. + * LTYPE should be 0 (it is ignored, but sometimes + * used to speculatively index an array). */ + PB_HTYPE_CALLBACK = 0x30 + +} pb_packed pb_type_t; + +#define PB_HTYPE(x) ((x) & 0xF0) +#define PB_LTYPE(x) ((x) & 0x0F) + +/* This structure is used for 'bytes' arrays. + * It has the number of bytes in the beginning, and after that an array. + * Note that actual structs used will have a different length of bytes array. + */ +typedef struct { + size_t size; + uint8_t bytes[1]; +} pb_bytes_array_t; + +/* The next three pb_field_ structures are used in auto-generated constants + * to specify struct fields. + */ + +typedef struct _pb_field_key_t pb_field_key_t; +typedef struct _pb_field_t pb_field_t; +typedef struct _pb_field_info_t pb_field_info_t; +/* pb_field_key_t is a structure for storing the tag and an index into an array + * that stores the actual field info. This approach is used to reduce code size + * as many numeric fields have the same field info. + * + * A uint8_t tag will limit the field tag to 128 as the MSB bit is used to indicate + * whether or not it's the last field in the message. You may change it to uint16_t + * if you want to specify larger field tag. Please also change the corresponding + * tag in pb_field_t + */ +typedef uint8_t tag_t; +struct _pb_field_key_t { + tag_t tag; /* The MSB bit of the tag is used to indicate it's the last field in the message */ + /* The MSB bit of the info_index is used to indicate the field info is retrieved + * from the global pb_common_aligned_field_info array instead of the auto-generated field + * info. + */ + uint8_t info_index; +} pb_packed; + +/* pb_field_t is a structure for storing the pb field descriptor information + * You can change field sizes here if you need structures + * larger than 256 bytes + */ +struct _pb_field_t { + tag_t tag; + pb_type_t type; + uint8_t data_offset; /* Offset of field data, relative to previous field. */ + int8_t size_offset; /* Offset of array size or has-boolean, relative to data */ + uint8_t data_size; /* Data size in bytes for a single item */ + uint8_t array_size; /* Maximum number of entries in array */ + + /* Field definitions for submessage + * OR default value for all other non-array, non-callback types + * If null, then field will zeroed. */ + const void *ptr; +} pb_packed; + +/* pb_field_info_t is the structure for holding together the key and the field info arrays. + * This is the structure that will be passed to pb_encode and pb_decode + */ +struct _pb_field_info_t { + const pb_field_key_t *field_keys; + const pb_field_t *field_info; +}; + +/* Iterator for pb_field_t list */ +typedef struct { + const pb_field_key_t *start_key; + const pb_field_key_t *current_key; + pb_field_t current; + const pb_field_t *field_info; + int field_index; + void *dest_struct; + void *pData; + void *pSize; +} pb_field_iterator_t; + +/* Functions for iterating through the field */ +void pb_field_init(pb_field_iterator_t *iter, const pb_field_info_t *fields, void *dest_struct); +bool pb_field_next(pb_field_iterator_t *iter); +bool pb_field_find(pb_field_iterator_t *iter, int tag); + +/* These macros are used to declare pb_field_t's in the constant array. */ +#define pb_membersize(st, m) (sizeof ((st*)0)->m) +#define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) +#define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) +#define pb_delta_end(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) + +/* The MSB bit of the pb_field_key_t.tag indicates whether or not it's the last element */ +#define PB_LAST_FIELD (0x1 << (pb_membersize(pb_field_key_t, tag) * 8 - 1)) +#define PB_IS_LAST(key) ((key)->tag & PB_LAST_FIELD) +#define PB_TAG_VAL(key) ((key)->tag & ~PB_LAST_FIELD) + +/* The MSB bit of the pb_field_key_tag.info_index indicates whether it's a common or generated field info */ +#define PB_COMMON_INFO_FLAG (0x1 << (pb_membersize(pb_field_key_t, info_index) * 8 -1)) + +/* A common field info is retrieved from the globally declared pb_common_aligned_field_info array or + * locally auto-generated field info array */ +#define PB_FIELD_INFO(key, info) (((key)->info_index & PB_COMMON_INFO_FLAG) ? \ + &pb_common_aligned_field_info[(key)->info_index ^ PB_COMMON_INFO_FLAG] : \ + &((info)[(key)->info_index])) + +/* Defines the indices for pb_common_aligned_field_info array + * Note that MSB bit PB_COMMON_INFO_FLAG is set to indicate the index + * should be used on pb_common_aligned_field_info instead of generated + * array + * */ +enum { + REQUIRED_BOOL_INFO = 0 | PB_COMMON_INFO_FLAG, + OPTIONAL_BOOL_INFO = 1 | PB_COMMON_INFO_FLAG, + REQUIRED_INT32_INFO = 2 | PB_COMMON_INFO_FLAG, + OPTIONAL_INT32_INFO = 3 | PB_COMMON_INFO_FLAG, + REQUIRED_INT64_INFO = 4 | PB_COMMON_INFO_FLAG, + OPTIONAL_INT64_INFO = 5 | PB_COMMON_INFO_FLAG, + REQUIRED_SINT32_INFO = 6 | PB_COMMON_INFO_FLAG, + OPTIONAL_SINT32_INFO = 7 | PB_COMMON_INFO_FLAG, + REQUIRED_SINT64_INFO = 8 | PB_COMMON_INFO_FLAG, + OPTIONAL_SINT64_INFO = 9 | PB_COMMON_INFO_FLAG, + REQUIRED_FIXED32_INFO = 10 | PB_COMMON_INFO_FLAG, + OPTIONAL_FIXED32_INFO = 11 | PB_COMMON_INFO_FLAG, + REQUIRED_FIXED64_INFO = 12 | PB_COMMON_INFO_FLAG, + OPTIONAL_FIXED64_INFO = 13 | PB_COMMON_INFO_FLAG, + + /* leave this enum to determine num of elements */ + MAX_COMMON_FIELD_INFO +}; + +#define NUM_COMMON_FIELD_INFO (MAX_COMMON_FIELD_INFO ^ PB_COMMON_INFO_FLAG) + +/* pb_common_aligned_field_info array holds the common field info that can be shared + * by any numeric fields whose previous field is memory aligned + */ +extern const pb_field_t pb_common_aligned_field_info[NUM_COMMON_FIELD_INFO]; + +#endif /* _PB_FIELD_H_ */ diff --git a/tests/Makefile b/tests/Makefile index 8e2df14..1901859 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,12 +1,23 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage LDFLAGS=--coverage -DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h -TESTS=test_decode1 test_encode1 decode_unittests encode_unittests +DEPS=../pb_decode.h ../pb_encode.h ../pb.h ../pb_field.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h aligntype.pb.h +TESTS=test_decode1 test_encode1 test_encode2 test_decode2 test_encode3 test_decode3 test_encode4 test_decode4 decode_unittests encode_unittests test_encode_callbacks test_decode_callbacks + +CC_VER := $(shell gcc --version | grep gcc) +ifneq "$(CC_VER)" "" +CFLAGS += -m32 +LDFLAGS += -m32 +endif +ifndef PB_PATH +PBPATHOPT=-I/usr/include -I/usr/local/include +else +PBPATHOPT=-I$(PB_PATH) +endif all: breakpoints $(TESTS) run_unittests clean: - rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno + rm -f breakpoints $(TESTS) *.pb *.pb.c *.pb.h *.o *.gcda *.gcno %.o: %.c %.o: %.c $(DEPS) @@ -16,20 +27,24 @@ pb_encode.o: ../pb_encode.c $(DEPS) $(CC) $(CFLAGS) -c -o $@ $< pb_decode.o: ../pb_decode.c $(DEPS) $(CC) $(CFLAGS) -c -o $@ $< +pb_field.o: ../pb_field.c $(DEPS) + $(CC) $(CFLAGS) -c -o $@ $< -test_decode1: test_decode1.o pb_decode.o person.pb.o -test_decode2: test_decode2.o pb_decode.o person.pb.o -test_decode3: test_decode3.o pb_decode.o alltypes.pb.o -test_encode1: test_encode1.o pb_encode.o person.pb.o -test_encode2: test_encode2.o pb_encode.o person.pb.o -test_encode3: test_encode3.o pb_encode.o alltypes.pb.o -test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o -test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o -decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o -encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o +test_decode1: test_decode1.o pb_decode.o pb_field.o person.pb.o +test_decode2: test_decode2.o pb_decode.o pb_field.o person.pb.o +test_decode3: test_decode3.o pb_decode.o pb_field.o alltypes.pb.o +test_decode4: test_decode4.o pb_decode.o pb_field.o aligntype.pb.o +test_encode1: test_encode1.o pb_encode.o pb_field.o person.pb.o +test_encode2: test_encode2.o pb_encode.o pb_field.o person.pb.o +test_encode3: test_encode3.o pb_encode.o pb_field.o alltypes.pb.o +test_encode4: test_encode4.o pb_encode.o pb_field.o aligntype.pb.o +test_decode_callbacks: test_decode_callbacks.o pb_decode.o pb_field.o callbacks.pb.o +test_encode_callbacks: test_encode_callbacks.o pb_encode.o pb_field.o callbacks.pb.o +decode_unittests: decode_unittests.o pb_decode.o pb_field.o unittestproto.pb.o +encode_unittests: encode_unittests.o pb_encode.o pb_field.o unittestproto.pb.o %.pb: %.proto - protoc -I. -I../generator -I/usr/include -o$@ $< + protoc -I. -I../generator $(PBPATHOPT) -o$@ $< %.pb.c %.pb.h: %.pb ../generator/nanopb_generator.py python ../generator/nanopb_generator.py $< @@ -41,26 +56,29 @@ coverage: run_unittests gcov pb_encode.gcda gcov pb_decode.gcda -run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks +run_unittests: $(TESTS) rm -f *.gcda ./decode_unittests > /dev/null ./encode_unittests > /dev/null [ "`./test_encode1 | ./test_decode1`" = \ - "`./test_encode1 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] + "`./test_encode1 | protoc --decode=Person -I. -I../generator $(PBPATHOPT) person.proto`" ] [ "`./test_encode2 | ./test_decode1`" = \ - "`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] + "`./test_encode2 | protoc --decode=Person -I. -I../generator $(PBPATHOPT) person.proto`" ] [ "`./test_encode2 | ./test_decode2`" = \ - "`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] + "`./test_encode2 | protoc --decode=Person -I. -I../generator $(PBPATHOPT) person.proto`" ] [ "`./test_encode_callbacks | ./test_decode_callbacks`" = \ "`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ] ./test_encode3 | ./test_decode3 - ./test_encode3 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null + ./test_encode3 | protoc --decode=AllTypes -I. -I../generator $(PBPATHOPT) alltypes.proto >/dev/null + + ./test_encode4 | ./test_decode4 + ./test_encode4 | protoc --decode=AlignTypes -I. -I../generator $(PBPATHOPT) aligntype.proto >/dev/null run_fuzztest: test_decode2 bash -c 'I=1; while true; do cat /dev/urandom | ./test_decode2 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done' diff --git a/tests/aligntype.proto b/tests/aligntype.proto new file mode 100644 index 0000000..06d9531 --- /dev/null +++ b/tests/aligntype.proto @@ -0,0 +1,80 @@ +import "nanopb.proto"; + +message SubMessage { + required string substuff1 = 1 [(nanopb).max_size = 16]; + required int32 substuff2 = 2; +} + +enum MyEnum { + First = 1; + Second = 2; + Truth = 42; +} + +message AlignTypes { + required bool req_bool = 1; + required int32 req_int32 = 2; + required SubMessage req_submsg = 3; + required int64 req_int64 = 4; + required string req_string = 5 [(nanopb).max_size = 16]; + required MyEnum req_enum = 6; + required bytes req_bytes = 7 [(nanopb).max_size = 16]; + required sint32 req_sint32 = 8; + required bool req_bool_2 = 9; + required sint64 req_sint64 = 10; + required bool req_bool_3 = 11; + required fixed32 req_fixed32 = 12; + required bool req_bool_4 = 13; + required fixed64 req_fixed64 = 14; + required bool req_bool_5 = 15; + required float req_float = 16; + required bool req_bool_6 = 17; + required double req_double = 18; + + optional bool opt_bool = 19; + optional int32 opt_int32 = 20; + optional SubMessage opt_submsg = 21; + optional int64 opt_int64 = 22; + optional string opt_string = 23 [(nanopb).max_size = 16]; + optional MyEnum opt_enum = 24; + optional bytes opt_bytes = 25 [(nanopb).max_size = 16]; + optional sint32 opt_sint32 = 26; + optional bool opt_bool_2 = 27; + optional sint64 opt_sint64 = 28; + optional bool opt_bool_3 = 29; + optional fixed32 opt_fixed32 = 30; + optional bool opt_bool_4 = 31; + optional fixed64 opt_fixed64 = 32; + optional bool opt_bool_5 = 33; + optional float opt_float = 34; + optional bool opt_bool_6 = 35; + optional double opt_double = 36; + + required bool req_bool_aligned = 37; + required int32 req_int32_aligned = 38; + required int64 req_int64_aligned = 39; + required MyEnum req_enum_aligned = 40; + required sint32 req_sint32_aligned = 41; + required sint64 req_sint64_aligned = 42; + required fixed32 req_fixed32_aligned = 43; + required fixed64 req_fixed64_aligned = 44; + required float req_float_aligned = 45; + required double req_double_aligned = 46; + + optional bool opt_bool_aligned = 47; + optional int32 opt_int32_aligned = 48; + optional int64 opt_int64_aligned = 49; + optional MyEnum opt_enum_aligned = 50; + optional sint32 opt_sint32_aligned = 51; + optional sint64 opt_sint64_aligned = 52; + optional fixed32 opt_fixed32_aligned = 53; + optional fixed64 opt_fixed64_aligned = 54; + optional float opt_float_aligned = 55; + optional double opt_double_aligned = 56; + + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + required int32 end = 99; +} + diff --git a/tests/test_decode4.c b/tests/test_decode4.c new file mode 100644 index 0000000..b602596 --- /dev/null +++ b/tests/test_decode4.c @@ -0,0 +1,139 @@ +/* Tests the decoding of all types. Currently only in the 'required' variety. + * This is the counterpart of test_encode3. + * Run e.g. ./test_encode3 | ./test_decode3 + */ + +#include +#include +#include +#include "aligntype.pb.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return false; \ + } + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_aligntypes(pb_istream_t *stream) +{ + AlignTypes aligntypes = {}; + + if (!pb_decode(stream, AlignTypes_fields, &aligntypes)) + return false; + + TEST(aligntypes.req_bool == true); + TEST(aligntypes.req_int32 == 1001); + TEST(strcmp(aligntypes.req_submsg.substuff1, "1002") == 0); + TEST(aligntypes.req_submsg.substuff2 == 1002); + TEST(aligntypes.req_int64 == 1003); + TEST(strcmp(aligntypes.req_string, "1004") == 0); + TEST(aligntypes.req_enum == MyEnum_Truth); + TEST(aligntypes.req_bytes.size == 4); + TEST(memcmp(aligntypes.req_bytes.bytes, "1005", 4) == 0); + TEST(aligntypes.req_sint32 == 1006); + TEST(aligntypes.req_bool_2 == true); + TEST(aligntypes.req_sint64 == 1007); + TEST(aligntypes.req_bool_3 == true); + TEST(aligntypes.req_fixed32 == 1008); + TEST(aligntypes.req_bool_4 == true); + TEST(aligntypes.req_fixed64 == 1009); + TEST(aligntypes.req_bool_5 == true); + TEST(aligntypes.req_float == 1010.0f); + TEST(aligntypes.req_bool_5 == true); + TEST(aligntypes.req_double == 1011.0f); + + TEST(aligntypes.has_opt_bool == true); + TEST(aligntypes.opt_bool == true); + TEST(aligntypes.has_opt_int32 == true); + TEST(aligntypes.opt_int32 == 1001); + TEST(aligntypes.has_opt_submsg == true); + TEST(strcmp(aligntypes.opt_submsg.substuff1, "1002") == 0); + TEST(aligntypes.opt_submsg.substuff2 == 1002); + TEST(aligntypes.has_opt_int64 == true); + TEST(aligntypes.opt_int64 == 1003); + TEST(aligntypes.has_opt_string == true); + TEST(strcmp(aligntypes.opt_string, "1004") == 0); + TEST(aligntypes.has_opt_enum == true); + TEST(aligntypes.opt_enum == MyEnum_Truth); + TEST(aligntypes.has_opt_bytes == true); + TEST(aligntypes.opt_bytes.size == 4); + TEST(memcmp(aligntypes.opt_bytes.bytes, "1005", 4) == 0); + TEST(aligntypes.has_opt_sint32 == true); + TEST(aligntypes.opt_sint32 == 1006); + TEST(aligntypes.has_opt_bool_2 == true); + TEST(aligntypes.opt_bool_2 == true); + TEST(aligntypes.has_opt_sint64 == true); + TEST(aligntypes.opt_sint64 == 1007); + TEST(aligntypes.has_opt_bool_3 == true); + TEST(aligntypes.opt_bool_3 == true); + TEST(aligntypes.has_opt_fixed32 == true); + TEST(aligntypes.opt_fixed32 == 1008); + TEST(aligntypes.has_opt_bool_4 == true); + TEST(aligntypes.opt_bool_4 == true); + TEST(aligntypes.has_opt_fixed64 == true); + TEST(aligntypes.opt_fixed64 == 1009); + TEST(aligntypes.has_opt_bool_5 == true); + TEST(aligntypes.opt_bool_5 == true); + TEST(aligntypes.has_opt_float == true); + TEST(aligntypes.opt_float == 1010.0f); + TEST(aligntypes.has_opt_bool_6 == true); + TEST(aligntypes.opt_bool_6 == true); + TEST(aligntypes.has_opt_double == true); + TEST(aligntypes.opt_double == 1011.0f); + + TEST(aligntypes.req_bool_aligned == true); + TEST(aligntypes.req_int32_aligned == 1001); + TEST(aligntypes.req_int64_aligned == 1003); + TEST(aligntypes.req_enum_aligned == MyEnum_Truth); + TEST(aligntypes.req_sint32_aligned == 1006); + TEST(aligntypes.req_sint64_aligned == 1007); + TEST(aligntypes.req_fixed32_aligned == 1008); + TEST(aligntypes.req_fixed64_aligned == 1009); + TEST(aligntypes.req_float_aligned == 1010.0f); + TEST(aligntypes.req_double_aligned == 1011.0f); + + TEST(aligntypes.has_opt_bool_aligned == true); + TEST(aligntypes.opt_bool_aligned == true); + TEST(aligntypes.has_opt_int32_aligned == true); + TEST(aligntypes.opt_int32_aligned == 1001); + TEST(aligntypes.has_opt_int64_aligned == true); + TEST(aligntypes.opt_int64_aligned == 1003); + TEST(aligntypes.has_opt_enum_aligned == true); + TEST(aligntypes.opt_enum_aligned == MyEnum_Truth); + TEST(aligntypes.has_opt_sint32_aligned == true); + TEST(aligntypes.opt_sint32_aligned == 1006); + TEST(aligntypes.has_opt_sint64_aligned == true); + TEST(aligntypes.opt_sint64_aligned == 1007); + TEST(aligntypes.has_opt_fixed32_aligned == true); + TEST(aligntypes.opt_fixed32_aligned == 1008); + TEST(aligntypes.has_opt_fixed64_aligned == true); + TEST(aligntypes.opt_fixed64_aligned == 1009); + TEST(aligntypes.has_opt_float_aligned == true); + TEST(aligntypes.opt_float_aligned == 1010.0f); + TEST(aligntypes.has_opt_double_aligned == true); + TEST(aligntypes.opt_double_aligned == 1011.0f); + + TEST(aligntypes.end == 1099); + + return true; +} + +int main() +{ + /* Read the data into buffer */ + uint8_t buffer[512]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_aligntypes(&stream)) + { + printf("Parsing failed.\n"); + return 1; + } else { + return 0; + } +} diff --git a/tests/test_encode4.c b/tests/test_encode4.c new file mode 100644 index 0000000..ddc4abc --- /dev/null +++ b/tests/test_encode4.c @@ -0,0 +1,117 @@ +/* Attempts to test all the datatypes supported by ProtoBuf. + * Currently only tests the 'required' variety. + */ + +#include +#include +#include "aligntype.pb.h" + +int main() +{ + /* Initialize the structure with constants */ + AlignTypes aligntypes = { + true, + 1001, + {"1002", 1002}, + 1003, + "1004", + MyEnum_Truth, + {4, "1005"}, + 1006, + true, + 1007, + true, + 1008, + true, + 1009, + true, + 1010.0f, + true, + 1011.0f, + + true, + true, + true, + 1001, + true, + {"1002", 1002}, + true, + 1003, + true, + "1004", + true, + MyEnum_Truth, + true, + {4, "1005"}, + true, + 1006, + true, + true, + true, + 1007, + true, + true, + true, + 1008, + true, + true, + true, + 1009, + true, + true, + true, + 1010.0f, + true, + true, + true, + 1011.0f, + + true, + 1001, + 1003, + MyEnum_Truth, + 1006, + 1007, + 1008, + 1009, + 1010.0f, + 1011.0f, + + true, + true, + true, + 1001, + true, + 1003, + true, + MyEnum_Truth, + true, + 1006, + true, + 1007, + true, + 1008, + true, + 1009, + true, + 1010.0f, + true, + 1011.0f, + + 1099 + }; + + uint8_t buffer[512]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AlignTypes_fields, &aligntypes)) + { + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + return 1; /* Failure */ + } +} diff --git a/tests/testperson.pb b/tests/testperson.pb deleted file mode 100644 index d1eb8cf..0000000 --- a/tests/testperson.pb +++ /dev/null @@ -1,3 +0,0 @@ - - Test Person7foobar@foobar.com" - 555-12345678 \ No newline at end of file