rik/nanopb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generating and encoding messages with dynamic allocaiton

Browse Source
This commit is contained in:
Martin Donath
2013-12-08 23:25:32 +01:00
committed by Petteri Aimonen
parent eff9e11150
commit 03d4a7c916
9 changed files with 373 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
generator/nanopb.proto
View File

@@ -12,6 +12,7 @@ option java_package = "fi.kapsi.koti.jpa.nanopb";
enum FieldType {
FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible.
FT_CALLBACK = 1; // Always generate a callback field.
FT_POINTER = 4; // Always generate a dynamically allocated field.
FT_STATIC = 2; // Generate a static field or raise an exception if not possible.
FT_IGNORE = 3; // Ignore the field completely.
}

40
generator/nanopb_generator.py
View File

@@ -200,6 +200,10 @@ class Field:
self.enc_size = 5 # protoc rejects enum values > 32 bits
elif desc.type == FieldD.TYPE_STRING:
self.pbtype = 'STRING'
if field_options.type == nanopb_pb2.FT_POINTER:
self.ctype = 'char'
self.enc_size = None
else:
if self.max_size is None:
can_be_static = False
else:
@@ -208,6 +212,10 @@ class Field:
self.enc_size = varint_max_size(self.max_size) + self.max_size
elif desc.type == FieldD.TYPE_BYTES:
self.pbtype = 'BYTES'
if field_options.type == nanopb_pb2.FT_POINTER:
self.ctype = self.struct_name + self.name + 't'
self.enc_size = None
else:
if self.max_size is None:
can_be_static = False
else:
@@ -231,6 +239,8 @@ class Field:
if field_options.type == nanopb_pb2.FT_STATIC:
self.allocation = 'STATIC'
elif field_options.type == nanopb_pb2.FT_POINTER:
self.allocation = 'POINTER'
elif field_options.type == nanopb_pb2.FT_CALLBACK:
self.allocation = 'CALLBACK'
self.ctype = 'pb_callback_t'
@@ -242,20 +252,36 @@ class Field:
return cmp(self.tag, other.tag)
def __str__(self):
if self.rules == 'OPTIONAL' and self.allocation == 'STATIC':
result = ' bool has_' + self.name + ';\n'
elif self.rules == 'REPEATED' and self.allocation == 'STATIC':
result = ' size_t ' + self.name + '_count;\n'
else:
result = ''
if self.allocation == 'POINTER':
if self.rules == 'REPEATED':
result += ' size_t ' + self.name + '_count;\n'
# Use struct definition, so recursive submessages are possible
if self.pbtype == 'MESSAGE':
result += ' struct _%s *%s;' % (self.ctype, self.name)
# String arrays need to be defined as pointers to pointers
elif self.rules == 'REPEATED' and self.pbtype == 'STRING':
result += ' %s **%s;' % (self.ctype, self.name)
else:
result += ' %s *%s;' % (self.ctype, self.name)
else:
if self.rules == 'OPTIONAL' and self.allocation == 'STATIC':
result += ' bool has_' + self.name + ';\n'
elif self.rules == 'REPEATED' and self.allocation == 'STATIC':
result += ' size_t ' + self.name + '_count;\n'
result += ' %s %s%s;' % (self.ctype, self.name, self.array_decl)
return result
def types(self):
'''Return definitions for any special types this field might need.'''
if self.pbtype == 'BYTES' and self.allocation == 'STATIC':
if self.pbtype == 'BYTES' and (self.allocation == 'STATIC' or self.allocation == 'POINTER'):
result = 'typedef struct {\n'
result += ' size_t size;\n'
if self.allocation == 'POINTER':
result += ' uint8_t *bytes;\n'
else:
result += ' uint8_t bytes[%d];\n' % self.max_size
result += '} %s;\n' % self.ctype
else:
@@ -303,7 +329,7 @@ class Field:
result = ' PB_FIELD2(%3d, ' % self.tag
result += '%-8s, ' % self.pbtype
result += '%s, ' % self.rules
result += '%s, ' % self.allocation
result += '%-8s, ' % self.allocation
result += '%s, ' % ("FIRST" if not prev_field_name else "OTHER")
result += '%s, ' % self.struct_name
result += '%s, ' % self.name

13
generator/nanopb_pb2.py
View File

@@ -12,7 +12,7 @@ import google.protobuf.descriptor_pb2
DESCRIPTOR = descriptor.FileDescriptor(
name='nanopb.proto',
package='',
serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*J\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions')
serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*Z\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\x0e\n\nFT_POINTER\x10\x04\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptionsB\x1a\n\x18\x66i.kapsi.koti.jpa.nanopb')
_FIELDTYPE = descriptor.EnumDescriptor(
name='FieldType',
@@ -29,23 +29,28 @@ _FIELDTYPE = descriptor.EnumDescriptor(
options=None,
type=None),
descriptor.EnumValueDescriptor(
name='FT_STATIC', index=2, number=2,
name='FT_POINTER', index=2, number=4,
options=None,
type=None),
descriptor.EnumValueDescriptor(
name='FT_IGNORE', index=3, number=3,
name='FT_STATIC', index=3, number=2,
options=None,
type=None),
descriptor.EnumValueDescriptor(
name='FT_IGNORE', index=4, number=3,
options=None,
type=None),
],
containing_type=None,
options=None,
serialized_start=199,
serialized_end=273,
serialized_end=289,
)
FT_DEFAULT = 0
FT_CALLBACK = 1
FT_POINTER = 4
FT_STATIC = 2
FT_IGNORE = 3

17
pb.h
View File

@@ -166,6 +166,7 @@ typedef uint8_t pb_type_t;
/**** Field allocation types ****/
#define PB_ATYPE_STATIC 0x00
#define PB_ATYPE_POINTER 0x80
#define PB_ATYPE_CALLBACK 0x40
#define PB_ATYPE_MASK 0xC0
@@ -370,6 +371,22 @@ struct _pb_extension_t {
pb_membersize(st, m[0]), \
pb_arraysize(st, m), ptr}
/* Allocated fields carry the size of the actual data, not the pointer */
#define PB_REQUIRED_POINTER(tag, st, m, fd, ltype, ptr) \
{tag, PB_ATYPE_POINTER | PB_HTYPE_REQUIRED | ltype, \
fd, 0, pb_membersize(st, m[0]), 0, ptr}
/* Optional fields don't need a has_ variable, as information would be redundant */
#define PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) \
{tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \
fd, 0, pb_membersize(st, m[0]), 0, ptr}
#define PB_REPEATED_POINTER(tag, st, m, fd, ltype, ptr) \
{tag, PB_ATYPE_POINTER | PB_HTYPE_REPEATED | ltype, \
fd, \
pb_delta(st, m ## _count, m), \
pb_membersize(st, m[0]), 0, ptr}
/* Callbacks are much like required fields except with special datatype. */
#define PB_REQUIRED_CALLBACK(tag, st, m, fd, ltype, ptr) \
{tag, PB_ATYPE_CALLBACK | PB_HTYPE_REQUIRED | ltype, \

43
pb_encode.c
View File

@@ -117,7 +117,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
if (count == 0)
return true;
if (count > field->array_size)
if (PB_ATYPE(field->type) != PB_ATYPE_POINTER && count > field->array_size)
PB_RETURN_ERROR(stream, "array max size exceeded");
/* We always pack arrays if the datatype allows it. */
@@ -170,8 +170,19 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie
{
if (!pb_encode_tag_for_field(stream, field))
return false;
/* Special case for strings */
if (PB_ATYPE(field->type) == PB_ATYPE_POINTER &&
PB_LTYPE(field->type) == PB_LTYPE_STRING)
{
if (!func(stream, field, *(const void**)p))
return false;
}
else
{
if (!func(stream, field, p))
return false;
}
p = (const char*)p + field->data_size;
}
}
@@ -192,12 +203,19 @@ static bool checkreturn encode_static_field(pb_ostream_t *stream,
if (field->size_offset)
pSize = (const char*)pData + field->size_offset;
else if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
pSize = *(const void**)pData ? &dummy : pData;
else
pSize = &dummy;
if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
pData = *(const void**)pData;
switch (PB_HTYPE(field->type))
{
case PB_HTYPE_REQUIRED:
if (!pData)
return false;
if (!pb_encode_tag_for_field(stream, field))
return false;
if (!func(stream, field, pData))
@@ -255,6 +273,7 @@ static bool checkreturn encode_field(pb_ostream_t *stream,
switch (PB_ATYPE(field->type))
{
case PB_ATYPE_STATIC:
case PB_ATYPE_POINTER:
return encode_static_field(stream, field, pData);
case PB_ATYPE_CALLBACK:
@@ -312,6 +331,9 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons
while (field->tag != 0)
{
pData = (const char*)pData + prev_size + field->data_offset;
if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
prev_size = sizeof(const void*);
else
prev_size = field->data_size;
/* Special case for static arrays */
@@ -550,10 +572,17 @@ bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, con
{
const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src;
if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
{
return pb_encode_string(stream, *(const uint8_t**)bytes->bytes, bytes->size);
}
else
{
if (bytes->size + offsetof(pb_bytes_array_t, bytes) > field->data_size)
PB_RETURN_ERROR(stream, "bytes size exceeded");
return pb_encode_string(stream, bytes->bytes, bytes->size);
}
}
bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src)
@@ -561,12 +590,22 @@ bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, co
/* strnlen() is not always available, so just use a for-loop */
size_t size = 0;
const char *p = (const char*)src;
if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
{
while (*p != '\0')
{
size++;
p++;
}
}
else
{
while (size < field->data_size && *p != '\0')
{
size++;
p++;
}
}
return pb_encode_string(stream, (const uint8_t*)src, size);
}

12
tests/alltypes_pointer/SConscript Normal file
View File

@@ -0,0 +1,12 @@
# Build and run a test that encodes and decodes a message that contains
# all of the Protocol Buffers data types.
Import("env")
env.NanopbProto(["alltypes", "alltypes.options"])
enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "$COMMON/pb_encode.o"])
# dec = env.Program(["decode_alltypes_pointer.c", "alltypes.pb.c", "$COMMON/pb_decode.o"])
env.RunTest(enc)
# env.RunTest([dec, "encode_alltypes.output"])

2
tests/alltypes_pointer/alltypes.options Normal file
View File

@@ -0,0 +1,2 @@
* type:FT_POINTER

93
tests/alltypes_pointer/alltypes.proto Normal file
View File

@@ -0,0 +1,93 @@
message SubMessage {
required string substuff1 = 1 [default = "1"];
required int32 substuff2 = 2 [default = 2];
optional fixed32 substuff3 = 3 [default = 3];
}
message EmptyMessage {
}
enum MyEnum {
Zero = 0;
First = 1;
Second = 2;
Truth = 42;
}
message AllTypes {
required int32 req_int32 = 1;
required int64 req_int64 = 2;
required uint32 req_uint32 = 3;
required uint64 req_uint64 = 4;
required sint32 req_sint32 = 5;
required sint64 req_sint64 = 6;
required bool req_bool = 7;
required fixed32 req_fixed32 = 8;
required sfixed32 req_sfixed32= 9;
required float req_float = 10;
required fixed64 req_fixed64 = 11;
required sfixed64 req_sfixed64= 12;
required double req_double = 13;
required string req_string = 14;
required bytes req_bytes = 15;
required SubMessage req_submsg = 16;
required MyEnum req_enum = 17;
required EmptyMessage req_emptymsg = 18;
repeated int32 rep_int32 = 21;
repeated int64 rep_int64 = 22;
repeated uint32 rep_uint32 = 23;
repeated uint64 rep_uint64 = 24;
repeated sint32 rep_sint32 = 25;
repeated sint64 rep_sint64 = 26;
repeated bool rep_bool = 27;
repeated fixed32 rep_fixed32 = 28;
repeated sfixed32 rep_sfixed32= 29;
repeated float rep_float = 30;
repeated fixed64 rep_fixed64 = 31;
repeated sfixed64 rep_sfixed64= 32;
repeated double rep_double = 33;
repeated string rep_string = 34;
repeated bytes rep_bytes = 35;
repeated SubMessage rep_submsg = 36;
repeated MyEnum rep_enum = 37;
repeated EmptyMessage rep_emptymsg = 38;
optional int32 opt_int32 = 41 [default = 4041];
optional int64 opt_int64 = 42 [default = 4042];
optional uint32 opt_uint32 = 43 [default = 4043];
optional uint64 opt_uint64 = 44 [default = 4044];
optional sint32 opt_sint32 = 45 [default = 4045];
optional sint64 opt_sint64 = 46 [default = 4046];
optional bool opt_bool = 47 [default = false];
optional fixed32 opt_fixed32 = 48 [default = 4048];
optional sfixed32 opt_sfixed32= 49 [default = 4049];
optional float opt_float = 50 [default = 4050];
optional fixed64 opt_fixed64 = 51 [default = 4051];
optional sfixed64 opt_sfixed64= 52 [default = 4052];
optional double opt_double = 53 [default = 4053];
optional string opt_string = 54 [default = "4054"];
optional bytes opt_bytes = 55 [default = "4055"];
optional SubMessage opt_submsg = 56;
optional MyEnum opt_enum = 57 [default = Second];
optional EmptyMessage opt_emptymsg = 58;
// Just to make sure that the size of the fields has been calculated
// properly, i.e. otherwise a bug in last field might not be detected.
required int32 end = 99;
extensions 200 to 255;
}

142
tests/alltypes_pointer/encode_alltypes_pointer.c Normal file
View File

@@ -0,0 +1,142 @@
/* Attempts to test all the datatypes supported by ProtoBuf.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <pb_encode.h>
#include "alltypes.pb.h"
#include "test_helpers.h"
int main(int argc, char **argv)
{
int mode = (argc > 1) ? atoi(argv[1]) : 0;
/* Initialize values to encode */
int32_t value_int32 = -1000;
int64_t value_int64 = -10000000000;
uint32_t value_uint32 = 1000;
uint64_t value_uint64 = 10000000000;
bool value_bool = true;
float value_float = 1000.0f;
double value_double = 1000.0f;
char *value_string = "1000";
AllTypes_req_bytes_t value_req_bytes;
AllTypes_rep_bytes_t value_rep_bytes;
AllTypes_opt_bytes_t value_opt_bytes;
SubMessage value_submessage = {0};
MyEnum value_enum = MyEnum_Truth;
EmptyMessage value_empty_message = {0};
/* Initialize the structure with constants */
AllTypes alltypes = {0};
alltypes.req_int32 = &value_int32;
alltypes.req_int64 = &value_int64;
alltypes.req_uint32 = &value_uint32;
alltypes.req_uint64 = &value_uint64;
alltypes.req_sint32 = &value_int32;
alltypes.req_sint64 = &value_int64;
alltypes.req_bool = &value_bool;
alltypes.req_fixed32 = &value_uint32;
alltypes.req_sfixed32 = &value_int32;
alltypes.req_float = &value_float;
alltypes.req_fixed64 = &value_uint64;
alltypes.req_sfixed64 = &value_int64;
alltypes.req_double = &value_double;
value_req_bytes.bytes = (uint8_t*)"1000";
value_req_bytes.size = 4;
alltypes.req_string = value_string;
alltypes.req_bytes = &value_req_bytes;
value_submessage.substuff1 = value_string;
value_submessage.substuff2 = &value_int32;
alltypes.req_submsg = &value_submessage;
alltypes.req_enum = &value_enum;
alltypes.req_emptymsg = &value_empty_message;
alltypes.rep_int32_count = 1; alltypes.rep_int32 = &value_int32;
alltypes.rep_int64_count = 1; alltypes.rep_int64 = &value_int64;
alltypes.rep_uint32_count = 1; alltypes.rep_uint32 = &value_uint32;
alltypes.rep_uint64_count = 1; alltypes.rep_uint64 = &value_uint64;
alltypes.rep_sint32_count = 1; alltypes.rep_sint32 = &value_int32;
alltypes.rep_sint64_count = 1; alltypes.rep_sint64 = &value_int64;
alltypes.rep_bool_count = 1; alltypes.rep_bool = &value_bool;
alltypes.rep_fixed32_count = 1; alltypes.rep_fixed32 = &value_uint32;
alltypes.rep_sfixed32_count = 1; alltypes.rep_sfixed32 = &value_int32;
alltypes.rep_float_count = 1; alltypes.rep_float = &value_float;
alltypes.rep_fixed64_count = 1; alltypes.rep_fixed64 = &value_uint64;
alltypes.rep_sfixed64_count = 1; alltypes.rep_sfixed64 = &value_int64;
alltypes.rep_double_count = 1; alltypes.rep_double = &value_double;
value_rep_bytes.bytes = (uint8_t*)"1000";
value_rep_bytes.size = 4;
alltypes.rep_string_count = 1; alltypes.rep_string = (char **)&value_string;
alltypes.rep_bytes_count = 0; alltypes.rep_bytes = &value_rep_bytes;
alltypes.rep_submsg_count = 1; alltypes.rep_submsg = &value_submessage;
alltypes.rep_enum_count = 1; alltypes.rep_enum = &value_enum;
alltypes.rep_emptymsg_count = 1; alltypes.rep_emptymsg = &value_empty_message;
if (mode != 0)
{
/* Fill in values for optional fields */
alltypes.opt_int32 = &value_int32;
alltypes.opt_int64 = &value_int64;
alltypes.opt_uint32 = &value_uint32;
alltypes.opt_uint64 = &value_uint64;
alltypes.opt_sint32 = &value_int32;
alltypes.opt_sint64 = &value_int64;
alltypes.opt_bool = &value_bool;
alltypes.opt_fixed32 = &value_uint32;
alltypes.opt_sfixed32 = &value_int32;
alltypes.opt_float = &value_float;
alltypes.opt_fixed64 = &value_uint64;
alltypes.opt_sfixed64 = &value_int64;
alltypes.opt_double = &value_double;
value_opt_bytes.bytes = (uint8_t*)"1000";
value_opt_bytes.size = 4;
alltypes.opt_string = value_string;
alltypes.opt_bytes = &value_opt_bytes;
alltypes.opt_submsg = &value_submessage;
alltypes.opt_enum = &value_enum;
alltypes.opt_emptymsg = &value_empty_message;
}
alltypes.end = &value_int32;
{
uint8_t buffer[4096];
pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
/* Now encode it and check if we succeeded. */
if (pb_encode(&stream, AllTypes_fields, &alltypes))
{
/*SET_BINARY_MODE(stdout);
fwrite(buffer, 1, stream.bytes_written, stdout);*/ /* TODO: use this to validate decoding, when implemented */
return 0; /* Success */
}
else
{
fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
return 1; /* Failure */
}
}
}