First version of header generator

git-svn-id: https://svn.kapsi.fi/jpa/nanopb@950 e3a754e5-d11d-0410-8d38-ebb782a927b9

generator/nanopb.proto

@@ -0,0 +1,18 @@
+// Custom options for defining:
+// - Maximum size of string/bytes
+// - Maximum number of elements in array
+//
+// These are used by nanopb to generate statically allocable structures
+// for memory-limited environments.
+
+import "google/protobuf/descriptor.proto";
+
+message NanoPBOptions {
+  optional int32 max_size = 1;
+  optional int32 max_count = 2;
+}
+
+extend google.protobuf.FieldOptions {
+    optional NanoPBOptions nanopb = 52001;
+}
+

generator/nanopb_generator.py

@@ -0,0 +1,344 @@
+'''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.'''
+
+import google.protobuf.descriptor_pb2 as descriptor
+import nanopb_pb2
+import os.path
+
+# Values are tuple (c type, pb ltype)
+FieldD = descriptor.FieldDescriptorProto
+datatypes = {
+    FieldD.TYPE_BOOL: ('bool', 'PB_LTYPE_VARINT'),
+    FieldD.TYPE_DOUBLE: ('double', 'PB_LTYPE_FIXED'),
+    FieldD.TYPE_FIXED32: ('uint32_t', 'PB_LTYPE_FIXED'),
+    FieldD.TYPE_FIXED64: ('uint64_t', 'PB_LTYPE_FIXED'),
+    FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED'),
+    FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT'),
+    FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT'),
+    FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED'),
+    FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED'),
+    FieldD.TYPE_SINT32: ('int32_t', 'PB_LTYPE_SVARINT'),
+    FieldD.TYPE_SINT64: ('int64_t', 'PB_LTYPE_SVARINT'),
+    FieldD.TYPE_UINT32: ('uint32_t', 'PB_LTYPE_VARINT'),
+    FieldD.TYPE_UINT64: ('uint64_t', 'PB_LTYPE_VARINT')
+}
+
+class Names:
+    '''Keeps a set of nested names and formats them to C identifier.
+    You can subclass this with your own implementation.
+    '''
+    def __init__(self, parts = ()):
+        if isinstance(parts, Names):
+            parts = parts.parts
+        self.parts = tuple(parts)
+    
+    def __str__(self):
+        return '_'.join(self.parts)
+
+    def __add__(self, other):
+        if isinstance(other, (str, unicode)):
+            return Names(self.parts + (other,))
+        elif isinstance(other, tuple):
+            return Names(self.parts + other)
+        else:
+            raise ValueError("Name parts should be of type str")
+    
+def names_from_type_name(type_name):
+    '''Parse Names() from FieldDescriptorProto type_name'''
+    if type_name[0] != '.':
+        raise NotImplementedError("Lookup of non-absolute type names is not supported")
+    return Names(type_name[1:].split('.'))
+
+class Enum:
+    def __init__(self, names, desc):
+        '''desc is EnumDescriptorProto'''
+        self.names = names + desc.name
+        self.values = [(self.names + x.name, x.number) for x in desc.value]
+    
+    def __str__(self):
+        result = 'typedef enum {\n'
+        result += ',\n'.join(["    %s = %d" % x for x in self.values])
+        result += '\n} %s;' % self.names
+        return result
+
+class Field:
+    def __init__(self, struct_name, desc):
+        '''desc is FieldDescriptorProto'''
+        self.tag = desc.number
+        self.struct_name = struct_name
+        self.name = desc.name
+        self.default = None
+        self.max_size = None
+        self.max_count = None
+        self.array_decl = ""
+        
+        # Parse nanopb-specific field options
+        if desc.options.HasExtension(nanopb_pb2.nanopb):
+            ext = desc.options.Extensions[nanopb_pb2.nanopb]
+            if ext.HasField("max_size"):
+                self.max_size = ext.max_size
+            if ext.HasField("max_count"):
+                self.max_count = ext.max_count
+        
+        if desc.HasField('default_value'):
+            self.default = desc.default_value
+        
+        # Decide HTYPE
+        # HTYPE is the high-order nibble of nanopb field description,
+        # defining whether value is required/optional/repeated.
+        is_callback = False
+        if desc.label == FieldD.LABEL_REQUIRED:
+            self.htype = 'PB_HTYPE_REQUIRED'
+        elif desc.label == FieldD.LABEL_OPTIONAL:
+            self.htype = 'PB_HTYPE_OPTIONAL'
+        elif desc.label == FieldD.LABEL_REPEATED:
+            if self.max_count is None:
+                is_callback = True
+            else:
+                self.htype = 'PB_HTYPE_ARRAY'
+                self.array_decl = '[%d]' % self.max_count
+        else:
+            raise NotImplementedError(desc.label)
+        
+        # Decide LTYPE and CTYPE
+        # LTYPE is the low-order nibble of nanopb field description,
+        # defining how to decode an individual value.
+        # CTYPE is the name of the c type to use in the struct.
+        if datatypes.has_key(desc.type):
+            self.ctype, self.ltype = datatypes[desc.type]
+        elif desc.type == FieldD.TYPE_ENUM:
+            self.ltype = 'PB_LTYPE_VARINT'
+            self.ctype = names_from_type_name(desc.type_name)
+            self.default = Names(self.ctype) + self.default
+        elif desc.type == FieldD.TYPE_STRING:
+            self.ltype = 'PB_LTYPE_STRING'
+            if self.max_size is None:
+                is_callback = True
+            else:
+                self.ctype = 'char'
+                self.array_decl += '[%d]' % self.max_size
+        elif desc.type == FieldD.TYPE_BYTES:
+            self.ltype = 'PB_LTYPE_BYTES'
+            if self.max_size is None:
+                is_callback = True
+            else:
+                self.ctype = 'PB_BYTES_ARRAY(%d)' % self.max_size
+        elif desc.type == FieldD.TYPE_MESSAGE:
+            self.ltype = 'PB_LTYPE_SUBMESSAGE'
+            self.ctype = names_from_type_name(desc.type_name)
+        else:
+            raise NotImplementedError(desc.type)
+        
+        if is_callback:
+            self.htype = 'PB_HTYPE_CALLBACK'
+            self.ctype = 'pb_callback_t'
+            self.array_decl = ''
+    
+    def __cmp__(self, other):
+        return cmp(self.tag, other.tag)
+    
+    def __str__(self):
+        if self.htype == 'PB_HTYPE_OPTIONAL':
+            result = '    bool has_' + self.name + ';\n'
+        elif self.htype == 'PB_HTYPE_ARRAY':
+            result = '    size_t ' + self.name + '_count;\n'
+        else:
+            result = ''
+        result += '    %s %s%s;' % (self.ctype, self.name, self.array_decl)
+        return result
+    
+    def default_decl(self):
+        '''Return definition for this field's default value.'''
+        if self.default is None:
+            return None
+        
+        if self.ltype == 'PB_LTYPE_STRING':
+            ctype = 'char'
+            if self.max_size is None:
+                array_decl = '[]'
+            else:
+                array_decl = '[%d]' % self.max_size
+            default = self.default.encode('string_escape')
+            default = default.replace('"', '\\"')
+            default = '"' + default + '"'
+        elif self.ltype == 'PB_LTYPE_BYTES':
+            data = self.default.decode('string_escape')
+            data = ['0x%02x' % ord(c) for c in data]
+            
+            if self.max_size is None:
+                ctype = 'PB_BYTES_ARRAY(%d)' % len(data)
+            else:
+                ctype = 'PB_BYTES_ARRAY(%d)' % self.max_size
+            
+            default = '{%d, {%s}}' % (len(data), ','.join(data))
+            array_decl = ''
+        else:
+            ctype, default = self.ctype, self.default
+            array_decl = ''
+        
+        return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default)
+    
+    def pb_field_t(self, prev_field_name):
+        '''Return the pb_field_t initializer to use in the constant array.
+        prev_field_name is the name of the previous field or None.
+        '''
+        result = '    {%d, ' % self.tag
+        result += self.htype
+        if self.ltype is not None:
+            result += ' | ' + self.ltype
+        result += ',\n'
+        
+        if prev_field_name is None:
+            result += '    offsetof(%s, %s),' % (self.struct_name, self.name)
+        else:
+            result += '    pb_delta(%s, %s, %s),' % (self.struct_name, self.name, prev_field_name)
+        
+        if self.htype == 'PB_HTYPE_OPTIONAL':
+            result += '\n    pb_delta(%s, has_%s, %s),' % (self.struct_name, self.name, self.name)
+        elif self.htype == 'PB_HTYPE_ARRAY':
+            result += '\n    pb_delta(%s, %s_count, %s),' % (self.struct_name, self.name, self.name)
+        else:
+            result += ' 0,'
+        
+        
+        if self.htype == 'PB_HTYPE_ARRAY':
+            result += '\n    pb_membersize(%s, %s[0]),' % (self.struct_name, self.name)
+            result += ('\n    pb_membersize(%s, %s) / pb_membersize(%s, %s[0]),'
+                       % (self.struct_name, self.name, self.struct_name, self.name))
+        else:
+            result += '\n    pb_membersize(%s, %s),' % (self.struct_name, self.name)
+            result += ' 0,'
+        
+        if self.ltype == 'PB_LTYPE_SUBMESSAGE':
+            result += '\n    &%s_fields}' % self.ctype
+        elif self.default is None or self.htype == 'PB_HTYPE_CALLBACK':
+            result += ' 0}'
+        else:
+            result += '\n    &%s_default}' % (self.struct_name + self.name)
+        
+        return result
+
+class Message:
+    def __init__(self, names, desc):
+        self.name = names
+        self.fields = [Field(self.name, f) for f in desc.field]
+        self.ordered_fields = self.fields[:]
+        self.ordered_fields.sort()
+    
+    def __cmp__(self, other):
+        '''Sort messages so that submessages are declared before the message
+        that uses them.
+        '''
+        if self.refers_to(other.name):
+            return 1
+        elif other.refers_to(self.name):
+            return -1
+        else:
+            return 0
+    
+    def refers_to(self, name):
+        '''Returns True if this message uses the specified type as field type.'''
+        for field in self.fields:
+            if str(field.ctype) == str(name):
+                return True
+        return False
+    
+    def __str__(self):
+        result = 'typedef struct {\n'
+        result += '\n'.join([str(f) for f in self.fields])
+        result += '\n} %s;' % self.name
+        return result
+    
+    def default_decl(self):
+        result = ""
+        for field in self.fields:
+            default = field.default_decl()
+            if default is not None:
+                result += default + '\n'
+        return result
+
+    def pb_field_t(self):
+        result = 'const pb_field_t %s_fields[] = {\n' % self.name
+        
+        prev = None
+        for field in self.ordered_fields:
+            result += field.pb_field_t(prev)
+            result += ',\n\n'
+            prev = field.name
+        
+        result = result[:-3] + '\n};'
+        return result
+
+def iterate_messages(desc, names = Names()):
+    '''Recursively find all messages. For each, yield name, DescriptorProto.'''
+    if hasattr(desc, 'message_type'):
+        submsgs = desc.message_type
+    else:
+        submsgs = desc.nested_type
+    
+    for submsg in submsgs:
+        sub_names = names + submsg.name
+        yield sub_names, submsg
+        
+        for x in iterate_messages(submsg, sub_names):
+            yield x
+
+def process_file(fdesc):
+    '''Takes a FileDescriptorProto and generate content for its header file.
+    Generates strings, which should be concatenated and stored to file.
+    '''
+    
+    yield '/* Automatically generated nanopb header */\n'
+    yield '#include <pb.h>\n\n'
+    
+    enums = []
+    messages = []
+    
+    for enum in fdesc.enum_type:
+        enums.append(Enum(Names(), enum))
+    
+    for names, message in iterate_messages(fdesc):
+        for enum in message.enum_type:
+            enums.append(Enum(names, enum))
+        
+        messages.append(Message(names, message))
+    
+    yield '/* Enum definitions */\n'
+    for enum in enums:
+        yield str(enum) + '\n\n'
+    
+    yield '/* Struct definitions */\n'
+    messages.sort()
+    for msg in messages:
+        yield str(msg) + '\n\n'
+        
+    yield '/* Default values for struct fields */\n'
+    for msg in messages:
+        yield msg.default_decl()
+    yield '\n'
+    
+    yield '/* Struct field encoding specification for nanopb */\n'
+    for msg in messages:
+        yield msg.pb_field_t() + '\n\n'
+    
+if __name__ == '__main__':
+    import sys
+    import os.path
+    
+    if len(sys.argv) != 2:
+        print "Usage: " + sys.argv[0] + " file.pb"
+        print "where file.pb has been compiled from .proto by:"
+        print "protoc -ofile.pb file.proto"
+        print "Output fill be written to file.h"
+        sys.exit(1)
+    
+    data = open(sys.argv[1]).read()
+    fdesc = descriptor.FileDescriptorSet.FromString(data)
+    
+    destfile = os.path.splitext(sys.argv[1])[0] + '.h'
+    
+    print "Writing to " + destfile
+    
+    destfile = open(destfile, 'w')
+    
+    for part in process_file(fdesc.file[0]):
+        destfile.write(part)

generator/nanopb_pb2.py

@@ -0,0 +1,71 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+
+from google.protobuf import descriptor
+from google.protobuf import message
+from google.protobuf import reflection
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+
+DESCRIPTOR = descriptor.FileDescriptor(
+  name='nanopb.proto',
+  package='',
+  serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"4\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05:?\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xa1\x96\x03 \x01(\x0b\x32\x0e.NanoPBOptions')
+
+
+NANOPB_FIELD_NUMBER = 52001
+nanopb = descriptor.FieldDescriptor(
+  name='nanopb', full_name='nanopb', index=0,
+  number=52001, type=11, cpp_type=10, label=1,
+  has_default_value=False, default_value=None,
+  message_type=None, enum_type=None, containing_type=None,
+  is_extension=True, extension_scope=None,
+  options=None)
+
+
+_NANOPBOPTIONS = descriptor.Descriptor(
+  name='NanoPBOptions',
+  full_name='NanoPBOptions',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    descriptor.FieldDescriptor(
+      name='max_size', full_name='NanoPBOptions.max_size', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    descriptor.FieldDescriptor(
+      name='max_count', full_name='NanoPBOptions.max_count', index=1,
+      number=2, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  serialized_start=50,
+  serialized_end=102,
+)
+
+import google.protobuf.descriptor_pb2
+
+
+class NanoPBOptions(message.Message):
+  __metaclass__ = reflection.GeneratedProtocolMessageType
+  DESCRIPTOR = _NANOPBOPTIONS
+  
+  # @@protoc_insertion_point(class_scope:NanoPBOptions)
+
+nanopb.message_type = _NANOPBOPTIONS
+google.protobuf.descriptor_pb2.FieldOptions.RegisterExtension(nanopb)
+# @@protoc_insertion_point(module_scope)

pb.h

@@ -120,7 +120,7 @@ struct { \
     uint8_t bytes[buffersize]; \
 }
 
-typedef PB_BYTES_ARRAY(1) pb_bytes_array_t;
+typedef PB_BYTES_ARRAY() pb_bytes_array_t;
 
 /* This structure is used for giving the callback function.
  * It is stored in the message structure and filled in by the method that

tests/Makefile

@@ -1,5 +1,5 @@
 CFLAGS=-ansi -Wall -Werror -I .. -g -O0
-DEPS=../pb_decode.c ../pb_decode.h ../pb.h
+DEPS=../pb_decode.c ../pb_decode.h ../pb.h person.h
 TESTS=test_decode1 decode_unittests
 
 all: $(TESTS)

tests/decode_unittests.c

@@ -104,6 +104,31 @@ int main()
         TEST((s = S("\x04""testfoobar"), pb_skip_string(&s) && s.bytes_left == 7))
     }
     
+    {
+        pb_istream_t s = S("\x01\xFF\xFF\x03");
+        pb_field_t f = {1, PB_LTYPE_VARINT, 0, 0, 4, 0, 0};
+        uint32_t d;
+        COMMENT("Test pb_dec_varint using uint32_t")
+        TEST(pb_dec_varint(&s, &f, &d) && d == 1)
+        
+        /* Verify that no more than data_size is written. */
+        d = 0;
+        f.data_size = 1;
+        TEST(pb_dec_varint(&s, &f, &d) && d == 0xFF)
+    }
+    
+    {
+        pb_istream_t s;
+        pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 4, 0, 0};
+        int32_t d;
+        
+        COMMENT("Test pb_dec_svarint using int32_t")
+        TEST((s = S("\x01"), pb_dec_svarint(&s, &f, &d) && d == -1))
+        TEST((s = S("\x02"), pb_dec_svarint(&s, &f, &d) && d == 1))
+        TEST((s = S("\xfe\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d) && d == INT32_MAX))
+        TEST((s = S("\xff\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d) && d == INT32_MIN))
+    }
+    
     if (status != 0)
         fprintf(stdout, "\n\nSome tests FAILED!\n");
     

tests/person.proto

@@ -1,7 +1,10 @@
+import "nanopb.proto";
+
 message Person {
-  required string name = 1;
+  required string name = 1 [(nanopb).max_size = 40];
   required int32 id = 2;
-  optional string email = 3;
+  optional string email = 3 [(nanopb).max_size = 40];
+  optional bytes test = 5 [default = "abc\x00\x01\x02"];
 
   enum PhoneType {
     MOBILE = 0;
@@ -10,9 +13,9 @@ message Person {
   }
 
   message PhoneNumber {
-    required string number = 1;
+    required string number = 1 [(nanopb).max_size = 40];
     optional PhoneType type = 2 [default = HOME];
   }
 
-  repeated PhoneNumber phone = 4;
+  repeated PhoneNumber phone = 4 [(nanopb).max_count = 5];
 }

tests/test_decode1.c

@@ -1,75 +1,8 @@
 #include <stdio.h>
 #include <string.h>
 #include <stddef.h>
-#include "pb_decode.h"
+#include <pb_decode.h>
-
+#include "person.h"
-/* Structures for "Person" message */
-
-typedef enum {
-    Person_PhoneType_MOBILE = 0,
-    Person_PhoneType_HOME = 1,
-    Person_PhoneType_WORK = 2
-} Person_PhoneType;
-
-typedef struct {
-    char number[40];
-    bool has_type;
-    Person_PhoneType type;
-} Person_PhoneNumber;
-
-typedef struct {
-    char name[40];
-    int32_t id;
-    bool has_email;
-    char email[40];
-    size_t phone_size;
-    Person_PhoneNumber phone[5];
-} Person;
-
-/* Field descriptions */
-
-
-const Person_PhoneType Person_PhoneNumber_type_default = Person_PhoneType_HOME;
-
-const pb_field_t Person_PhoneNumber_fields[] = {
-    {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING,
-        offsetof(Person_PhoneNumber, number), 0,
-        pb_membersize(Person_PhoneNumber, number), 0, 0},
-        
-    {2, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT,
-        pb_delta(Person_PhoneNumber, type, number),
-        pb_delta(Person_PhoneNumber, has_type, type),
-        pb_membersize(Person_PhoneNumber, type), 0,
-        &Person_PhoneNumber_type_default},
-    
-    PB_LAST_FIELD
-};
-
-const pb_field_t Person_fields[] = {
-    {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING,
-        offsetof(Person, name), 0,
-        pb_membersize(Person, name), 0, 0},
-    
-    {2, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT,
-        pb_delta(Person, id, name), 0,
-        pb_membersize(Person, id), 0, 0},
-    
-    {3, PB_HTYPE_OPTIONAL | PB_LTYPE_STRING,
-        offsetof(Person, email) - offsetof(Person, id),
-        pb_delta(Person, has_email, email),
-        pb_membersize(Person, email), 0, 0},
-    
-    {4, PB_HTYPE_ARRAY | PB_LTYPE_SUBMESSAGE,
-        offsetof(Person, phone) - offsetof(Person, email),
-        pb_delta(Person, phone_size, phone),
-        pb_membersize(Person, phone[0]),
-        pb_arraysize(Person, phone),
-        Person_PhoneNumber_fields},
-    
-    PB_LAST_FIELD
-};
-
-/* And now, the actual test program */
 
 bool print_person(pb_istream_t *stream)
 {
@@ -81,7 +14,7 @@ bool print_person(pb_istream_t *stream)
     
     printf("Person: name '%s' id '%d' email '%s'\n", person.name, person.id, person.email);
     
-    for (i = 0; i < person.phone_size; i++)
+    for (i = 0; i < person.phone_count; i++)
     {
         Person_PhoneNumber *phone = &person.phone[i];
         printf("PhoneNumber: number '%s' type '%d'\n", phone->number, phone->type);