Improve .proto options parsing.

Options can now be defined on command line, file, message or in field
scope.

Update issue 12
Status: Started

generator/nanopb.proto

@@ -20,6 +20,14 @@ message NanoPBOptions {
 // Extensions: 1010 (all types)                                                                                                                
 // --------------------------------   
 
+extend google.protobuf.FileOptions {
+    optional NanoPBOptions nanopb_fileopt = 1010;
+}
+
+extend google.protobuf.MessageOptions {
+    optional NanoPBOptions nanopb_msgopt = 1010;
+}
+
 extend google.protobuf.FieldOptions {
     optional NanoPBOptions nanopb = 1010;
 }

generator/nanopb_generator.py

@@ -79,7 +79,7 @@ def names_from_type_name(type_name):
     return Names(type_name[1:].split('.'))
 
 class Enum:
-    def __init__(self, names, desc):
+    def __init__(self, names, desc, enum_options):
         '''desc is EnumDescriptorProto'''
         self.names = names + desc.name
         self.values = [(self.names + x.name, x.number) for x in desc.value]
@@ -91,7 +91,7 @@ class Enum:
         return result
 
 class Field:
-    def __init__(self, struct_name, desc):
+    def __init__(self, struct_name, desc, field_options):
         '''desc is FieldDescriptorProto'''
         self.tag = desc.number
         self.struct_name = struct_name
@@ -101,13 +101,12 @@ class Field:
         self.max_count = None
         self.array_decl = ""
         
-        # Parse nanopb-specific field options
+        # Parse field options
-        if desc.options.HasExtension(nanopb_pb2.nanopb):
+        if field_options.HasField("max_size"):
-            ext = desc.options.Extensions[nanopb_pb2.nanopb]
+            self.max_size = field_options.max_size
-            if ext.HasField("max_size"):
+        
-                self.max_size = ext.max_size
+        if field_options.HasField("max_count"):
-            if ext.HasField("max_count"):
+            self.max_count = field_options.max_count
-                self.max_count = ext.max_count
         
         if desc.HasField('default_value'):
             self.default = desc.default_value
@@ -284,9 +283,9 @@ class Field:
 
 
 class Message:
-    def __init__(self, names, desc):
+    def __init__(self, names, desc, message_options):
         self.name = names
-        self.fields = [Field(self.name, f) for f in desc.field]
+        self.fields = [Field(self.name, f, get_nanopb_suboptions(f, message_options)) for f in desc.field]
         self.ordered_fields = self.fields[:]
         self.ordered_fields.sort()
 
@@ -356,7 +355,7 @@ def iterate_messages(desc, names = Names()):
         for x in iterate_messages(submsg, sub_names):
             yield x
 
-def parse_file(fdesc):
+def parse_file(fdesc, file_options):
     '''Takes a FileDescriptorProto and returns tuple (enum, messages).'''
     
     enums = []
@@ -368,12 +367,13 @@ def parse_file(fdesc):
         base_name = Names()
     
     for enum in fdesc.enum_type:
-        enums.append(Enum(base_name, enum))
+        enums.append(Enum(base_name, enum, file_options))
     
     for names, message in iterate_messages(fdesc, base_name):
-        messages.append(Message(names, message))
+        message_options = get_nanopb_suboptions(message, file_options)
+        messages.append(Message(names, message, message_options))
         for enum in message.enum_type:
-            enums.append(Enum(names, enum))
+            enums.append(Enum(names, enum, message_options))
     
     return enums, messages
 
@@ -513,6 +513,7 @@ def generate_source(headername, enums, messages):
 import sys
 import os.path    
 from optparse import OptionParser
+import google.protobuf.text_format as text_format
 
 optparser = OptionParser(
     usage = "Usage: nanopb_generator.py [options] file.pb ...",
@@ -522,6 +523,30 @@ optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", defa
     help="Exclude file from generated #include list.")
 optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False,
     help="Don't print anything except errors.")
+optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
+    help="Print more information.")
+optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[],
+    help="Set generator option (max_size, max_count etc.).")
+
+def get_nanopb_suboptions(subdesc, options):
+    '''Get copy of options, and merge information from subdesc.'''
+    new_options = nanopb_pb2.NanoPBOptions()
+    new_options.CopyFrom(options)
+    
+    if isinstance(subdesc.options, descriptor.FieldOptions):
+        ext_type = nanopb_pb2.nanopb
+    elif isinstance(subdesc.options, descriptor.FileOptions):
+        ext_type = nanopb_pb2.nanopb_fileopt
+    elif isinstance(subdesc.options, descriptor.MessageOptions):
+        ext_type = nanopb_pb2.nanopb_msgopt
+    else:
+        raise Exception("Unknown options type")
+    
+    if subdesc.options.HasExtension(ext_type):
+        ext = subdesc.options.Extensions[ext_type]
+        new_options.MergeFrom(ext)
+    
+    return new_options
 
 def process(filenames, options):
     '''Process the files given on the command line.'''
@@ -530,10 +555,24 @@ def process(filenames, options):
         optparser.print_help()
         return False
     
+    if options.quiet:
+        options.verbose = False
+    
+    toplevel_options = nanopb_pb2.NanoPBOptions()
+    for s in options.settings:
+        text_format.Merge(s, toplevel_options)
+    
     for filename in filenames:
         data = open(filename, 'rb').read()
         fdesc = descriptor.FileDescriptorSet.FromString(data)
-        enums, messages = parse_file(fdesc.file[0])
+        
+        file_options = get_nanopb_suboptions(fdesc.file[0], toplevel_options)
+        
+        if options.verbose:
+            print "Options for " + filename + ":"
+            print text_format.MessageToString(file_options)
+        
+        enums, messages = parse_file(fdesc.file[0], file_options)
         
         noext = os.path.splitext(filename)[0]
         headername = noext + '.pb.h'
@@ -545,7 +584,7 @@ def process(filenames, options):
         
         # List of .proto files that should not be included in the C header file
         # even if they are mentioned in the source .proto.
-        excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto']
+        excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude
         dependencies = [d for d in fdesc.file[0].dependency if d not in excludes]
         
         header = open(headername, 'w')

generator/nanopb_pb2.py

@@ -7,15 +7,33 @@ from google.protobuf import descriptor_pb2
 # @@protoc_insertion_point(imports)
 
 
+import google.protobuf.descriptor_pb2
+
 DESCRIPTOR = descriptor.FileDescriptor(
   name='nanopb.proto',
   package='',
-  serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"4\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions')
+  serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"4\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions')
 
 
+NANOPB_FILEOPT_FIELD_NUMBER = 1010
+nanopb_fileopt = descriptor.FieldDescriptor(
+  name='nanopb_fileopt', full_name='nanopb_fileopt', index=0,
+  number=1010, type=11, cpp_type=10, label=1,
+  has_default_value=False, default_value=None,
+  message_type=None, enum_type=None, containing_type=None,
+  is_extension=True, extension_scope=None,
+  options=None)
+NANOPB_MSGOPT_FIELD_NUMBER = 1010
+nanopb_msgopt = descriptor.FieldDescriptor(
+  name='nanopb_msgopt', full_name='nanopb_msgopt', index=1,
+  number=1010, type=11, cpp_type=10, label=1,
+  has_default_value=False, default_value=None,
+  message_type=None, enum_type=None, containing_type=None,
+  is_extension=True, extension_scope=None,
+  options=None)
 NANOPB_FIELD_NUMBER = 1010
 nanopb = descriptor.FieldDescriptor(
-  name='nanopb', full_name='nanopb', index=0,
+  name='nanopb', full_name='nanopb', index=2,
   number=1010, type=11, cpp_type=10, label=1,
   has_default_value=False, default_value=None,
   message_type=None, enum_type=None, containing_type=None,
@@ -57,8 +75,7 @@ _NANOPBOPTIONS = descriptor.Descriptor(
   serialized_end=102,
 )
 
-import google.protobuf.descriptor_pb2
+DESCRIPTOR.message_types_by_name['NanoPBOptions'] = _NANOPBOPTIONS
-
 
 class NanoPBOptions(message.Message):
   __metaclass__ = reflection.GeneratedProtocolMessageType
@@ -66,6 +83,10 @@ class NanoPBOptions(message.Message):
   
   # @@protoc_insertion_point(class_scope:NanoPBOptions)
 
+nanopb_fileopt.message_type = _NANOPBOPTIONS
+google.protobuf.descriptor_pb2.FileOptions.RegisterExtension(nanopb_fileopt)
+nanopb_msgopt.message_type = _NANOPBOPTIONS
+google.protobuf.descriptor_pb2.MessageOptions.RegisterExtension(nanopb_msgopt)
 nanopb.message_type = _NANOPBOPTIONS
 google.protobuf.descriptor_pb2.FieldOptions.RegisterExtension(nanopb)
 # @@protoc_insertion_point(module_scope)

tests/Makefile

@@ -55,7 +55,7 @@ coverage: run_unittests
 	gcov pb_encode.gcda
 	gcov pb_decode.gcda
 
-run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields
+run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields test_options
 	rm -f *.gcda
 	
 	./decode_unittests > /dev/null
@@ -82,5 +82,13 @@ run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 te
 	
 	./test_missing_fields
 
+test_options: options.pb.h options.expected
+	for p in $$(grep . options.expected); do \
+	    if ! grep -qF "$$p" $<; then \
+	        echo Expected: $$p; \
+	        exit 1; \
+	    fi \
+	done
+
 run_fuzztest: test_decode2
 	bash -c 'I=1; while true; do cat /dev/urandom | ./test_decode2 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done'

tests/options.expected

@@ -0,0 +1,3 @@
+char filesize[20];
+char msgsize[30];
+char fieldsize[40];

tests/options.proto

@@ -0,0 +1,28 @@
+/* Test nanopb option parsing.
+ * options.expected lists the patterns that are searched for in the output.
+ */
+
+import "nanopb.proto";
+
+// File level options
+option (nanopb_fileopt).max_size = 20;
+
+message Message1
+{
+    required string filesize = 1;
+}
+
+// Message level options
+message Message2
+{
+    option (nanopb_msgopt).max_size = 30;
+    required string msgsize = 1;
+}
+
+// Field level options
+message Message3
+{
+    required string fieldsize = 1 [(nanopb).max_size = 40];
+}
+
+