Merge pull request #710 'Automatically generate declarations'

5 files changed, 606 insertions, 1 deletions

diff --git a/scripts/finddeclarations.pl b/scripts/finddeclarations.pl
new file mode 100755
index 0000000000..1b1a57b9b7
--- /dev/null
+++ b/scripts/finddeclarations.pl
@@ -0,0 +1,50 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+if ($ARGV[0] eq '--help') {
+  print << "EOF";
+Usage:
+
+  $0 definitions.c
+EOF
+  exit;
+}
+
+my ($cfname, $sfname, $gfname, $cpp) = @ARGV;
+
+my $F;
+
+open $F, "<", $cfname;
+
+my $text = join "", <$F>;
+
+close $F;
+
+my $s = qr/(?>\s*)/aso;
+my $w = qr/(?>\w+)/aso;
+my $argname = qr/$w(?:\[(?>\w+)\])?/aso;
+my $type_regex = qr/(?:$w$s\**$s)+/aso;
+my $arg_regex = qr/(?:$type_regex$s$argname)/aso;
+
+while ($text =~ /
+    (?<=\n)         # Definition starts at the start of line
+    $type_regex     # Return type
+    $s$w            # Function name
+    $s\($s
+    (?:
+       $arg_regex(?:$s,$s$arg_regex)*+
+       ($s,$s\.\.\.)?                   # varargs function
+      |void
+    )?
+    $s\)
+    (?:$s FUNC_ATTR_$w(?:\((?>[^)]*)\))?)*+ # Optional attributes
+    (?=$s;)         # Ending semicolon
+  /axsogp) {
+  my $match = "${^MATCH}";
+  my $s = "${^PREMATCH}";
+  $s =~ s/[^\n]++//g;
+  my $line = 1 + length $s;
+  print "${cfname}:${line}: $match\n";
+}
diff --git a/scripts/gendeclarations.lua b/scripts/gendeclarations.lua
new file mode 100755
index 0000000000..311ed8b527
--- /dev/null
+++ b/scripts/gendeclarations.lua
@@ -0,0 +1,232 @@
+#!/usr/bin/lua
+
+local fname = arg[1]
+local static_fname = arg[2]
+local non_static_fname = arg[3]
+local cpp = arg[4]
+
+cpp = cpp:gsub(' %-DINCLUDE_GENERATED_DECLARATIONS ', ' ')
+
+local lpeg = require('lpeg')
+
+local fold = function (func, ...)
+  local result = nil
+  for i, v in ipairs({...}) do
+    if result == nil then
+      result = v
+    else
+      result = func(result, v)
+    end
+  end
+  return result
+end
+
+local folder = function (func)
+  return function (...)
+    return fold(func, ...)
+  end
+end
+
+local lit = lpeg.P
+local set = function(...)
+  return lpeg.S(fold(function (a, b) return a .. b end, ...))
+end
+local any_character = lpeg.P(1)
+local rng = function(s, e) return lpeg.R(s .. e) end
+local concat = folder(function (a, b) return a * b end)
+local branch = folder(function (a, b) return a + b end)
+local one_or_more = function(v) return v ^ 1 end
+local two_or_more = function(v) return v ^ 2 end
+local any_amount = function(v) return v ^ 0 end
+local one_or_no = function(v) return v ^ -1 end
+local look_behind = lpeg.B
+local look_ahead = function(v) return #v end
+local neg_look_ahead = function(v) return -v end
+local neg_look_behind = function(v) return -look_behind(v) end
+
+local w = branch(
+  rng('a', 'z'),
+  rng('A', 'Z'),
+  lit('_')
+)
+local aw = branch(
+  w,
+  rng('0', '9')
+)
+local s = set(' ', '\n', '\t')
+local raw_word = concat(w, any_amount(aw))
+local right_word = concat(
+  raw_word,
+  neg_look_ahead(aw)
+)
+local word = concat(
+  neg_look_behind(aw),
+  right_word
+)
+local spaces = any_amount(branch(
+  s,
+  -- Comments are really handled by preprocessor, so the following is not needed
+  concat(
+    lit('/*'),
+    any_amount(concat(
+      neg_look_ahead(lit('*/')),
+      any_character
+    )),
+    lit('*/')
+  ),
+  concat(
+    lit('//'),
+    any_amount(concat(
+      neg_look_ahead(lit('\n')),
+      any_character
+    )),
+    lit('\n')
+  )
+))
+local typ_part = concat(
+  word,
+  any_amount(concat(
+    spaces,
+    lit('*')
+  )),
+  spaces
+)
+local typ = one_or_more(typ_part)
+local typ_id = two_or_more(typ_part)
+local arg = typ_id         -- argument name is swallowed by typ
+local pattern = concat(
+  typ_id,                  -- return type with function name
+  spaces,
+  lit('('),
+  spaces,
+  one_or_no(branch(        -- function arguments
+    concat(
+      arg,                 -- first argument, does not require comma
+      any_amount(concat(   -- following arguments, start with a comma
+        spaces,
+        lit(','),
+        spaces,
+        arg,
+        any_amount(concat(
+          lit('['),
+          spaces,
+          any_amount(aw),
+          spaces,
+          lit(']')
+        ))
+      )),
+      one_or_no(concat(
+        spaces,
+        lit(','),
+        spaces,
+        lit('...')
+      ))
+    ),
+    lit('void')            -- also accepts just void
+  )),
+  spaces,
+  lit(')'),
+  any_amount(concat(       -- optional attributes
+    spaces,
+    lit('FUNC_ATTR_'),
+    any_amount(aw),
+    one_or_no(concat(      -- attribute argument
+      spaces,
+      lit('('),
+      any_amount(concat(
+        neg_look_ahead(lit(')')),
+        any_character
+      )),
+      lit(')')
+    ))
+  )),
+  look_ahead(concat(       -- definition must be followed by "{"
+    spaces,
+    lit('{')
+  ))
+)
+
+if fname == '--help' then
+  print'Usage:'
+  print()
+  print'  gendeclarations.lua definitions.c static.h non-static.h "cc -E …"'
+  os.exit()
+end
+
+local pipe = io.popen(cpp .. ' -DDO_NOT_DEFINE_EMPTY_ATTRIBUTES ' .. fname, 'r')
+local text = pipe:read('*a')
+if not pipe:close() then
+  os.exit(2)
+end
+
+local header = [[
+#ifndef DEFINE_FUNC_ATTRIBUTES
+# define DEFINE_FUNC_ATTRIBUTES
+#endif
+#include "nvim/func_attr.h"
+#undef DEFINE_FUNC_ATTRIBUTES
+]]
+
+local footer = [[
+#include "nvim/func_attr.h"
+]]
+
+local non_static = header
+local static = header
+
+local filepattern = '^# %d+ "[^"]-/?([^"/]+)"'
+local curfile
+
+init = 0
+curfile = nil
+neededfile = fname:match('[^/]+$')
+while init ~= nil do
+  init = text:find('\n', init)
+  if init == nil then
+    break
+  end
+  init = init + 1
+  if text:sub(init, init) == '#' then
+    file = text:match(filepattern, init)
+    if file ~= nil then
+      curfile = file
+    end
+  elseif curfile == neededfile then
+    s = init
+    e = pattern:match(text, init)
+    if e ~= nil then
+      local declaration = text:sub(s, e - 1)
+      -- Comments are really handled by preprocessor, so the following is not 
+      -- needed
+      declaration = declaration:gsub('/%*.-%*/', '')
+      declaration = declaration:gsub('//.-\n', '\n')
+
+      declaration = declaration:gsub('\n', ' ')
+      declaration = declaration:gsub('%s+', ' ')
+      declaration = declaration:gsub(' ?%( ?', '(')
+      declaration = declaration:gsub(' ?%) ?', ')')
+      declaration = declaration:gsub(' ?, ?', ', ')
+      declaration = declaration:gsub(' ?(%*+) ?', ' %1')
+      declaration = declaration:gsub(' ?(FUNC_ATTR_)', ' %1')
+      declaration = declaration:gsub(' $', '')
+      declaration = declaration .. ';\n'
+      if text:sub(s, s + 5) == 'static' then
+        static = static .. declaration
+      else
+        non_static = non_static .. declaration
+      end
+    end
+  end
+end
+
+non_static = non_static .. footer
+static = static .. footer
+
+local F
+F = io.open(static_fname, 'w')
+F:write(static)
+F:close()
+
+F = io.open(non_static_fname, 'w')
+F:write(non_static)
+F:close()
diff --git a/scripts/movedocs.pl b/scripts/movedocs.pl
new file mode 100755
index 0000000000..923c633f13
--- /dev/null
+++ b/scripts/movedocs.pl
@@ -0,0 +1,180 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+if ($ARGV[0] eq '--help') {
+  print << "EOF";
+Usage:
+
+  $0 file.h file.c
+
+Removes documentation attached to function declarations in file.h and adds them 
+to function definitions found in file.c.
+
+  $0 file.c
+
+Moves documentation attached to function declaration present in the same file as 
+the definition.
+EOF
+  exit 0;
+}
+
+my $hfile = shift @ARGV;
+my @cfiles = @ARGV;
+
+my %docs = ();
+my $F;
+
+sub write_lines {
+  my $file = shift;
+  my @lines = @_;
+
+  my $F;
+
+  open $F, '>', $file;
+  print $F (join "", @lines);
+  close $F;
+}
+
+if (@cfiles) {
+  open $F, '<', $hfile
+    or die "Failed to open $hfile.";
+
+  my @hlines = ();
+
+  my $lastdoc = '';
+
+  while (<$F>) {
+    if (/^\/\/\/?/) {
+      $lastdoc .= $_;
+    } elsif (/^\S.*?(\w+)\(.*(?:,|\);?|FUNC_ATTR_\w+;?)$/) {
+      die "Documentation for $1 was already defined" if (defined $docs{$1});
+      if ($lastdoc ne '') {
+        $docs{$1} = $lastdoc;
+        $lastdoc = '';
+      }
+      push @hlines, $_;
+    } elsif ($lastdoc ne '') {
+      push @hlines, $lastdoc;
+      $lastdoc = '';
+      push @hlines, $_;
+    } else {
+      push @hlines, $_;
+    }
+  }
+
+  close $F;
+
+  my %clines_hash = ();
+
+  for my $cfile (@cfiles) {
+    open $F, '<', $cfile
+      or die "Failed to open $cfile.";
+
+    my @clines = ();
+
+    while (<$F>) {
+      if (/^\S.*?(\w+)\(.*[,)]$/ and defined $docs{$1}) {
+        push @clines, $docs{$1};
+        delete $docs{$1};
+      } elsif (/^(?!static\s)\S.*?(\w+)\(.*[,)]$/ and not defined $docs{$1}) {
+        print STDERR "Documentation not defined for $1\n";
+      }
+      push @clines, $_;
+    }
+
+    close $F;
+
+    $clines_hash{$cfile} = \@clines;
+  }
+
+  while (my ($func, $value) = each %docs) {
+    die "Function not found: $func\n";
+  }
+
+  write_lines($hfile, @hlines);
+  while (my ($cfile, $clines) = each %clines_hash) {
+    write_lines($cfile, @$clines);
+  }
+} else {
+  open $F, '<', $hfile;
+
+  my @lines;
+
+  my $lastdoc = '';
+  my $defstart = '';
+  my $funcname;
+
+  sub clear_lastdoc {
+    if ($lastdoc ne '') {
+      push @lines, $lastdoc;
+      $lastdoc = '';
+    }
+  }
+
+  sub record_lastdoc {
+    my $funcname = shift;
+    if ($lastdoc ne '') {
+      $docs{$funcname} = $lastdoc;
+      $lastdoc = '';
+    }
+  }
+
+  sub add_doc {
+    my $funcname = shift;
+    if (defined $docs{$funcname}) {
+      push @lines, $docs{$funcname};
+      delete $docs{$funcname};
+    }
+  }
+
+  sub clear_defstart {
+    push @lines, $defstart;
+    $defstart = '';
+  }
+
+  while (<$F>) {
+    if (/\/\*/ .. /\*\// and not /\/\*.*?\*\//) {
+      push @lines, $_;
+    } elsif (/^\/\/\/?/) {
+      $lastdoc .= $_;
+    } elsif (/^\S.*?(\w+)\(.*(?:,|(\);?))$/) {
+      if (not $2) {
+        $defstart .= $_;
+        $funcname = $1;
+      } elsif ($2 eq ');') {
+        record_lastdoc $1;
+        push @lines, $_;
+      } elsif ($2 eq ')') {
+        clear_lastdoc;
+        add_doc $1;
+        push @lines, $_;
+      }
+    } elsif ($defstart ne '') {
+      $defstart .= $_;
+      if (/[{}]/) {
+        clear_lastdoc;
+        clear_defstart;
+      } elsif (/\);$/) {
+        record_lastdoc $funcname;
+        clear_defstart;
+      } elsif (/\)$/) {
+        clear_lastdoc;
+        add_doc $funcname;
+        clear_defstart;
+      }
+    } else {
+      clear_lastdoc;
+      push @lines, $_;
+    }
+  }
+
+  close $F;
+
+  while (my ($func, $value) = each %docs) {
+    die "Function not found: $func\n";
+  }
+
+  write_lines($hfile, @lines);
+}
diff --git a/scripts/msgpack-gen.lua b/scripts/msgpack-gen.lua
index 8d7bb8ea59..c8a09c8e96 100644
--- a/scripts/msgpack-gen.lua
+++ b/scripts/msgpack-gen.lua
@@ -94,7 +94,9 @@ output:write([[
 ]])
 
 for i = 1, #headers do
-  output:write('\n#include "nvim/'..headers[i]..'"')
+  if headers[i]:sub(-12) ~= '.generated.h' then
+    output:write('\n#include "nvim/'..headers[i]..'"')
+  end
 end
 
 output:write([[
diff --git a/scripts/stripdecls.py b/scripts/stripdecls.py
new file mode 100755
index 0000000000..34be2a1578
--- /dev/null
+++ b/scripts/stripdecls.py
@@ -0,0 +1,141 @@
+#!/usr/bin/python
+# vim: set fileencoding=utf-8:
+
+from __future__ import print_function, unicode_literals, division
+
+from clang.cindex import Index, CursorKind
+from collections import namedtuple, OrderedDict, defaultdict
+import sys
+import os
+
+
+DECL_KINDS = {
+  CursorKind.FUNCTION_DECL,
+}
+
+
+Strip = namedtuple('Strip', 'start_line start_column end_line end_column')
+
+
+def main(progname, cfname, only_static, move_all):
+  only_static = False
+
+  cfname = os.path.abspath(os.path.normpath(cfname))
+
+  hfname1 = os.path.splitext(cfname)[0] + os.extsep + 'h'
+  hfname2 = os.path.splitext(cfname)[0] + '_defs' + os.extsep + 'h'
+
+  files_to_modify = (cfname, hfname1, hfname2)
+
+  index = Index.create()
+  src_dirname = os.path.join(os.path.dirname(__file__), '..', 'src')
+  src_dirname = os.path.abspath(os.path.normpath(src_dirname))
+  relname = os.path.join(src_dirname, 'nvim')
+  unit = index.parse(cfname, args=('-I' + src_dirname,
+                                   '-DUNIX',
+                                   '-DEXITFREE',
+                                   '-DFEAT_USR_CMDS',
+                                   '-DFEAT_CMDL_COMPL',
+                                   '-DFEAT_COMPL_FUNC',
+                                   '-DPROTO',
+                                   '-DUSE_MCH_ERRMSG'))
+  cursor = unit.cursor
+
+  tostrip = defaultdict(OrderedDict)
+  definitions = set()
+
+  for child in cursor.get_children():
+    if not (child.location and child.location.file):
+      continue
+    fname = os.path.abspath(os.path.normpath(child.location.file.name))
+    if fname not in files_to_modify:
+      continue
+    if child.kind not in DECL_KINDS:
+      continue
+    if only_static and next(child.get_tokens()).spelling == 'static':
+      continue
+
+    if child.is_definition() and fname == cfname:
+      definitions.add(child.spelling)
+    else:
+      stripdict = tostrip[fname]
+      assert(child.spelling not in stripdict)
+      stripdict[child.spelling] = Strip(
+        child.extent.start.line,
+        child.extent.start.column,
+        child.extent.end.line,
+        child.extent.end.column,
+      )
+
+  for (fname, stripdict) in tostrip.items():
+    if not move_all:
+      for name in set(stripdict) - definitions:
+        stripdict.pop(name)
+
+    if not stripdict:
+      continue
+
+    if fname.endswith('.h'):
+      is_h_file = True
+      include_line = next(reversed(stripdict.values())).start_line + 1
+    else:
+      is_h_file = False
+      include_line = next(iter(stripdict.values())).start_line
+
+    lines = None
+    generated_existed = os.path.exists(fname + '.generated.h')
+    with open(fname, 'rb') as F:
+      lines = list(F)
+
+    stripped = []
+
+    for name, position in reversed(stripdict.items()):
+      sl = slice(position.start_line - 1, position.end_line)
+      if is_h_file:
+        include_line -= sl.stop - sl.start
+      stripped += lines[sl]
+      lines[sl] = ()
+
+    if not generated_existed:
+      lines[include_line:include_line] = [
+        '#ifdef INCLUDE_GENERATED_DECLARATIONS\n',
+        '# include "{0}.generated.h"\n'.format(os.path.relpath(fname, relname)),
+        '#endif\n',
+      ]
+
+    with open(fname, 'wb') as F:
+      F.writelines(lines)
+
+
+if __name__ == '__main__':
+  progname = sys.argv[0]
+  args = sys.argv[1:]
+  if not args or '--help' in args:
+    print('Usage:')
+    print('')
+    print('  {0} [--static [--all]] file.c...'.format(progname))
+    print('')
+    print('Stripts all declarations from file.c, file.h and file_defs.h.')
+    print('If --static argument is given then only static declarations are')
+    print('stripped. Declarations are stripped only if corresponding')
+    print('definition is found unless --all argument was given.')
+    print('')
+    print('Note: it is assumed that static declarations starts with "static"')
+    print('      keyword.')
+    sys.exit(0 if args else 1)
+
+  if args[0] == '--static':
+    only_static = True
+    args = args[1:]
+  else:
+    only_static = False
+
+  if args[0] == '--all':
+    move_all = True
+    args = args[1:]
+  else:
+    move_all = False
+
+  for cfname in args:
+    print('Processing {0}'.format(cfname))
+    main(progname, cfname, only_static, move_all)