02/03: [APISETS] Introduce a script that will generate apisets based on wine apisets. Functions that are present in ReactOS will be forwarded, the rest stubbed. CORE-13231

[Mark Jansen]

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=3ff8f1bb09e6cbc46329c…

commit 3ff8f1bb09e6cbc46329ce368fbe7ded4733e95d
Author:     Mark Jansen <mark.jansen(a)reactos.org>
AuthorDate: Sat Apr 7 03:24:07 2018 +0200
Commit:     Mark Jansen <mark.jansen(a)reactos.org>
CommitDate: Sat Apr 7 14:50:58 2018 +0200

    [APISETS] Introduce a script that will generate apisets based on wine apisets.
    Functions that are present in ReactOS will be forwarded, the rest stubbed.
    CORE-13231
---
 dll/CMakeLists.txt            |   1 +
 dll/apisets/CMakeLists.txt.in |  39 ++++
 dll/apisets/update.py         | 456 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 496 insertions(+)

diff --git a/dll/CMakeLists.txt b/dll/CMakeLists.txt
index 2e913af571..c9f4f72195 100644
--- a/dll/CMakeLists.txt
+++ b/dll/CMakeLists.txt
@@ -1,5 +1,6 @@
 
 add_subdirectory(3rdparty)
+#add_subdirectory(apisets)
 add_subdirectory(appcompat)
 add_subdirectory(cpl)
 add_subdirectory(directx)
diff --git a/dll/apisets/CMakeLists.txt.in b/dll/apisets/CMakeLists.txt.in
new file mode 100644
index 0000000000..2399d21bf1
--- /dev/null
+++ b/dll/apisets/CMakeLists.txt.in
@@ -0,0 +1,39 @@
+
+# This file is generated by update.py, please edit CMakeLists.txt.in instead
+# Generated from %WINE_GIT_VERSION%
+
+project(apisets)
+
+function (add_apiset apiset_name baseaddress)
+    spec2def(${apiset_name}.dll ${apiset_name}.spec ADD_IMPORTLIB)
+
+    add_definitions(
+        -D_CTYPE_DISABLE_MACROS
+        -D_NO_INLINING
+        -D__CRT__NO_INLINE
+        -D__STDC_WANT_SECURE_LIB__=0
+        -D_INC_STRING
+        -D_CTYPE_DEFINED
+        -D_WCTYPE_DEFINED
+        -D_CRT_ERRNO_DEFINED)
+
+    add_library(${apiset_name} SHARED
+        ${CMAKE_CURRENT_BINARY_DIR}/${apiset_name}_stubs.c
+        ${CMAKE_CURRENT_BINARY_DIR}/${apiset_name}.def)
+
+    add_dependencies(${apiset_name} xdk)
+    set_module_type(${apiset_name} win32dll ENTRYPOINT 0 UNICODE IMAGEBASE
${baseaddress})
+
+    if(NOT MSVC)
+        add_target_compile_flags(${apiset_name} "-fno-builtin")
+    else()
+        add_target_compile_flags(${apiset_name} "/wd4026 /wd4273")
+    endif()
+
+    add_importlibs(${apiset_name} ${ARGN} kernel32 ntdll)
+
+    add_cd_file(TARGET ${apiset_name} DESTINATION reactos/system32 FOR all)
+endfunction()
+
+# Apisets will be appended
+
diff --git a/dll/apisets/update.py b/dll/apisets/update.py
new file mode 100644
index 0000000000..2d7ce387a7
--- /dev/null
+++ b/dll/apisets/update.py
@@ -0,0 +1,456 @@
+'''
+PROJECT:     ReactOS apisets generator
+LICENSE:     MIT (https://spdx.org/licenses/MIT)
+PURPOSE:     Create apiset forwarders based on Wine apisets
+COPYRIGHT:   Copyright 2017,2018 Mark Jansen (mark.jansen(a)reactos.org)
+'''
+
+import os
+import re
+import sys
+from collections import defaultdict
+import subprocess
+
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+IGNORE_OPTIONS = ('-norelay', '-ret16', '-ret64',
'-register', '-private',
+                  '-noname', '-ordinal', '-i386',
'-arch=', '-stub')
+
+# Figure these out later
+FUNCTION_BLACKLIST = [
+    # api-ms-win-crt-utility-l1-1-0_stubs.c(6):
+    # error C2169: '_abs64': intrinsic function, cannot be defined
+    '_abs64',
+    '_byteswap_uint64', '_byteswap_ulong', '_byteswap_ushort',
+    '_rotl64', '_rotr64',
+]
+
+SPEC_HEADER = [
+    '\n',
+    '# This file is autogenerated by update.py\n',
+    '\n'
+]
+
+
+class InvalidSpecError(Exception):
+    def __init__(self, message):
+        Exception.__init__(self, message)
+
+class Arch(object):
+    none = 0
+    i386 = 1
+    x86_64 = 2
+    arm = 4
+    arm64 = 8
+    Any = i386 | x86_64 | arm | arm64
+
+    FROM_STR = {
+        'i386': i386,
+        'x86_64': x86_64,
+        'arm': arm,
+        'arm64': arm64,
+        'any': Any,
+        'win32': i386,
+        'win64': x86_64,
+    }
+
+    TO_STR = {
+        i386: 'i386',
+        x86_64: 'x86_64',
+        arm: 'arm',
+        arm64: 'arm64',
+    }
+
+    def __init__(self, initial=none):
+        self._val = initial
+
+    def add(self, text):
+        self._val |= sum([Arch.FROM_STR[arch] for arch in text.split(',')])
+        assert self._val != 0
+
+    def has(self, val):
+        return (self._val & val) != 0
+
+    def to_str(self):
+        arch_str = []
+        for value in Arch.TO_STR:
+            if value & self._val:
+                arch_str.append(Arch.TO_STR[value])
+        return ','.join(arch_str)
+
+    def __len__(self):
+        return bin(self._val).count("1")
+
+    def __add__(self, other):
+        return Arch(self._val | other._val) # pylint: disable=W0212
+
+    def __sub__(self, other):
+        return Arch(self._val & ~other._val) # pylint: disable=W0212
+
+    def __gt__(self, other):
+        return self._val > other._val       # pylint: disable=W0212
+
+    def __lt__(self, other):
+        return self._val < other._val       # pylint: disable=W0212
+
+    def __eq__(self, other):
+        return self._val == other._val      # pylint: disable=W0212
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+ALIAS_DLL = {
+    'ucrtbase': 'msvcrt',
+    'kernelbase': 'kernel32',
+    'shcore': 'shlwapi',
+    'combase': 'ole32',
+
+    # These modules cannot be linked against in ROS, so forward it
+    'cfgmgr32': 'setupapi', # Forward everything
+    'wmi': 'advapi32',      # Forward everything
+}
+
+class SpecEntry(object):
+    def __init__(self, text, spec):
+        self.spec = spec
+        self._ord = None
+        self.callconv = None
+        self.name = None
+        self.arch = Arch()
+        self._forwarder = None
+        self.init(text)
+        self.noname = False
+        if self.name == '@':
+            self.noname = True
+            if self._forwarder:
+                self.name = self._forwarder[1]
+
+    def init(self, text):
+        tokens = re.split(r'([\s\(\)#;])', text.strip())
+        tokens = [token for token in tokens if token and not token.isspace()]
+        idx = []
+        for comment in ['#', ';']:
+            if comment in tokens:
+                idx.append(tokens.index(comment))
+        idx = sorted(idx)
+        if idx:
+            tokens = tokens[:idx[0]]
+        if not tokens:
+            raise InvalidSpecError(text)
+        self._ord = tokens[0]
+        assert self._ord == '@' or self._ord.isdigit(), text
+        tokens = tokens[1:]
+        self.callconv = tokens.pop(0)
+        self.name = tokens.pop(0)
+        while self.name.startswith(IGNORE_OPTIONS):
+            if self.name.startswith('-arch='):
+                self.arch.add(self.name[6:])
+            elif self.name == '-i386':
+                self.arch.add('i386')
+            self.name = tokens.pop(0)
+        if not self.arch:
+            self.arch = Arch(Arch.Any)
+        assert not self.name.startswith('-'), text
+        if not tokens:
+            return
+        if tokens[0] == '(':
+            assert ')' in tokens, text
+            arg = tokens.pop(0)
+            while True:
+                arg = tokens.pop(0)
+                if arg == ')':
+                    break
+        if not tokens:
+            return
+        assert len(tokens) == 1, text
+        self._forwarder = tokens.pop(0).split('.', 2)
+        if len(self._forwarder) == 1:
+            self._forwarder = ['self', self._forwarder[0]]
+        assert len(self._forwarder) in (0, 2), self._forwarder
+        if self._forwarder[0] in ALIAS_DLL:
+            self._forwarder[0] = ALIAS_DLL[self._forwarder[0]]
+
+    def resolve_forwarders(self, module_lookup, try_modules):
+        if self._forwarder:
+            assert self._forwarder[1] == self.name,
'{}:{}'.format(self._forwarder[1], self.name)
+        if self.noname and self.name == '@':
+            return 0    # cannot search for this function
+        self._forwarder = []
+        self.arch = Arch()
+        for module_name in try_modules:
+            assert module_name in module_lookup, module_name
+            module = module_lookup[module_name]
+            fwd_arch = module.find_arch(self.name)
+            callconv = module.find_callconv(self.name)
+            if fwd_arch:
+                self.arch = fwd_arch
+                self._forwarder = [module_name, self.name]
+                self.callconv = callconv
+                return 1
+        return 0
+
+    def extra_forwarders(self, function_lookup, module_lookup):
+        if self._forwarder:
+            return 1
+        if self.noname and self.name == '@':
+            return 0    # cannot search for this function
+        lst = function_lookup.get(self.name, None)
+        if lst:
+            modules = list(set([func.spec.name for func in lst]))
+            if len(modules) > 1:
+                mod = None
+                arch = Arch()
+                for module in modules:
+                    mod_arch = module_lookup[module].find_arch(self.name)
+                    if mod is None or mod_arch > arch:
+                        mod = module
+                        arch = mod_arch
+                modules = [mod]
+            mod = modules[0]
+            self._forwarder = [mod, self.name]
+            mod = module_lookup[mod]
+            self.arch = mod.find_arch(self.name)
+            self.callconv = mod.find_callconv(self.name)
+            return 1
+        return 0
+
+    def forwarder_module(self):
+        if self._forwarder:
+            return self._forwarder[0]
+
+    def forwarder(self):
+        if self._forwarder:
+            return 1
+        return 0
+
+    def write(self, spec_file):
+        name = self.name
+        opts = ''
+        estimate_size = 0
+        if self.noname:
+            opts = '{} -noname'.format(opts)
+        if self.name == '@':
+            assert self._ord != '@'
+            name = 'Ordinal' + self._ord
+        if not self._forwarder:
+            spec_file.write('{} stub{} {}\n'.format(self._ord, opts, name))
+            estimate_size += 0x1000
+        else:
+            assert self.arch != Arch(), self.name
+            args = '()'
+            callconv = 'stdcall'
+            fwd = '.'.join(self._forwarder)
+            name = self.name if not self.noname else '@'
+            arch = self.arch
+            if self.callconv == 'extern':
+                args = ''
+                callconv = 'extern'
+                if arch.has(Arch.x86_64):
+                    fwd = '{}.__imp_{}'.format(*self._forwarder)
+                    self.arch = arch - Arch(Arch.x86_64)
+                    estimate_size += self.write(spec_file)
+                    self.arch = arch
+                    arch = Arch(Arch.x86_64)
+                else:
+                    fwd = '{}._imp__{}'.format(*self._forwarder)
+            if arch != Arch(Arch.Any):
+                opts = '{} -arch={}'.format(opts, arch.to_str())
+            spec_file.write('{ord} {cc}{opts} {name}{args}
{fwd}\n'.format(ord=self._ord,
+                                                                           cc=callconv,
+                                                                           opts=opts,
+                                                                           name=name,
+                                                                           args=args,
+                                                                           fwd=fwd))
+            estimate_size += 0x100
+        return estimate_size
+
+
+
+class SpecFile(object):
+    def __init__(self, fullpath, name):
+        self._path = fullpath
+        self.name = name
+        self._entries = []
+        self._functions = defaultdict(list)
+        self._estimate_size = 0
+
+    def parse(self):
+        with open(self._path, 'rb') as specfile:
+            for line in specfile.readlines():
+                if line:
+                    try:
+                        entry = SpecEntry(line, self)
+                        self._entries.append(entry)
+                        self._functions[entry.name].append(entry)
+                    except InvalidSpecError:
+                        pass
+        return (sum([entry.forwarder() for entry in self._entries]), len(self._entries))
+
+    def add_functions(self, function_lookup):
+        for entry in self._entries:
+            function_lookup[entry.name].append(entry)
+
+    def find(self, name):
+        return self._functions.get(name, None)
+
+    def find_arch(self, name):
+        functions = self.find(name)
+        arch = Arch()
+        if functions:
+            for func in functions:
+                arch += func.arch
+        return arch
+
+    def find_callconv(self, name):
+        functions = self.find(name)
+        callconv = None
+        if functions:
+            for func in functions:
+                if not callconv:
+                    callconv = func.callconv
+                elif callconv != func.callconv:
+                    assert callconv != 'extern', 'Cannot have data/function
with same name'
+                    callconv = func.callconv
+        return callconv
+
+    def resolve_forwarders(self, module_lookup):
+        modules = self.forwarder_modules()
+        total = 0
+        for entry in self._entries:
+            total += entry.resolve_forwarders(module_lookup, modules)
+        return (total, len(self._entries))
+
+    def extra_forwarders(self, function_lookup, module_lookup):
+        total = 0
+        for entry in self._entries:
+            total += entry.extra_forwarders(function_lookup, module_lookup)
+        return (total, len(self._entries))
+
+    def forwarder_modules(self):
+        modules = defaultdict(int)
+        for entry in self._entries:
+            module = entry.forwarder_module()
+            if module:
+                modules[module] += 1
+        return sorted(modules, key=modules.get, reverse=True)
+
+    def write(self, spec_file):
+        written = set(FUNCTION_BLACKLIST)
+        self._estimate_size = 0
+        for entry in self._entries:
+            if entry.name not in written:
+                self._estimate_size += entry.write(spec_file)
+                written.add(entry.name)
+
+    def write_cmake(self, cmakelists, baseaddress):
+        seen = set()
+        # ntdll and kernel32 are linked against everything, self = internal,
+        # we cannot link cfgmgr32 and wmi?
+        ignore = ['ntdll', 'kernel32', 'self',
'cfgmgr32', 'wmi']
+        forwarders = self.forwarder_modules()
+        fwd_strings = [x for x in forwarders if not (x in seen or x in ignore or
seen.add(x))]
+        fwd_strings = ' '.join(fwd_strings)
+        name = self.name
+        baseaddress = '0x{:8x}'.format(baseaddress)
+        cmakelists.write('add_apiset({} {} {})\n'.format(name, baseaddress,
fwd_strings))
+        return self._estimate_size
+
+
+
+def generate_specnames(dll_dir):
+    win32 = os.path.join(dll_dir, 'win32')
+    for dirname in os.listdir(win32):
+        fullpath = os.path.join(win32, dirname, dirname + '.spec')
+        if not os.path.isfile(fullpath):
+            if '.' in dirname:
+                fullpath = os.path.join(win32, dirname, dirname.rsplit('.', 1)[0]
+ '.spec')
+                if not os.path.isfile(fullpath):
+                    continue
+            else:
+                continue
+        yield (fullpath, dirname)
+    # Special cases
+    yield (os.path.join(dll_dir, 'ntdll', 'def', 'ntdll.spec'),
'ntdll')
+    yield (os.path.join(dll_dir, 'appcompat', 'apphelp',
'apphelp.spec'), 'apphelp')
+    yield (os.path.join(dll_dir, '..', 'win32ss', 'user',
'user32', 'user32.spec'), 'user32')
+    yield (os.path.join(dll_dir, '..', 'win32ss', 'gdi',
'gdi32', 'gdi32.spec'), 'gdi32')
+
+def run(wineroot):
+    wine_apisets = []
+    ros_modules = []
+
+    module_lookup = {}
+    function_lookup = defaultdict(list)
+
+    version = subprocess.check_output(["git", "describe"],
cwd=wineroot).strip()
+
+    print 'Reading Wine apisets for', version
+    wine_apiset_path = os.path.join(wineroot, 'dlls')
+    for dirname in os.listdir(wine_apiset_path):
+        if not dirname.startswith('api-'):
+            continue
+        if not os.path.isdir(os.path.join(wine_apiset_path, dirname)):
+            continue
+        fullpath = os.path.join(wine_apiset_path, dirname, dirname + '.spec')
+        spec = SpecFile(fullpath, dirname)
+        wine_apisets.append(spec)
+
+    print 'Parsing Wine apisets,',
+    total = (0, 0)
+    for apiset in wine_apisets:
+        total = tuple(map(sum, zip(apiset.parse(), total)))
+    print 'found', total[0], '/', total[1], 'forwarders'
+
+    print 'Reading ReactOS modules'
+    for fullpath, dllname in generate_specnames(os.path.dirname(SCRIPT_DIR)):
+        spec = SpecFile(fullpath, dllname)
+        ros_modules.append(spec)
+
+    print 'Parsing ReactOS modules'
+    for module in ros_modules:
+        module.parse()
+        assert module.name not in module_lookup, module.name
+        module_lookup[module.name] = module
+        module.add_functions(function_lookup)
+
+    print 'First pass, resolving forwarders,',
+    total = (0, 0)
+    for apiset in wine_apisets:
+        total = tuple(map(sum, zip(apiset.resolve_forwarders(module_lookup), total)))
+    print 'found', total[0], '/', total[1], 'forwarders'
+
+    print 'Second pass, searching extra forwarders,',
+    total = (0, 0)
+    for apiset in wine_apisets:
+        total = tuple(map(sum, zip(apiset.extra_forwarders(function_lookup,
module_lookup), total)))
+    print 'found', total[0], '/', total[1], 'forwarders'
+
+    print 'Writing apisets'
+    for apiset in wine_apisets:
+        with open(os.path.join(SCRIPT_DIR, apiset.name + '.spec'), 'wb')
as out_spec:
+            out_spec.writelines(SPEC_HEADER)
+            apiset.write(out_spec)
+
+    print 'Writing CMakeLists.txt'
+    with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt.in'), 'rb') as
template:
+        data = template.read()
+        data = data.replace('%WINE_GIT_VERSION%', version)
+    baseaddress = 0x60000000
+    with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt'), 'wb') as
cmakelists:
+        cmakelists.write(data)
+        for apiset in wine_apisets:
+            baseaddress += apiset.write_cmake(cmakelists, baseaddress)
+            baseaddress += (0x10000 - baseaddress) % 0x10000
+    print 'Done'
+
+def main(paths):
+    for path in paths:
+        if path:
+            run(path)
+            return
+    print 'No path specified,'
+    print 'either pass it as argument, or set the environment variable
"WINE_SRC_ROOT"'
+
+if __name__ == '__main__':
+    main(sys.argv[1:] + [os.environ.get('WINE_SRC_ROOT')])