7 Apr
2018
7 Apr
'18
1:24 p.m.
https://git.reactos.org/?p=reactos.git;a=commitdiff;h=3ff8f1bb09e6cbc46329c…
commit 3ff8f1bb09e6cbc46329ce368fbe7ded4733e95d
Author: Mark Jansen <mark.jansen(a)reactos.org>
AuthorDate: Sat Apr 7 03:24:07 2018 +0200
Commit: Mark Jansen <mark.jansen(a)reactos.org>
CommitDate: Sat Apr 7 14:50:58 2018 +0200
[APISETS] Introduce a script that will generate apisets based on wine apisets.
Functions that are present in ReactOS will be forwarded, the rest stubbed.
CORE-13231
---
dll/CMakeLists.txt | 1 +
dll/apisets/CMakeLists.txt.in | 39 ++++
dll/apisets/update.py | 456 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 496 insertions(+)
diff --git a/dll/CMakeLists.txt b/dll/CMakeLists.txt
index 2e913af571..c9f4f72195 100644
--- a/dll/CMakeLists.txt
+++ b/dll/CMakeLists.txt
@@ -1,5 +1,6 @@
add_subdirectory(3rdparty)
+#add_subdirectory(apisets)
add_subdirectory(appcompat)
add_subdirectory(cpl)
add_subdirectory(directx)
diff --git a/dll/apisets/CMakeLists.txt.in b/dll/apisets/CMakeLists.txt.in
new file mode 100644
index 0000000000..2399d21bf1
--- /dev/null
+++ b/dll/apisets/CMakeLists.txt.in
@@ -0,0 +1,39 @@
+
+# This file is generated by update.py, please edit CMakeLists.txt.in instead
+# Generated from %WINE_GIT_VERSION%
+
+project(apisets)
+
+function (add_apiset apiset_name baseaddress)
+ spec2def(${apiset_name}.dll ${apiset_name}.spec ADD_IMPORTLIB)
+
+ add_definitions(
+ -D_CTYPE_DISABLE_MACROS
+ -D_NO_INLINING
+ -D__CRT__NO_INLINE
+ -D__STDC_WANT_SECURE_LIB__=0
+ -D_INC_STRING
+ -D_CTYPE_DEFINED
+ -D_WCTYPE_DEFINED
+ -D_CRT_ERRNO_DEFINED)
+
+ add_library(${apiset_name} SHARED
+ ${CMAKE_CURRENT_BINARY_DIR}/${apiset_name}_stubs.c
+ ${CMAKE_CURRENT_BINARY_DIR}/${apiset_name}.def)
+
+ add_dependencies(${apiset_name} xdk)
+ set_module_type(${apiset_name} win32dll ENTRYPOINT 0 UNICODE IMAGEBASE
${baseaddress})
+
+ if(NOT MSVC)
+ add_target_compile_flags(${apiset_name} "-fno-builtin")
+ else()
+ add_target_compile_flags(${apiset_name} "/wd4026 /wd4273")
+ endif()
+
+ add_importlibs(${apiset_name} ${ARGN} kernel32 ntdll)
+
+ add_cd_file(TARGET ${apiset_name} DESTINATION reactos/system32 FOR all)
+endfunction()
+
+# Apisets will be appended
+
diff --git a/dll/apisets/update.py b/dll/apisets/update.py
new file mode 100644
index 0000000000..2d7ce387a7
--- /dev/null
+++ b/dll/apisets/update.py
@@ -0,0 +1,456 @@
+'''
+PROJECT: ReactOS apisets generator
+LICENSE: MIT (https://spdx.org/licenses/MIT)
+PURPOSE: Create apiset forwarders based on Wine apisets
+COPYRIGHT: Copyright 2017,2018 Mark Jansen (mark.jansen(a)reactos.org)
+'''
+
+import os
+import re
+import sys
+from collections import defaultdict
+import subprocess
+
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+IGNORE_OPTIONS = ('-norelay', '-ret16', '-ret64',
'-register', '-private',
+ '-noname', '-ordinal', '-i386',
'-arch=', '-stub')
+
+# Figure these out later
+FUNCTION_BLACKLIST = [
+ # api-ms-win-crt-utility-l1-1-0_stubs.c(6):
+ # error C2169: '_abs64': intrinsic function, cannot be defined
+ '_abs64',
+ '_byteswap_uint64', '_byteswap_ulong', '_byteswap_ushort',
+ '_rotl64', '_rotr64',
+]
+
+SPEC_HEADER = [
+ '\n',
+ '# This file is autogenerated by update.py\n',
+ '\n'
+]
+
+
+class InvalidSpecError(Exception):
+ def __init__(self, message):
+ Exception.__init__(self, message)
+
+class Arch(object):
+ none = 0
+ i386 = 1
+ x86_64 = 2
+ arm = 4
+ arm64 = 8
+ Any = i386 | x86_64 | arm | arm64
+
+ FROM_STR = {
+ 'i386': i386,
+ 'x86_64': x86_64,
+ 'arm': arm,
+ 'arm64': arm64,
+ 'any': Any,
+ 'win32': i386,
+ 'win64': x86_64,
+ }
+
+ TO_STR = {
+ i386: 'i386',
+ x86_64: 'x86_64',
+ arm: 'arm',
+ arm64: 'arm64',
+ }
+
+ def __init__(self, initial=none):
+ self._val = initial
+
+ def add(self, text):
+ self._val |= sum([Arch.FROM_STR[arch] for arch in text.split(',')])
+ assert self._val != 0
+
+ def has(self, val):
+ return (self._val & val) != 0
+
+ def to_str(self):
+ arch_str = []
+ for value in Arch.TO_STR:
+ if value & self._val:
+ arch_str.append(Arch.TO_STR[value])
+ return ','.join(arch_str)
+
+ def __len__(self):
+ return bin(self._val).count("1")
+
+ def __add__(self, other):
+ return Arch(self._val | other._val) # pylint: disable=W0212
+
+ def __sub__(self, other):
+ return Arch(self._val & ~other._val) # pylint: disable=W0212
+
+ def __gt__(self, other):
+ return self._val > other._val # pylint: disable=W0212
+
+ def __lt__(self, other):
+ return self._val < other._val # pylint: disable=W0212
+
+ def __eq__(self, other):
+ return self._val == other._val # pylint: disable=W0212
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ALIAS_DLL = {
+ 'ucrtbase': 'msvcrt',
+ 'kernelbase': 'kernel32',
+ 'shcore': 'shlwapi',
+ 'combase': 'ole32',
+
+ # These modules cannot be linked against in ROS, so forward it
+ 'cfgmgr32': 'setupapi', # Forward everything
+ 'wmi': 'advapi32', # Forward everything
+}
+
+class SpecEntry(object):
+ def __init__(self, text, spec):
+ self.spec = spec
+ self._ord = None
+ self.callconv = None
+ self.name = None
+ self.arch = Arch()
+ self._forwarder = None
+ self.init(text)
+ self.noname = False
+ if self.name == '@':
+ self.noname = True
+ if self._forwarder:
+ self.name = self._forwarder[1]
+
+ def init(self, text):
+ tokens = re.split(r'([\s\(\)#;])', text.strip())
+ tokens = [token for token in tokens if token and not token.isspace()]
+ idx = []
+ for comment in ['#', ';']:
+ if comment in tokens:
+ idx.append(tokens.index(comment))
+ idx = sorted(idx)
+ if idx:
+ tokens = tokens[:idx[0]]
+ if not tokens:
+ raise InvalidSpecError(text)
+ self._ord = tokens[0]
+ assert self._ord == '@' or self._ord.isdigit(), text
+ tokens = tokens[1:]
+ self.callconv = tokens.pop(0)
+ self.name = tokens.pop(0)
+ while self.name.startswith(IGNORE_OPTIONS):
+ if self.name.startswith('-arch='):
+ self.arch.add(self.name[6:])
+ elif self.name == '-i386':
+ self.arch.add('i386')
+ self.name = tokens.pop(0)
+ if not self.arch:
+ self.arch = Arch(Arch.Any)
+ assert not self.name.startswith('-'), text
+ if not tokens:
+ return
+ if tokens[0] == '(':
+ assert ')' in tokens, text
+ arg = tokens.pop(0)
+ while True:
+ arg = tokens.pop(0)
+ if arg == ')':
+ break
+ if not tokens:
+ return
+ assert len(tokens) == 1, text
+ self._forwarder = tokens.pop(0).split('.', 2)
+ if len(self._forwarder) == 1:
+ self._forwarder = ['self', self._forwarder[0]]
+ assert len(self._forwarder) in (0, 2), self._forwarder
+ if self._forwarder[0] in ALIAS_DLL:
+ self._forwarder[0] = ALIAS_DLL[self._forwarder[0]]
+
+ def resolve_forwarders(self, module_lookup, try_modules):
+ if self._forwarder:
+ assert self._forwarder[1] == self.name,
'{}:{}'.format(self._forwarder[1], self.name)
+ if self.noname and self.name == '@':
+ return 0 # cannot search for this function
+ self._forwarder = []
+ self.arch = Arch()
+ for module_name in try_modules:
+ assert module_name in module_lookup, module_name
+ module = module_lookup[module_name]
+ fwd_arch = module.find_arch(self.name)
+ callconv = module.find_callconv(self.name)
+ if fwd_arch:
+ self.arch = fwd_arch
+ self._forwarder = [module_name, self.name]
+ self.callconv = callconv
+ return 1
+ return 0
+
+ def extra_forwarders(self, function_lookup, module_lookup):
+ if self._forwarder:
+ return 1
+ if self.noname and self.name == '@':
+ return 0 # cannot search for this function
+ lst = function_lookup.get(self.name, None)
+ if lst:
+ modules = list(set([func.spec.name for func in lst]))
+ if len(modules) > 1:
+ mod = None
+ arch = Arch()
+ for module in modules:
+ mod_arch = module_lookup[module].find_arch(self.name)
+ if mod is None or mod_arch > arch:
+ mod = module
+ arch = mod_arch
+ modules = [mod]
+ mod = modules[0]
+ self._forwarder = [mod, self.name]
+ mod = module_lookup[mod]
+ self.arch = mod.find_arch(self.name)
+ self.callconv = mod.find_callconv(self.name)
+ return 1
+ return 0
+
+ def forwarder_module(self):
+ if self._forwarder:
+ return self._forwarder[0]
+
+ def forwarder(self):
+ if self._forwarder:
+ return 1
+ return 0
+
+ def write(self, spec_file):
+ name = self.name
+ opts = ''
+ estimate_size = 0
+ if self.noname:
+ opts = '{} -noname'.format(opts)
+ if self.name == '@':
+ assert self._ord != '@'
+ name = 'Ordinal' + self._ord
+ if not self._forwarder:
+ spec_file.write('{} stub{} {}\n'.format(self._ord, opts, name))
+ estimate_size += 0x1000
+ else:
+ assert self.arch != Arch(), self.name
+ args = '()'
+ callconv = 'stdcall'
+ fwd = '.'.join(self._forwarder)
+ name = self.name if not self.noname else '@'
+ arch = self.arch
+ if self.callconv == 'extern':
+ args = ''
+ callconv = 'extern'
+ if arch.has(Arch.x86_64):
+ fwd = '{}.__imp_{}'.format(*self._forwarder)
+ self.arch = arch - Arch(Arch.x86_64)
+ estimate_size += self.write(spec_file)
+ self.arch = arch
+ arch = Arch(Arch.x86_64)
+ else:
+ fwd = '{}._imp__{}'.format(*self._forwarder)
+ if arch != Arch(Arch.Any):
+ opts = '{} -arch={}'.format(opts, arch.to_str())
+ spec_file.write('{ord} {cc}{opts} {name}{args}
{fwd}\n'.format(ord=self._ord,
+ cc=callconv,
+ opts=opts,
+ name=name,
+ args=args,
+ fwd=fwd))
+ estimate_size += 0x100
+ return estimate_size
+
+
+
+class SpecFile(object):
+ def __init__(self, fullpath, name):
+ self._path = fullpath
+ self.name = name
+ self._entries = []
+ self._functions = defaultdict(list)
+ self._estimate_size = 0
+
+ def parse(self):
+ with open(self._path, 'rb') as specfile:
+ for line in specfile.readlines():
+ if line:
+ try:
+ entry = SpecEntry(line, self)
+ self._entries.append(entry)
+ self._functions[entry.name].append(entry)
+ except InvalidSpecError:
+ pass
+ return (sum([entry.forwarder() for entry in self._entries]), len(self._entries))
+
+ def add_functions(self, function_lookup):
+ for entry in self._entries:
+ function_lookup[entry.name].append(entry)
+
+ def find(self, name):
+ return self._functions.get(name, None)
+
+ def find_arch(self, name):
+ functions = self.find(name)
+ arch = Arch()
+ if functions:
+ for func in functions:
+ arch += func.arch
+ return arch
+
+ def find_callconv(self, name):
+ functions = self.find(name)
+ callconv = None
+ if functions:
+ for func in functions:
+ if not callconv:
+ callconv = func.callconv
+ elif callconv != func.callconv:
+ assert callconv != 'extern', 'Cannot have data/function
with same name'
+ callconv = func.callconv
+ return callconv
+
+ def resolve_forwarders(self, module_lookup):
+ modules = self.forwarder_modules()
+ total = 0
+ for entry in self._entries:
+ total += entry.resolve_forwarders(module_lookup, modules)
+ return (total, len(self._entries))
+
+ def extra_forwarders(self, function_lookup, module_lookup):
+ total = 0
+ for entry in self._entries:
+ total += entry.extra_forwarders(function_lookup, module_lookup)
+ return (total, len(self._entries))
+
+ def forwarder_modules(self):
+ modules = defaultdict(int)
+ for entry in self._entries:
+ module = entry.forwarder_module()
+ if module:
+ modules[module] += 1
+ return sorted(modules, key=modules.get, reverse=True)
+
+ def write(self, spec_file):
+ written = set(FUNCTION_BLACKLIST)
+ self._estimate_size = 0
+ for entry in self._entries:
+ if entry.name not in written:
+ self._estimate_size += entry.write(spec_file)
+ written.add(entry.name)
+
+ def write_cmake(self, cmakelists, baseaddress):
+ seen = set()
+ # ntdll and kernel32 are linked against everything, self = internal,
+ # we cannot link cfgmgr32 and wmi?
+ ignore = ['ntdll', 'kernel32', 'self',
'cfgmgr32', 'wmi']
+ forwarders = self.forwarder_modules()
+ fwd_strings = [x for x in forwarders if not (x in seen or x in ignore or
seen.add(x))]
+ fwd_strings = ' '.join(fwd_strings)
+ name = self.name
+ baseaddress = '0x{:8x}'.format(baseaddress)
+ cmakelists.write('add_apiset({} {} {})\n'.format(name, baseaddress,
fwd_strings))
+ return self._estimate_size
+
+
+
+def generate_specnames(dll_dir):
+ win32 = os.path.join(dll_dir, 'win32')
+ for dirname in os.listdir(win32):
+ fullpath = os.path.join(win32, dirname, dirname + '.spec')
+ if not os.path.isfile(fullpath):
+ if '.' in dirname:
+ fullpath = os.path.join(win32, dirname, dirname.rsplit('.', 1)[0]
+ '.spec')
+ if not os.path.isfile(fullpath):
+ continue
+ else:
+ continue
+ yield (fullpath, dirname)
+ # Special cases
+ yield (os.path.join(dll_dir, 'ntdll', 'def', 'ntdll.spec'),
'ntdll')
+ yield (os.path.join(dll_dir, 'appcompat', 'apphelp',
'apphelp.spec'), 'apphelp')
+ yield (os.path.join(dll_dir, '..', 'win32ss', 'user',
'user32', 'user32.spec'), 'user32')
+ yield (os.path.join(dll_dir, '..', 'win32ss', 'gdi',
'gdi32', 'gdi32.spec'), 'gdi32')
+
+def run(wineroot):
+ wine_apisets = []
+ ros_modules = []
+
+ module_lookup = {}
+ function_lookup = defaultdict(list)
+
+ version = subprocess.check_output(["git", "describe"],
cwd=wineroot).strip()
+
+ print 'Reading Wine apisets for', version
+ wine_apiset_path = os.path.join(wineroot, 'dlls')
+ for dirname in os.listdir(wine_apiset_path):
+ if not dirname.startswith('api-'):
+ continue
+ if not os.path.isdir(os.path.join(wine_apiset_path, dirname)):
+ continue
+ fullpath = os.path.join(wine_apiset_path, dirname, dirname + '.spec')
+ spec = SpecFile(fullpath, dirname)
+ wine_apisets.append(spec)
+
+ print 'Parsing Wine apisets,',
+ total = (0, 0)
+ for apiset in wine_apisets:
+ total = tuple(map(sum, zip(apiset.parse(), total)))
+ print 'found', total[0], '/', total[1], 'forwarders'
+
+ print 'Reading ReactOS modules'
+ for fullpath, dllname in generate_specnames(os.path.dirname(SCRIPT_DIR)):
+ spec = SpecFile(fullpath, dllname)
+ ros_modules.append(spec)
+
+ print 'Parsing ReactOS modules'
+ for module in ros_modules:
+ module.parse()
+ assert module.name not in module_lookup, module.name
+ module_lookup[module.name] = module
+ module.add_functions(function_lookup)
+
+ print 'First pass, resolving forwarders,',
+ total = (0, 0)
+ for apiset in wine_apisets:
+ total = tuple(map(sum, zip(apiset.resolve_forwarders(module_lookup), total)))
+ print 'found', total[0], '/', total[1], 'forwarders'
+
+ print 'Second pass, searching extra forwarders,',
+ total = (0, 0)
+ for apiset in wine_apisets:
+ total = tuple(map(sum, zip(apiset.extra_forwarders(function_lookup,
module_lookup), total)))
+ print 'found', total[0], '/', total[1], 'forwarders'
+
+ print 'Writing apisets'
+ for apiset in wine_apisets:
+ with open(os.path.join(SCRIPT_DIR, apiset.name + '.spec'), 'wb')
as out_spec:
+ out_spec.writelines(SPEC_HEADER)
+ apiset.write(out_spec)
+
+ print 'Writing CMakeLists.txt'
+ with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt.in'), 'rb') as
template:
+ data = template.read()
+ data = data.replace('%WINE_GIT_VERSION%', version)
+ baseaddress = 0x60000000
+ with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt'), 'wb') as
cmakelists:
+ cmakelists.write(data)
+ for apiset in wine_apisets:
+ baseaddress += apiset.write_cmake(cmakelists, baseaddress)
+ baseaddress += (0x10000 - baseaddress) % 0x10000
+ print 'Done'
+
+def main(paths):
+ for path in paths:
+ if path:
+ run(path)
+ return
+ print 'No path specified,'
+ print 'either pass it as argument, or set the environment variable
"WINE_SRC_ROOT"'
+
+if __name__ == '__main__':
+ main(sys.argv[1:] + [os.environ.get('WINE_SRC_ROOT')])