[reactos] 42/74: [TOOLS] Add a python script for syncing wine patches

[Jérôme Gardou]

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=f149b8ce86a2bfeb328f6…
commit f149b8ce86a2bfeb328f620fa10caf821f2ee271
Author:     Jérôme Gardou <jerome.gardou(a)reactos.org>
AuthorDate: Sat Jan 4 02:14:02 2020 +0100
Commit:     Jérôme Gardou <zefklop(a)users.noreply.github.com>
CommitDate: Wed Feb 26 18:19:18 2020 +0100
    [TOOLS] Add a python script for syncing wine patches
    This requires the pygit2 module
    usage : ./winesync.py module <wine-tag> <winestaging-tag>
    for instance ./winesync.py d3dx9 wine-4.1 v4.1
    This requires to have a wine git checkout and a wine-staging checkout
    configuration is done through yaml file named <module>.cfg specifying
    the following:
     - file mappings
     - directory mappings
     - latest wine version the module is synced with
    it then creates a local branch in the wine checkout, based on the given
    tag, and then the staging script is ran on top of it.
    Thanks to the mappings defined in the module configuration file, it then
    create individual commits in the reactos git checkout with reworked
    path.
    In case of problem, it stops and lets you amend the latest commit and go
    along with the process once this is done. (Makefile.in modified, new or
    removed files, patches not cleanly applied)
    Staging patches are added into the <module>_staging directory for the
    ease of reverting them later. (TODO)
    See previous [WINESYNC] commits to have an overview of the result
---
 .gitignore                     |   1 +
 sdk/tools/winesync/winesync.py | 354 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 355 insertions(+)
diff --git a/.gitignore b/.gitignore
index 125424b56af..e7288b7e643 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@ modules/AHK_Tests
 .settings
 build
 .vscode
+sdk/tools/winesync/winesync.cfg
diff --git a/sdk/tools/winesync/winesync.py b/sdk/tools/winesync/winesync.py
new file mode 100644
index 00000000000..a63eb406049
--- /dev/null
+++ b/sdk/tools/winesync/winesync.py
@@ -0,0 +1,354 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import string
+import argparse
+import subprocess
+import fnmatch
+import pygit2
+import yaml
+
+def string_to_valid_file_name(to_convert):
+    valid_chars = '-_.()' + string.ascii_letters + string.digits
+    result = ''
+    for c in to_convert:
+        if c in valid_chars:
+            result += c
+        else:
+            result += '_'
+    # strip final dot, if any
+    if result.endswith('.'):
+        return result[:-1]
+    return result
+
+class wine_sync:
+    def __init__(self, module):
+        if os.path.isfile('winesync.cfg'):
+            with open('winesync.cfg', 'r') as file_input:
+                config = yaml.load(file_input, Loader=yaml.FullLoader)
+            self.reactos_src = config['repos']['reactos']
+            self.wine_src = config['repos']['wine']
+            self.wine_staging_src = config['repos']['wine-staging']
+        else:
+            config = { }
+            self.reactos_src = input('Please enter the path to the reactos git tree:
')
+            self.wine_src = input('Please enter the path to the wine git tree: ')
+            self.wine_staging_src = input('Please enter the path to the wine-staging
git tree: ')
+            config['repos'] = { 'reactos' : self.reactos_src,
+                                'wine': self.wine_src,
+                                'wine-staging': self.wine_staging_src }
+            with open('winesync.cfg', 'w') as file_output:
+                yaml.dump(config, file_output)
+
+        self.wine_repo = pygit2.Repository(self.wine_src)
+        self.wine_staging_repo = pygit2.Repository(self.wine_staging_src)
+        self.reactos_repo = pygit2.Repository(self.reactos_src)
+
+        # read the index from the reactos tree
+        self.reactos_index = self.reactos_repo.index
+        self.reactos_index.read()
+
+        # get the actual state for the asked module
+        self.module = module
+        with open(module + '.cfg', 'r') as file_input:
+            self.module_cfg = yaml.load(file_input, Loader=yaml.FullLoader)
+
+        self.staged_patch_dir = os.path.join('sdk', 'tools',
'winesync', self.module + '_staging')
+
+    def create_or_checkout_wine_branch(self, wine_tag, wine_staging_tag):
+        wine_branch_name = 'winesync-' + wine_tag + '-' +
wine_staging_tag
+        branch = self.wine_repo.lookup_branch(wine_branch_name)
+        if branch is None:
+            # get our target commits
+            wine_target_commit = self.wine_repo.revparse_single(wine_tag)
+            if isinstance(wine_target_commit, pygit2.Tag):
+                wine_target_commit = wine_target_commit.target
+
+            wine_staging_target_commit =
self.wine_staging_repo.revparse_single(wine_staging_tag)
+            if isinstance(wine_staging_target_commit, pygit2.Tag):
+                wine_staging_target_commit = wine_staging_target_commit.target
+
+            self.wine_repo.branches.local.create(wine_branch_name,
self.wine_repo.revparse_single('HEAD'))
+            self.wine_repo.checkout(self.wine_repo.lookup_branch(wine_branch_name))
+            self.wine_repo.reset(wine_target_commit, pygit2.GIT_RESET_HARD)
+
+            # do the same for the wine-staging tree
+            self.wine_staging_repo.branches.local.create(wine_branch_name,
self.wine_staging_repo.revparse_single('HEAD'))
+
self.wine_staging_repo.checkout(self.wine_staging_repo.lookup_branch(wine_branch_name))
+            self.wine_staging_repo.reset(wine_staging_target_commit,
pygit2.GIT_RESET_HARD)
+
+            # run the wine-staging script
+            subprocess.call(['bash', '-c', self.wine_staging_src +
'/patches/patchinstall.sh DESTDIR=' + self.wine_src + ' --all
--backend=git-am'])
+
+            # delete the branch we created
+
self.wine_staging_repo.checkout(self.wine_staging_repo.lookup_branch('master'))
+            self.wine_staging_repo.branches.delete(wine_branch_name)
+        else:
+            self.wine_repo.checkout(self.wine_repo.lookup_branch(wine_branch_name))
+
+        return wine_branch_name
+
+    # helper function for resolving wine tree path to reactos one
+    # Note : it doesn't care about the fact that the file actually exists or not
+    def wine_to_reactos_path(self, wine_path):
+        if wine_path in self.module_cfg['files']:
+            # we have a direct mapping
+            return self.module_cfg['files'][wine_path]
+
+        if not '/' in wine_path:
+            # root files should have a direct mapping
+            return None
+
+        wine_dir, wine_file = os.path.split(wine_path)
+        if wine_dir in self.module_cfg['directories']:
+            # we have a mapping for the directory
+            return os.path.join(self.module_cfg['directories'][wine_dir],
wine_file)
+
+        # no match
+        return None
+
+    def sync_wine_commit(self, wine_commit, in_staging, staging_patch_index):
+        # Get the diff object
+        diff = self.wine_repo.diff(wine_commit.parents[0], wine_commit)
+
+        modified_files = []
+        ignored_files = []
+        warning_message = ''
+        complete_patch = ''
+
+        if in_staging:
+            # see if we already applied this patch
+            patch_file_name =
f'{staging_patch_index:04}-{string_to_valid_file_name(wine_commit.message.splitlines()[0])}.diff'
+            patch_path = os.path.join(self.reactos_src, self.staged_patch_dir,
patch_file_name)
+            if os.path.isfile(patch_path):
+                print(f'Skipping patch as {patch_path} already exists')
+                return True, ''
+
+        for delta in diff.deltas:
+            if delta.old_file.path == '/dev/null':
+                # check if we should care
+                new_reactos_path = self.wine_to_reactos_path(delta.new_file.path)
+                if not new_reactos_path is None:
+                    warning_message += 'file ' + delta.new_file.path + ' is
added to the wine tree !\n'
+                    old_reactos_path = '/dev/null'
+                else:
+                    old_reactos_path = None
+            elif delta.new_file.path == '/dev/null':
+                # check if we should care
+                old_reactos_path = self.wine_to_reactos_path(delta.old_file.path)
+                if not old_reactos_path is None:
+                    warning_message += 'file ' + delta.old_file.path + ' is
removed from the wine tree !\n'
+                    new_reactos_path = '/dev/null'
+                else:
+                    new_reactos_path = None
+            elif delta.new_file.path.endswith('Makefile.in'):
+                warning_message += 'file ' + delta.new_file.path + ' was
modified !\n'
+                # no need to warn that those are ignored, we just did.
+                continue
+            else:
+                new_reactos_path = self.wine_to_reactos_path(delta.new_file.path)
+                old_reactos_path = self.wine_to_reactos_path(delta.old_file.path)
+
+            if (new_reactos_path is not None) or (old_reactos_path is not None):
+                # print('Must apply diff: ' + old_reactos_path + ' -->
' + new_reactos_path)
+                new_blob = self.wine_repo.get(wine_commit.tree[delta.new_file.path].id)
+                old_blob =
self.wine_repo.get(wine_commit.parents[0].tree[delta.old_file.path].id)
+
+                blob_patch = pygit2.Patch.create_from(
+                    old=old_blob,
+                    new=new_blob,
+                    old_as_path=old_reactos_path,
+                    new_as_path=new_reactos_path)
+
+                # print(str(wine_commit.id))
+                # print(blob_patch.text)
+
+                # this doesn't work
+                # reactos_diff = pygit2.Diff.parse_diff(blob_patch.text)
+                # reactos_repo.apply(reactos_diff)
+                try:
+                    subprocess.run(['git', '-C', self.reactos_src,
'apply', '--reject'], input=blob_patch.data, check=True)
+                except subprocess.CalledProcessError as err:
+                    warning_message += 'Error while applying patch to ' +
new_reactos_path + '\n'
+                self.reactos_index.add(new_reactos_path)
+
+                complete_patch += blob_patch.text
+
+                modified_files += [delta.old_file.path, delta.new_file.path]
+            else:
+                ignored_files += [delta.old_file.path, delta.new_file.path]
+
+        if not modified_files:
+            # We applied nothing
+            return False, ''
+
+        print('Applied patches from wine commit ' + str(wine_commit.id))
+
+        if ignored_files:
+            warning_message += 'WARNING : some files were ignored: ' + '
'.join(ignored_files) + '\n'
+
+        if not in_staging:
+            self.module_cfg['tags']['wine'] = str(wine_commit.id)
+            with open(self.module + '.cfg', 'w') as file_output:
+                yaml.dump(self.module_cfg, file_output)
+            self.reactos_index.add(f'sdk/tools/winesync/{self.module}.cfg')
+        else:
+            # Add the staging patch
+            # do not save the wine commit ID in <module>.cfg, as it's a local
one for staging patches
+            if not os.path.isdir(os.path.join(self.reactos_src, self.staged_patch_dir)):
+                os.mkdir(os.path.join(self.reactos_src, self.staged_patch_dir))
+            with open(patch_path, 'w') as file_output:
+                file_output.write(complete_patch)
+            self.reactos_index.add(os.path.join(self.staged_patch_dir, patch_file_name))
+
+        self.reactos_index.write()
+
+        commit_msg = f'[WINESYNC] {wine_commit.message}\n'
+        if (in_staging):
+            commit_msg += f'wine-staging patch by {wine_commit.author.name}
<{wine_commit.author.email}>'
+        else:
+            commit_msg += f'wine commit id {str(wine_commit.id)} by
{wine_commit.author.name} <{wine_commit.author.email}>'
+
+        self.reactos_repo.create_commit('HEAD',
+            pygit2.Signature('winesync', &#39;ros-dev(a)reactos.org&#39;)rg'),
+            self.reactos_repo.default_signature,
+            commit_msg,
+            self.reactos_index.write_tree(),
+            [self.reactos_repo.head.target])
+
+        if (warning_message != ''):
+            warning_message += 'If needed, amend the current commit in your reactos
tree and start this script again'
+
+            if not in_staging:
+                warning_message += f'You can see the details of the wine commit here:
https://source.winehq.org/git/wine.git/commit/{str(wine_commit.id)}'
+            else:
+                warning_message += 'Do not forget to run\n'
+                warning_message += f'git diff HEAD^
\':(exclude)sdk/tools/winesync/{patch_file_name}\' >
sdk/tools/winesync/{patch_file_name}\n'
+                warning_message += 'after your correction and then\n'
+                warning_message += f'git add
sdk/tools/winesync/{patch_file_name}\n'
+                warning_message += 'before running "git commit
--amend"'
+
+        return True, warning_message
+
+    def revert_staged_patchset(self):
+        # revert all of this in one commmit
+        staged_patch_dir_path = os.path.join(self.reactos_src, self.staged_patch_dir)
+        if not os.path.isdir(staged_patch_dir_path):
+            return True
+
+        has_patches = False
+
+        for patch_file_name in sorted(os.listdir(staged_patch_dir_path), reverse=True):
+            patch_path = os.path.join(staged_patch_dir_path, patch_file_name)
+            if not os.path.isfile(patch_path):
+                continue
+
+            has_patches = True
+
+            with open(patch_path, 'rb') as patch_file:
+                try:
+                    subprocess.run(['git', '-C', self.reactos_src,
'apply', '-R', '--reject'], stdin=patch_file, check=True)
+                except subprocess.CalledProcessError as err:
+                    print(f'Error while reverting patch {patch_file_name}')
+                    print('Please check, remove the offending patch with git rm, and
relaunch this script')
+                    return False
+
+            self.reactos_index.remove(os.path.join(self.staged_patch_dir,
patch_file_name))
+            self.reactos_index.write()
+            os.remove(patch_path)
+
+        if not has_patches:
+            return True
+
+        self.reactos_index.add_all([f for f in
self.module_cfg['files'].values()])
+        self.reactos_index.add_all([f'{d}/*.*' for d in
self.module_cfg['directories'].values()])
+        self.reactos_index.write()
+
+        self.reactos_repo.create_commit(
+            'HEAD',
+            self.reactos_repo.default_signature,
+            self.reactos_repo.default_signature,
+            f'[WINESYNC]: revert wine-staging patchset for {self.module}',
+            self.reactos_index.write_tree(),
+            [self.reactos_repo.head.target])
+        return True
+
+    def sync_to_wine(self, wine_tag, wine_staging_tag):
+        # Get our target commit
+        wine_target_commit = self.wine_repo.revparse_single(wine_tag)
+        if isinstance(wine_target_commit, pygit2.Tag):
+            wine_target_commit = wine_target_commit.target
+        # print(f'wine target commit is {wine_target_commit}')
+
+        # get the wine commit id where we left
+        in_staging = False
+        wine_last_sync =
self.wine_repo.revparse_single(self.module_cfg['tags']['wine'])
+        if (isinstance(wine_last_sync, pygit2.Tag)):
+            if not self.revert_staged_patchset():
+                return
+            wine_last_sync = wine_last_sync.target
+        if (isinstance(wine_last_sync, pygit2.Commit)):
+            wine_last_sync = wine_last_sync.id
+
+        # create a branch to keep things clean
+        wine_branch_name = self.create_or_checkout_wine_branch(wine_tag,
wine_staging_tag)
+
+        finished_sync = True
+        staging_patch_index = 1
+
+        # walk each commit between last sync and the asked tag/revision
+        wine_commit_walker = self.wine_repo.walk(self.wine_repo.head.target,
pygit2.GIT_SORT_TOPOLOGICAL | pygit2.GIT_SORT_REVERSE)
+        wine_commit_walker.hide(wine_last_sync)
+        for wine_commit in wine_commit_walker:
+            applied_patch, warning_message = self.sync_wine_commit(wine_commit,
in_staging, staging_patch_index)
+
+            if str(wine_commit.id) == str(wine_target_commit):
+                print('We are now in staging territory')
+                in_staging = True
+
+            if not applied_patch:
+                continue
+
+            if in_staging:
+                staging_patch_index += 1
+
+            if warning_message != '':
+                print(warning_message)
+                finished_sync = False
+                break
+
+        # we're done without error
+        if finished_sync:
+            # update wine tag and commit
+            self.module_cfg['tags']['wine'] = wine_tag
+            with open(self.module + '.cfg', 'w') as file_output:
+                yaml.dump(self.module_cfg, file_output)
+
+            self.reactos_index.add(f'sdk/tools/winesync/{self.module}.cfg')
+            self.reactos_index.write()
+            self.reactos_repo.create_commit(
+                'HEAD',
+                self.reactos_repo.default_signature,
+                self.reactos_repo.default_signature,
+                f'[WINESYNC]: {self.module} is now in sync with wine-staging
{wine_tag}',
+                self.reactos_index.write_tree(),
+                [self.reactos_repo.head.target])
+
+            print('The branch ' + wine_branch_name + ' was created in your
wine repository. You might want to delete it, but you should keep it in case you want to
sync more module up to this wine version')
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('module', help='The module you want to sync.
<module>.cfg must exist in the current directory')
+    parser.add_argument('wine_tag', help='The wine tag or commit id to sync
to')
+    parser.add_argument('wine_staging_tag', help='The wine staging tag or
commit id to pick wine staged patches from')
+
+    args = parser.parse_args()
+
+    syncator = wine_sync(args.module)
+
+    return syncator.sync_to_wine(args.wine_tag, args.wine_staging_tag)
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file