backports: add full kernel integration support

This enables support for using the backports project
to integrate device drivers from a future version of Linux
into an older version of Linux. What you end up seeing is
a backports submenu when configuring your kernel and the
ability to select specific device drivers from subsystems
supported through the Linux backports project.

At this time enabling one device driver from a future version
of Linux will require using only the latest version of the
subsystem modules and other subsystem drivers. For example
enabling cfg80211 and mac80211 from a future version of Linux
will require you to only use future version of the respective
device drivers. In order to enable the backported version of
802.11 drivers for example, you will have to enable first:

Networking support -->
  Wireless -->

But under that menu disable all options, then jump to the backports
submenu to now enable:

Backports -->
  cfg80211
  mac80211
  Wireless LAN  --->
    etc

You build these device drivers modular or built-in to the kernel.
Integration support requires only slight modifications to the original
kernel sources, one to the top level Kconfig to add our entry, and also
the top level Makefile to enable backports code to be part of the
built-in vmlinux.

Support for integration takes advantage over the existing infrastructure
added by Johannes to keep track of each indvidual change done by the
backports infrastructure if --gitdebug is used.

mcgrof@drvbp1 ~/backports (git::master)$ time ./gentree.py --clean \
	/home/mcgrof/linux-next /home/mcgrof/build/backports-20141023
Copy original source files ...
Applying patches from patches to /home/mcgrof/build/backports-20141023
...
Modify Kconfig tree ...
Rewrite Makefiles and Kconfig files ...
Done!

real    1m27.942s
user    13m23.752s
sys     0m47.608s

1   3.0.101             [  OK  ]
2   3.1.10              [  OK  ]
3   3.2.62              [  OK  ]
4   3.3.8               [  OK  ]
5   3.4.104             [  OK  ]
6   3.5.7               [  OK  ]
7   3.6.11              [  OK  ]
8   3.7.10              [  OK  ]
9   3.8.13              [  OK  ]
10  3.9.11              [  OK  ]
11  3.10.58             [  OK  ]
12  3.11.10             [  OK  ]
13  3.12.31             [  OK  ]
14  3.13.11             [  OK  ]
15  3.14.22             [  OK  ]
16  3.15.10             [  OK  ]
17  3.16.6              [  OK  ]
18  3.17.1              [  OK  ]
19  3.18-rc1            [  OK  ]

real    42m44.838s
user    1190m5.092s
sys     140m37.208s

Signed-off-by: Luis R. Rodriguez <mcgrof@suse.com>

1 files changed, 66 insertions, 6 deletions

diff --git a/gentree.py b/gentree.py
index 90334ef7..00e2d46f 100755
--- a/gentree.py
+++ b/gentree.py
@@ -16,6 +16,7 @@ from lib import bpgpg as gpg
 from lib import bpkup as kup
 from lib.tempdir import tempdir
 from lib import bpreqs as reqs
+from lib import bpversion as gen_version
 
 class Bp_Identity(object):
     """
@@ -233,6 +234,9 @@ def add_automatic_backports(args):
     export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)')
     bpi = kconfig.get_backport_info(os.path.join(args.bpid.target_dir, 'compat', 'Kconfig'))
     configtree = kconfig.ConfigTree(os.path.join(args.bpid.target_dir, 'Kconfig'), args.bpid)
+    ignore=['Kconfig.kernel', 'Kconfig.versions']
+    configtree.verify_sources(ignore=ignore)
+    git_debug_snapshot(args, "verify sources for automatic backports")
     all_selects = configtree.all_selects()
     for sym, vals in bpi.items():
         if sym.startswith('BPAUTO_BUILD_'):
@@ -640,6 +644,9 @@ def _main():
                              'and we use git ls-tree to get the files.')
     parser.add_argument('--clean', const=True, default=False, action="store_const",
                         help='Clean output directory instead of erroring if it isn\'t empty')
+    parser.add_argument('--integrate', const=True, default=False, action="store_const",
+                        help='Integrate a future backported kernel solution into ' +
+                             'an older kernel tree source directory.')
     parser.add_argument('--refresh', const=True, default=False, action="store_const",
                         help='Refresh patches as they are applied, the source dir will be modified!')
     parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
@@ -679,9 +686,8 @@ def _main():
     # same prefix for packaging as with kernel integration but
     # there are already some users of the CPTCFG prefix.
     bpid = None
-    integrate = False
-    if integrate:
-        bpid = Bp_Identity(integrate = integrate,
+    if args.integrate:
+        bpid = Bp_Identity(integrate = args.integrate,
                            kconfig_prefix = 'CONFIG_',
                            project_prefix = 'BACKPORT_',
                            project_dir = args.outdir,
@@ -690,7 +696,7 @@ def _main():
                            kconfig_source_var = '$BACKPORT_DIR',
                            )
     else:
-        bpid = Bp_Identity(integrate = integrate,
+        bpid = Bp_Identity(integrate = args.integrate,
                            kconfig_prefix = 'CPTCFG_',
                            project_prefix = '',
                            project_dir = args.outdir,
@@ -764,6 +770,11 @@ def process(kerneldir, copy_list_file, git_revision=None,
                 test_cocci, profile_cocci)
     rel_prep = None
 
+    if bpid.integrate:
+        if args.kup_test or args.test_cocci or args.profile_cocci or args.refresh:
+            logwrite('Cannot use integration with:\n\tkup_test\n\ttest_cocci\n\tprofile_cocci\n\trefresh\n');
+            sys.exit(1)
+
     # start processing ...
     if (args.kup or args.kup_test):
         git_paranoia(source_dir, logwrite)
@@ -798,6 +809,10 @@ def process(kerneldir, copy_list_file, git_revision=None,
     check_output_dir(bpid.target_dir, args.clean)
 
     # do the copy
+    backport_integrate_files = [
+            ('Makefile.kernel', 'Makefile'),
+            ('Kconfig.integrate', 'Kconfig'),
+            ]
     backport_package_files = [(x, x) for x in [
         'Makefile',
         'kconf/',
@@ -819,6 +834,8 @@ def process(kerneldir, copy_list_file, git_revision=None,
 
     if not bpid.integrate:
         backport_files += backport_package_files
+    else:
+        backport_files += backport_integrate_files
 
     if not args.git_revision:
         logwrite('Copy original source files ...')
@@ -888,6 +905,21 @@ def process(kerneldir, copy_list_file, git_revision=None,
 
     apply_patches(args, "backport", source_dir, 'patches', bpid.target_dir, logwrite)
 
+    # Kernel integration requires Kconfig.versions already generated for you,
+    # we cannot do this for a package as we have no idea what kernel folks
+    # will be using.
+    if bpid.integrate:
+        kver = gen_version.kernelversion(bpid.project_dir)
+        rel_specs = gen_version.get_rel_spec_stable(kver)
+        if not rel_specs:
+            logwrite('Cannot parse source kernel version, update parser')
+            sys.exit(1)
+        data = gen_version.genkconfig_versions(rel_specs)
+        fo = open(os.path.join(bpid.target_dir, 'Kconfig.versions'), 'w')
+        fo.write(data)
+        fo.close()
+        git_debug_snapshot(args, "generate kernel version requirement Kconfig file")
+
     # some post-processing is required
     configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid)
     ignore=['Kconfig.kernel', 'Kconfig.versions']
@@ -905,6 +937,15 @@ def process(kerneldir, copy_list_file, git_revision=None,
         configtree.force_tristate_modular()
         git_debug_snapshot(args, "force tristate options modular")
 
+    ignore = [os.path.join(bpid.target_dir, x) for x in [
+                'Kconfig.package.hacks',
+                'Kconfig.versions',
+                'Kconfig',
+                ]
+            ]
+    configtree.adjust_backported_configs(ignore=ignore, orig_symbols=orig_symbols)
+    git_debug_snapshot(args, "adjust backports config symbols we port")
+
     configtree.modify_selects()
     git_debug_snapshot(args, "convert select to depends on")
 
@@ -944,8 +985,15 @@ def process(kerneldir, copy_list_file, git_revision=None,
 
     # rewrite Makefile and source symbols
 
+    # symbols we know only we can provide under the backport project prefix
+    # for which we need an exception.
+    skip_orig_syms = [ bpid.project_prefix + x for x in [
+            'INTEGRATE',
+            ]
+    ]
+    parse_orig_syms = [x for x in orig_symbols if x not in skip_orig_syms ]
     regexes = []
-    for some_symbols in [orig_symbols[i:i + 50] for i in range(0, len(orig_symbols), 50)]:
+    for some_symbols in [parse_orig_syms[i:i + 50] for i in range(0, len(parse_orig_syms), 50)]:
         r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
         regexes.append(re.compile(r, re.MULTILINE))
     for root, dirs, files in os.walk(bpid.target_dir):
@@ -967,7 +1015,10 @@ def process(kerneldir, copy_list_file, git_revision=None,
     git_debug_snapshot(args, "rename config symbol / srctree usage")
 
     # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
-    maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel'))
+    if bpid.integrate:
+        maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile'))
+    else:
+        maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel'))
     disable_kconfig = []
     disable_makefile = []
     for sym in maketree.get_impossible_symbols():
@@ -1017,6 +1068,15 @@ def process(kerneldir, copy_list_file, git_revision=None,
         fo.close()
     git_debug_snapshot(args, "disable unsatisfied Makefile parts")
 
+    if bpid.integrate:
+        f = open(os.path.join(bpid.project_dir, 'Kconfig'), 'a')
+        f.write('source "backports/Kconfig"\n')
+        f.close()
+        git_debug_snapshot(args, "hooked backport to top level Kconfig")
+
+        apply_patches(args, "integration", source_dir, 'integration-patches/',
+                      bpid.project_dir, logwrite)
+
     if (args.kup or args.kup_test):
         req = reqs.Req()
         req.kup()